def format_listen_address(listen_address: Union[IPv4Address, IPv6Address, str]) -> str:
"""Format a listen_address.
Wraps IPv6 address in brackets.
Arguments:
listen_address {str} -- Preformatted listen_address
Returns:
{str} -- Formatted listen_address
"""
fmt = str(listen_address)
if isinstance(listen_address, str):
from ipaddress import ip_address
try:
listen_address = ip_address(listen_address)
except ValueError as err:
log.error(err)
pass
if (
isinstance(listen_address, (IPv4Address, IPv6Address))
and listen_address.version == 6
):
fmt = f"[{str(listen_address)}]"
return fmt
def validate_image(value):
"""Convert file path to URL path.
Arguments:
value {FilePath} -- Path to logo file.
Returns:
{str} -- Formatted logo path
"""
config_path = Path(os.environ["hyperglass_directory"])
base_path = [v for v in value.split("/") if v != ""]
if base_path[0] not in ("images", "custom"):
raise ValueError(
f"Logo files must be in the 'custom/' directory of your hyperglass directory. Got: {value}"
)
if base_path[0] == "custom":
file = config_path / "static" / "custom" / "/".join(base_path[1:])
else:
file = config_path / "static" / "images" / "/".join(base_path[1:])
log.error(file)
if not file.exists():
raise ValueError(f"'{str(file)}' does not exist")
base_index = file.parts.index(base_path[0])
return "/".join(file.parts[base_index:])
async def clear_cache():
"""Clear the Redis cache on shutdown."""
try:
await clear_redis_cache(db=params.cache.database, config=REDIS_CONFIG)
except RuntimeError as e:
log.error(str(e))
pass
def opener():
"""Set up an SSH tunnel according to a device's configuration."""
tunnel_kwargs = {
"ssh_username": proxy.credential.username,
"remote_bind_address": (self.device._target, self.device.port),
"local_bind_address": ("localhost", 0),
"skip_tunnel_checkup": False,
"gateway_timeout": params.request_timeout - 2,
}
if proxy.credential._method == "password":
# Use password auth if no key is defined.
tunnel_kwargs[
"ssh_password"] = proxy.credential.password.get_secret_value(
)
else:
# Otherwise, use key auth.
tunnel_kwargs["ssh_pkey"] = proxy.credential.key.as_posix()
if proxy.credential._method == "encrypted_key":
# If the key is encrypted, use the password field as the
# private key password.
tunnel_kwargs[
"ssh_private_key_password"] = proxy.credential.password.get_secret_value(
)
try:
return open_tunnel(proxy._target, proxy.port, **tunnel_kwargs)
except BaseSSHTunnelForwarderError as scrape_proxy_error:
log.error(f"Error connecting to device {self.device.name} via "
f"proxy {proxy.name}")
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.name,
proxy=proxy.name,
error=str(scrape_proxy_error),
)
def opener():
"""Set up an SSH tunnel according to a device's configuration."""
try:
return open_tunnel(
proxy._target,
proxy.port,
ssh_username=proxy.credential.username,
ssh_password=proxy.credential.password.get_secret_value(),
remote_bind_address=(self.device._target, self.device.port),
local_bind_address=("localhost", 0),
skip_tunnel_checkup=False,
gateway_timeout=params.request_timeout - 2,
)
except BaseSSHTunnelForwarderError as scrape_proxy_error:
log.error(
f"Error connecting to device {self.device.name} via "
f"proxy {proxy.name}"
)
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=proxy.name,
error=str(scrape_proxy_error),
)
async def build_ui(app_path):
"""Execute `next build` & `next export` from UI directory.
Raises:
RuntimeError: Raised if exit code is not 0.
RuntimeError: Raised when any other error occurs.
"""
try:
timeout = os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"]
log.info("Found UI build timeout environment variable: {}", timeout)
timeout = int(timeout)
except KeyError:
timeout = 90
ui_dir = Path(__file__).parent.parent / "ui"
build_dir = app_path / "static" / "ui"
build_command = "node_modules/.bin/next build"
export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir)
all_messages = []
for command in (build_command, export_command):
try:
proc = await asyncio.create_subprocess_shell(
cmd=command,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=ui_dir,
)
stdout, stderr = await asyncio.wait_for(proc.communicate(),
timeout=timeout)
messages = stdout.decode("utf-8").strip()
errors = stderr.decode("utf-8").strip()
if proc.returncode != 0:
raise RuntimeError(
f"\nMessages:\n{messages}\nErrors:\n{errors}")
await proc.wait()
all_messages.append(messages)
except asyncio.TimeoutError:
raise RuntimeError(
f"{timeout} second timeout exceeded while building UI")
except Exception as err:
log.error(err)
raise RuntimeError(str(err))
return "\n".join(all_messages)
def _validate_config(config: Union[Dict, List],
importer: Callable) -> HyperglassModel:
validated = None
try:
if isinstance(config, Dict):
validated = importer(**config)
elif isinstance(config, List):
validated = importer(config)
except ValidationError as err:
log.error(str(err))
raise ConfigInvalid(err.errors()) from None
return validated
def _parse_response(self, response):
if self.parse:
parsed = {}
try:
parsed = response.json()
except JSONDecodeError:
try:
parsed = _json.loads(response)
except (JSONDecodeError, TypeError):
log.error("Error parsing JSON for response {}",
repr(response))
parsed = {"data": response.text}
else:
parsed = response
return parsed
def __init__(
self,
message: str = "",
level: str = "warning",
keywords: Optional[List[str]] = None,
) -> None:
"""Initialize the hyperglass base exception class."""
self._message = message
self._level = level
self._keywords = keywords or []
if self._level == "warning":
log.error(repr(self))
elif self._level == "danger":
log.critical(repr(self))
else:
log.info(repr(self))
def __init__(self, message="", level="warning", keywords=None):
"""Initialize the hyperglass base exception class.
Keyword Arguments:
message {str} -- Error message (default: {""})
level {str} -- Error severity (default: {"warning"})
keywords {list} -- 'Important' keywords (default: {None})
"""
self._message = message
self._level = level
self._keywords = keywords or []
if self._level == "warning":
log.error(repr(self))
elif self._level == "danger":
log.critical(repr(self))
else:
log.info(repr(self))
def format_listen_address(
listen_address: Union[IPv4Address, IPv6Address, str]) -> str:
"""Format a listen_address. Wraps IPv6 address in brackets."""
fmt = str(listen_address)
if isinstance(listen_address, str):
try:
listen_address = ip_address(listen_address)
except ValueError as err:
log.error(err)
pass
if (isinstance(listen_address, (IPv4Address, IPv6Address))
and listen_address.version == 6):
fmt = f"[{str(listen_address)}]"
return fmt
def network_info_sync(resource: str):
"""Get ASN, Containing Prefix, and other info about an internet resource."""
data = {v: "" for v in REPLACE_KEYS.values()}
try:
if resource is not None:
whoisdata = run_whois_sync(resource)
if whoisdata:
# If the response is not empty, parse it.
data = parse_whois(whoisdata)
except Exception as err:
log.error(str(err))
return data
def network_info_sync(*targets: str) -> Dict[str, Dict[str, str]]:
"""Get ASN, Containing Prefix, and other info about an internet resource."""
targets = [str(t) for t in targets]
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
# Set default data structure.
data = {t: {k: "" for k in DEFAULT_KEYS} for t in targets}
# Get all cached bgp.tools data.
cached = cache.get_dict(CACHE_KEY)
# Try to use cached data for each of the items in the list of
# resources.
for t in targets:
if t in cached:
# Reassign the cached network info to the matching resource.
data[t] = cached[t]
log.debug("Using cached network info for {}", t)
# Remove cached items from the resource list so they're not queried.
targets = [t for t in targets if t not in cached]
try:
if targets:
whoisdata = run_whois_sync(targets)
if whoisdata:
# If the response is not empty, parse it.
data.update(parse_whois(whoisdata, targets))
# Cache the response
for t in targets:
cache.set_dict(CACHE_KEY, t, data[t])
log.debug("Cached network info for {}", t)
except Exception as err:
log.error(str(err))
return data
def __init__(
self,
message: str = "",
level: str = "warning",
keywords: Optional[List[str]] = None,
) -> None:
"""Initialize the hyperglass base exception class."""
self._message = message
self._level = level
self._keywords = keywords or []
if self._level == "warning":
log.error(repr(self))
elif self._level == "danger":
log.critical(repr(self))
else:
log.info(repr(self))
if all(sys.exc_info()):
# Rich will raise a ValueError if print_exception() is used
# outside of a try/except block. Only use Rich for traceback
# printing if the exception is caught.
console.print_exception(extra_lines=6)
async def send_webhook(query_data: Query, request: Request, timestamp: datetime):
"""If webhooks are enabled, get request info and send a webhook.
Args:
query_data (Query): Valid query
request (Request): Starlette/FastAPI request
Returns:
int: Returns 1 regardless of result
"""
try:
if params.logging.http is not None:
headers = await process_headers(headers=request.headers)
if headers.get("x-real-ip") is not None:
host = headers["x-real-ip"]
elif headers.get("x-forwarded-for") is not None:
host = headers["x-forwarded-for"]
else:
host = request.client.host
network_info = await bgptools.network_info(host)
async with Webhook(params.logging.http) as hook:
await hook.send(
query={
**query_data.export_dict(pretty=True),
"headers": headers,
"source": host,
"network": network_info.get(host, {}),
"timestamp": timestamp,
}
)
except Exception as err:
log.error(
"Error sending webhook to {}: {}", params.logging.http.provider, str(err)
)
def rpki_state(prefix, asn):
"""Get RPKI state and map to expected integer."""
log.debug("Validating RPKI State for {p} via AS{a}", p=prefix, a=asn)
state = 3
ro = f"{prefix}@{asn}"
cached = cache.get_dict(CACHE_KEY, ro)
if cached is not None:
state = cached
else:
ql = 'query GetValidation {{ validation(prefix: "{}", asn: {}) {{ state }} }}'
query = ql.format(prefix, asn)
try:
with BaseExternal(
base_url="https://rpki.cloudflare.com") as client:
response = client._post("/api/graphql", data={"query": query})
validation_state = (response.get("data",
{}).get("validation", {}).get(
"state", "DEFAULT"))
state = RPKI_STATE_MAP[validation_state]
cache.set_dict(CACHE_KEY, ro, state)
except Exception as err:
log.error(str(err))
state = 3
msg = "RPKI Validation State for {} via AS{} is {}".format(
prefix, asn, RPKI_NAME_MAP[state])
if cached is not None:
msg += " [CACHED]"
log.debug(msg)
return state
def error(*args, **kwargs):
msg = ", ".join(args)
kwargs = {k: str(v) for k, v in kwargs.items()}
error_msg = msg.format(**kwargs)
log.error(error_msg)
return RuntimeError(error_msg)
def validate_ip(value, query_type, query_vrf): # noqa: C901
"""Ensure input IP address is both valid and not within restricted allocations.
Arguments:
value {str} -- Unvalidated IP Address
query_type {str} -- Valid query type
query_vrf {object} -- Matched query vrf
Raises:
ValueError: Raised if input IP address is not an IP address.
ValueError: Raised if IP address is valid, but is within a restricted range.
Returns:
Union[IPv4Address, IPv6Address] -- Validated IP address object
"""
query_type_params = getattr(params.queries, query_type)
try:
# Attempt to use IP object factory to create an IP address object
valid_ip = ip_network(value)
except ValueError:
raise InputInvalid(
params.messages.invalid_input,
target=value,
query_type=query_type_params.display_name,
)
# Test the valid IP address to determine if it is:
# - Unspecified (See RFC5735, RFC2373)
# - Loopback (See RFC5735, RFC2373)
# - Otherwise IETF Reserved
# ...and returns an error if so.
if valid_ip.is_reserved or valid_ip.is_unspecified or valid_ip.is_loopback:
raise InputInvalid(
params.messages.invalid_input,
target=value,
query_type=query_type_params.display_name,
)
ip_version = valid_ip.version
vrf_afi = getattr(query_vrf, f"ipv{ip_version}")
if vrf_afi is None:
raise InputInvalid(
params.messages.feature_not_enabled,
feature=f"IPv{ip_version}",
device_name=f"VRF {query_vrf.display_name}",
)
for ace in [
a for a in vrf_afi.access_list if a.network.version == ip_version
]:
if _member_of(valid_ip, ace.network):
if query_type == "bgp_route" and _prefix_range(
valid_ip, ace.ge, ace.le):
pass
if ace.action == "permit":
log.debug("{t} is allowed by access-list {a}",
t=str(valid_ip),
a=repr(ace))
break
elif ace.action == "deny":
raise InputNotAllowed(
params.messages.acl_denied,
target=str(valid_ip),
denied_network=str(ace.network),
)
# Handling logic for host queries, e.g. 192.0.2.1 vs. 192.0.2.0/24
if valid_ip.num_addresses == 1:
# For a host query with ping or traceroute query types, convert
# the query_target to an IP address instead of a network.
if query_type in ("ping", "traceroute"):
new_ip = valid_ip.network_address
log.debug(
"Converted '{o}' to '{n}' for '{q}' query",
o=valid_ip,
n=new_ip,
q=query_type,
)
valid_ip = new_ip
# Get the containing prefix for a host query if:
# - Query type is bgp_route
# - force_cidr option is enabled
# - Query target is not a private address/network
elif (query_type in ("bgp_route", ) and vrf_afi.force_cidr
and not valid_ip.is_private):
log.debug("Getting containing prefix for {q}", q=str(valid_ip))
ip_str = str(valid_ip.network_address)
network_info = network_info_sync(ip_str)
containing_prefix = network_info.get(ip_str, {}).get("prefix")
if containing_prefix is None:
log.error(
"Unable to find containing prefix for {}. Got: {}",
str(valid_ip),
network_info,
)
raise InputInvalid("{q} does not have a containing prefix",
q=ip_str)
try:
valid_ip = ip_network(containing_prefix)
log.debug("Containing prefix: {p}", p=str(valid_ip))
except ValueError as err:
log.error(
"Unable to find containing prefix for {q}. Error: {e}",
q=str(valid_ip),
e=err,
)
raise InputInvalid(
"{q} does does not have a containing prefix", q=valid_ip)
# For a host query with bgp_route query type and force_cidr
# disabled, convert the host query to a single IP address.
elif query_type in ("bgp_route", ) and not vrf_afi.force_cidr:
valid_ip = valid_ip.network_address
log.debug("Validation passed for {ip}", ip=value)
return valid_ip
async def collect(self) -> Iterable: # noqa: C901
"""Connect to a device running hyperglass-agent via HTTP."""
log.debug("Query parameters: {}", self.query)
client_params = {
"headers": {
"Content-Type": "application/json"
},
"timeout": params.request_timeout,
}
if self.device.ssl is not None and self.device.ssl.enable:
with self.device.ssl.cert.open("r") as file:
cert = file.read()
if not cert:
raise RestError(
"SSL Certificate for device {d} has not been imported",
level="danger",
d=self.device.display_name,
)
http_protocol = "https"
client_params.update({"verify": str(self.device.ssl.cert)})
log.debug(
(f"Using {str(self.device.ssl.cert)} to validate connection "
f"to {self.device.name}"))
else:
http_protocol = "http"
endpoint = "{protocol}://{address}:{port}/query/".format(
protocol=http_protocol,
address=self.device._target,
port=self.device.port)
log.debug("URL endpoint: {}", endpoint)
try:
async with httpx.AsyncClient(**client_params) as http_client:
responses = ()
for query in self.query:
encoded_query = await jwt_encode(
payload=query,
secret=self.device.credential.password.
get_secret_value(),
duration=params.request_timeout,
)
log.debug("Encoded JWT: {}", encoded_query)
raw_response = await http_client.post(
endpoint, json={"encoded": encoded_query})
log.debug("HTTP status code: {}", raw_response.status_code)
raw = raw_response.text
log.debug("Raw Response:\n{}", raw)
if raw_response.status_code == 200:
decoded = await jwt_decode(
payload=raw_response.json()["encoded"],
secret=self.device.credential.password.
get_secret_value(),
)
log.debug("Decoded Response:\n{}", decoded)
responses += (decoded, )
elif raw_response.status_code == 204:
raise ResponseEmpty(
params.messages.no_output,
device_name=self.device.display_name,
)
else:
log.error(raw_response.text)
except httpx.exceptions.HTTPError as rest_error:
msg = parse_exception(rest_error)
log.error("Error connecting to device {}: {}", self.device.name,
msg)
raise RestError(
params.messages.connection_error,
device_name=self.device.display_name,
error=msg,
)
except OSError as ose:
log.critical(str(ose))
raise RestError(
params.messages.connection_error,
device_name=self.device.display_name,
error="System error",
)
except CertificateError as cert_error:
log.critical(str(cert_error))
msg = parse_exception(cert_error)
raise RestError(
params.messages.connection_error,
device_name=self.device.display_name,
error=f"{msg}: {cert_error}",
)
if raw_response.status_code != 200:
log.error("Response code is {}", raw_response.status_code)
raise RestError(
params.messages.connection_error,
device_name=self.device.display_name,
error=params.messages.general,
)
if not responses:
log.error("No response from device {}", self.device.name)
raise RestError(
params.messages.connection_error,
device_name=self.device.display_name,
error=params.messages.no_response,
)
return responses
async def collect(self, host: str = None, port: int = None) -> Sequence:
"""Connect directly to a device.
Directly connects to the router via Netmiko library, returns the
command output.
"""
driver = _map_driver(self.device.nos)
if host is not None:
log.debug(
"Connecting to {} via proxy {} [{}]",
self.device.name,
self.device.proxy.name,
f"{host}:{port}",
)
else:
log.debug("Connecting directly to {}", self.device.name)
global_args = driver_global_args.get(self.device.nos, {})
driver_kwargs = {
"host": host or self.device._target,
"port": port or self.device.port,
"auth_username": self.device.credential.username,
"timeout_transport": math.floor(params.request_timeout * 1.25),
"transport": "asyncssh",
"auth_strict_key": False,
"ssh_known_hosts_file": False,
"ssh_config_file": False,
**global_args,
}
if self.device.credential._method == "password":
# Use password auth if no key is defined.
driver_kwargs[
"auth_password"] = self.device.credential.password.get_secret_value(
)
else:
# Otherwise, use key auth.
driver_kwargs[
"auth_private_key"] = self.device.credential.key.as_posix()
if self.device.credential._method == "encrypted_key":
# If the key is encrypted, use the password field as the
# private key password.
driver_kwargs[
"auth_private_key_passphrase"] = self.device.credential.password.get_secret_value(
)
driver = driver(**driver_kwargs)
driver.logger = log.bind(logger_name=f"scrapli.driver-{driver._host}")
try:
responses = ()
async with driver as connection:
await connection.get_prompt()
for query in self.query:
raw = await connection.send_command(query)
responses += (raw.result, )
log.debug(
f'Raw response for command "{query}":\n{raw.result}')
except ScrapliTimeout as err:
log.error(err)
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.request_timeout,
)
except (ScrapliAuthenticationFailed, KeyVerificationFailed) as err:
log.error(
"Error authenticating to device {loc}: {e}",
loc=self.device.name,
e=str(err),
)
raise AuthError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.authentication_error,
)
except ScrapliException as err:
log.error(err)
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.no_response,
)
if not responses:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.no_response,
)
return responses
async def scrape_proxied(self):
"""Connect to a device via an SSH proxy.
Connects to the router via Netmiko library via the sshtunnel
library, returns the command output.
"""
log.debug(
f"Connecting to {self.device.proxy} via sshtunnel library...")
try:
tunnel = sshtunnel.open_tunnel(
self.device.proxy.address,
self.device.proxy.port,
ssh_username=self.device.proxy.credential.username,
ssh_password=self.device.proxy.credential.password.
get_secret_value(),
remote_bind_address=(self.device.address, self.device.port),
local_bind_address=("localhost", 0),
skip_tunnel_checkup=False,
gateway_timeout=params.request_timeout - 2,
)
except sshtunnel.BaseSSHTunnelForwarderError as scrape_proxy_error:
log.error(f"Error connecting to device {self.device.name} via "
f"proxy {self.device.proxy.name}")
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=self.device.proxy.name,
error=str(scrape_proxy_error),
)
def handle_timeout(*args, **kwargs):
tunnel.close()
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=self.device.proxy.name,
error=params.messages.request_timeout,
)
signal.signal(signal.SIGALRM, handle_timeout)
signal.alarm(params.request_timeout - 1)
with tunnel:
log.debug(
"Established tunnel with {d}. Local bind port: {p}",
d=self.device.proxy,
p=tunnel.local_bind_port,
)
scrape_host = {
"host": "localhost",
"port": tunnel.local_bind_port,
"device_type": self.device.nos,
"username": self.device.credential.username,
"password": self.device.credential.password.get_secret_value(),
**self.netmiko_args,
}
try:
log.debug("Connecting to {loc}...", loc=self.device.name)
nm_connect_direct = ConnectHandler(**scrape_host)
responses = ()
for query in self.query:
raw = nm_connect_direct.send_command(query)
responses += (raw, )
log.debug(f'Raw response for command "{query}":\n{raw}')
nm_connect_direct.disconnect()
except (NetMikoTimeoutException,
NetmikoTimeoutError) as scrape_error:
log.error(
"Timeout connecting to device {loc}: {e}",
loc=self.device.name,
e=str(scrape_error),
)
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=self.device.proxy.name,
error=params.messages.request_timeout,
)
except (NetMikoAuthenticationException,
NetmikoAuthError) as auth_error:
log.error(
"Error authenticating to device {loc}: {e}",
loc=self.device.name,
e=str(auth_error),
)
raise AuthError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=self.device.proxy.name,
error=params.messages.authentication_error,
) from None
except sshtunnel.BaseSSHTunnelForwarderError:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=self.device.proxy.name,
error=params.messages.general,
)
if not responses:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.no_response,
)
signal.alarm(0)
return await self.parsed_response(responses)
def __exit__(self, exc_type=None, exc_value=None, traceback=None):
"""Close connection on exit."""
if exc_type is not None:
log.error(traceback)
self._session.close()
async def build_frontend( # noqa: C901
dev_mode: bool,
dev_url: str,
prod_url: str,
params: Dict,
app_path: Path,
force: bool = False,
):
"""Perform full frontend UI build process.
Securely creates temporary file, writes frontend configuration
parameters to file as JSON. Then writes the name of the temporary
file to /tmp/hyperglass.env.json as {"configFile": <file_name> }.
Webpack reads /tmp/hyperglass.env.json, loads the temporary file,
and sets its contents to Node environment variables during the build
process.
After the build is successful, the temporary file is automatically
closed during garbage collection.
Arguments:
dev_mode {bool} -- Development Mode
dev_url {str} -- Development Mode URL
prod_url {str} -- Production Mode URL
params {dict} -- Frontend Config paramters
Raises:
RuntimeError: Raised if errors occur during build process.
Returns:
{bool} -- True if successful
"""
import hashlib
import tempfile
from favicons import Favicons
from hyperglass.constants import __version__
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
package_json = await read_package_json()
env_vars = {
"_HYPERGLASS_CONFIG_": params,
"_HYPERGLASS_VERSION_": __version__,
"_HYPERGLASS_PACKAGE_JSON_": package_json,
"_HYPERGLASS_APP_PATH_": str(app_path),
}
# Set NextJS production/development mode and base URL based on
# developer_mode setting.
if dev_mode:
env_vars.update({"NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url})
else:
env_vars.update({"NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url})
# Check if hyperglass/ui/node_modules has been initialized. If not,
# initialize it.
initialized = await check_node_modules()
if initialized:
log.debug("node_modules is already initialized")
elif not initialized:
log.debug("node_modules has not been initialized. Starting initialization...")
node_setup = await node_initial(dev_mode)
if node_setup == "":
log.debug("Re-initialized node_modules")
images_dir = app_path / "static" / "images"
favicon_dir = images_dir / "favicons"
try:
if not favicon_dir.exists():
favicon_dir.mkdir()
async with Favicons(
source=params["web"]["logo"]["favicon"],
output_directory=favicon_dir,
base_url="/images/favicons/",
) as favicons:
await favicons.generate()
log.debug("Generated {} favicons", favicons.completed)
env_vars.update({"_HYPERGLASS_FAVICONS_": favicons.formats()})
env_json = json.dumps(env_vars, default=str)
# Create SHA256 hash from all parameters passed to UI, use as
# build identifier.
build_id = hashlib.sha256(env_json.encode()).hexdigest()
# Read hard-coded environment file from last build. If build ID
# matches this build's ID, don't run a new build.
if env_file.exists() and not force:
with env_file.open("r") as ef:
ef_id = json.load(ef).get("buildId", "empty")
log.debug("Previous Build ID: {id}", id=ef_id)
if ef_id == build_id:
log.debug(
"UI parameters unchanged since last build, skipping UI build..."
)
return True
# Create temporary file. json file extension is added for easy
# webpack JSON parsing.
temp_file = tempfile.NamedTemporaryFile(
mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode
)
log.info("Starting UI build...")
log.debug(
f"Created temporary UI config file: '{temp_file.name}' for build {build_id}"
)
with Path(temp_file.name).open("w+") as temp:
temp.write(env_json)
# Write "permanent" file (hard-coded named) for Node to read.
env_file.write_text(
json.dumps({"configFile": temp_file.name, "buildId": build_id})
)
# While temporary file is still open, initiate UI build process.
if not dev_mode or force:
initialize_result = await node_initial(dev_mode)
build_result = await build_ui(app_path=app_path)
if initialize_result:
log.debug(initialize_result)
elif initialize_result == "":
log.debug("Re-initialized node_modules")
if build_result:
log.success("Completed UI build")
elif dev_mode and not force:
log.debug("Running in developer mode, did not build new UI files")
migrate_images(app_path, params)
generate_opengraph(
Path(params["web"]["opengraph"]["image"]),
1200,
630,
images_dir,
params["web"]["theme"]["colors"]["black"],
)
except Exception as err:
log.error(err)
raise RuntimeError(str(err)) from None
return True
async def scrape_direct(self):
"""Connect directly to a device.
Directly connects to the router via Netmiko library, returns the
command output.
"""
log.debug(f"Connecting directly to {self.device.name}...")
scrape_host = {
"host": self.device.address,
"port": self.device.port,
"device_type": self.device.nos,
"username": self.device.credential.username,
"password": self.device.credential.password.get_secret_value(),
**self.netmiko_args,
}
try:
nm_connect_direct = ConnectHandler(**scrape_host)
def handle_timeout(*args, **kwargs):
nm_connect_direct.disconnect()
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
error=params.messages.request_timeout,
)
signal.signal(signal.SIGALRM, handle_timeout)
signal.alarm(params.request_timeout - 1)
responses = ()
for query in self.query:
raw = nm_connect_direct.send_command(query)
responses += (raw, )
log.debug(f'Raw response for command "{query}":\n{raw}')
nm_connect_direct.disconnect()
except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error:
log.error(str(scrape_error))
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.request_timeout,
)
except (NetMikoAuthenticationException,
NetmikoAuthError) as auth_error:
log.error(
"Error authenticating to device {loc}: {e}",
loc=self.device.name,
e=str(auth_error),
)
raise AuthError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.authentication_error,
)
if not responses:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.no_response,
)
signal.alarm(0)
return await self.parsed_response(responses)
async def collect(self, host: str = None, port: int = None) -> Iterable:
"""Connect directly to a device.
Directly connects to the router via Netmiko library, returns the
command output.
"""
if host is not None:
log.debug(
"Connecting to {} via proxy {} [{}]",
self.device.name,
self.device.proxy.name,
f"{host}:{port}",
)
else:
log.debug("Connecting directly to {}", self.device.name)
netmiko_args = {
"host": host or self.device._target,
"port": port or self.device.port,
"device_type": self.device.nos,
"username": self.device.credential.username,
"password": self.device.credential.password.get_secret_value(),
"global_delay_factor": params.netmiko_delay_factor,
"timeout": math.floor(params.request_timeout * 1.25),
"session_timeout": math.ceil(params.request_timeout - 1),
}
try:
nm_connect_direct = ConnectHandler(**netmiko_args)
responses = ()
for query in self.query:
raw = nm_connect_direct.send_command(query)
responses += (raw,)
log.debug(f'Raw response for command "{query}":\n{raw}')
nm_connect_direct.disconnect()
except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error:
log.error(str(scrape_error))
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.request_timeout,
)
except (NetMikoAuthenticationException, NetmikoAuthError) as auth_error:
log.error(
"Error authenticating to device {loc}: {e}",
loc=self.device.name,
e=str(auth_error),
)
raise AuthError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.authentication_error,
)
if not responses:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.display_name,
proxy=None,
error=params.messages.no_response,
)
return responses
user_config = _config_optional(CONFIG_MAIN)
# Read raw debug value from config to enable debugging quickly.
set_log_level(logger=log, debug=user_config.get("debug", True))
_user_commands = _config_optional(CONFIG_COMMANDS)
_user_devices = _config_required(CONFIG_DEVICES)
# Map imported user config files to expected schema:
try:
params = _params.Params(**user_config)
commands = _commands.Commands.import_params(_user_commands)
devices = _routers.Routers._import(_user_devices.get("routers", {}))
except ValidationError as validation_errors:
errors = validation_errors.errors()
log.error(errors)
for error in errors:
raise ConfigInvalid(
field=": ".join([str(item) for item in error["loc"]]),
error_msg=error["msg"],
)
_validate_nos_commands(devices.all_nos, commands)
set_cache_env(db=params.cache.database,
host=params.cache.host,
port=params.cache.port)
# Re-evaluate debug state after config is validated
set_log_level(logger=log, debug=params.debug)
async def collect(self, host: str = None, port: int = None) -> Iterable:
"""Connect directly to a device.
Directly connects to the router via Netmiko library, returns the
command output.
"""
if host is not None:
log.debug(
"Connecting to {} via proxy {} [{}]",
self.device.name,
self.device.proxy.name,
f"{host}:{port}",
)
else:
log.debug("Connecting directly to {}", self.device.name)
global_args = netmiko_nos_globals.get(self.device.nos, {})
send_args = netmiko_nos_send_args.get(self.device.nos, {})
driver_kwargs = {
"host": host or self.device._target,
"port": port or self.device.port,
"device_type": self.device.nos,
"username": self.device.credential.username,
"global_delay_factor": params.netmiko_delay_factor,
"timeout": math.floor(params.request_timeout * 1.25),
"session_timeout": math.ceil(params.request_timeout - 1),
**global_args,
}
if "_telnet" in self.device.nos:
# Telnet devices with a low delay factor (default) tend to
# throw login errors.
driver_kwargs["global_delay_factor"] = 2
if self.device.credential._method == "password":
# Use password auth if no key is defined.
driver_kwargs[
"password"] = self.device.credential.password.get_secret_value(
)
else:
# Otherwise, use key auth.
driver_kwargs["use_keys"] = True
driver_kwargs["key_file"] = self.device.credential.key
if self.device.credential._method == "encrypted_key":
# If the key is encrypted, use the password field as the
# private key password.
driver_kwargs[
"passphrase"] = self.device.credential.password.get_secret_value(
)
try:
nm_connect_direct = ConnectHandler(**driver_kwargs)
responses = ()
for query in self.query:
raw = nm_connect_direct.send_command(query, **send_args)
responses += (raw, )
log.debug(f'Raw response for command "{query}":\n{raw}')
nm_connect_direct.disconnect()
except NetMikoTimeoutException as scrape_error:
log.error(str(scrape_error))
raise DeviceTimeout(
params.messages.connection_error,
device_name=self.device.name,
proxy=None,
error=params.messages.request_timeout,
)
except NetMikoAuthenticationException as auth_error:
log.error(
"Error authenticating to device {loc}: {e}",
loc=self.device.name,
e=str(auth_error),
)
raise AuthError(
params.messages.connection_error,
device_name=self.device.name,
proxy=None,
error=params.messages.authentication_error,
)
if not responses:
raise ScrapeError(
params.messages.connection_error,
device_name=self.device.name,
proxy=None,
error=params.messages.no_response,
)
return responses
