def wrapper(*args, **kwargs):
key = f'{decorated.__name__}{args}{kwargs}'
ret = cache.get(key)
metrics_tags = [f'function:{decorated.__name__}']
if ret is None:
stats.increment('cache.miss', tags=metrics_tags)
ret = decorated(*args, **kwargs)
try:
cache.set(key, ret, timeout=timeout)
except AttributeError:
stats.increment('cache.fail', tags=metrics_tags)
LOG.warn(event='failed to write result to cache', key=key)
else:
stats.increment('cache.hit', tags=metrics_tags)
return ret
def match_node(node_value: Any, modify=True, sources: SourceData = None, discovery_type: str = 'default') -> SourceData:
"""
Checks a node against all sources, using the node_match_key and source_match_key
to determine if the node should receive the source in its configuration.
:param discovery_type: type of XDS request, used to determine if a scoped source should be evaluated
:param node_value: value from the node portion of the envoy discovery request
:param modify: switch to enable or disable modifications via Modifiers
:param sources: the data sources to match the node against
"""
if source_metadata.is_stale:
# Log/emit metric and manually refresh sources.
stats.increment('sources.stale')
LOG.warn(
'Sources have not been refreshed in 2 minutes',
last_update=source_metadata.updated.isoformat(),
instance_count=source_metadata.count
)
sources_refresh()
if sources is None:
sources: SourceData = read_sources()
ret = SourceData()
for scope, instances in sources.scopes.items():
if config.node_matching is False:
ret.scopes[scope] = instances
continue
for instance in instances:
source_value = extract_source_key(instance)
# If a single expression evaluates true, the remaining are not evaluated/executed.
# This saves (a small amount of) computation, which helps when the server starts
# to receive thousands of requests. The list has been ordered descending by what
# we think will more commonly be true.
match = (
contains(source_value, node_value)
or node_value == source_value
or is_wildcard(node_value)
or is_wildcard(source_value)
or is_debug_request(node_value)
)
if match:
ret.scopes[scope].append(instance)
if modify:
return apply_modifications(ret)
return ret
def deserialize_config(content):
try:
envoy_configuration = yaml.safe_load(content)
except ParserError as e:
LOG.msg(
error=repr(e),
context=e.context,
context_mark=e.context_mark,
note=e.note,
problem=e.problem,
problem_mark=e.problem_mark,
)
raise HTTPException(
status_code=500,
detail='Failed to load configuration, there may be '
'a syntax error in the configured templates.'
)
return envoy_configuration
def init_app() -> FastAPI:
# Warm the sources once before starting
sources_refresh()
LOG.info('Initial fetch of Sources completed')
application = FastAPI(
title='Sovereign',
version=__versionstr__,
debug=config.debug_enabled
)
application.include_router(discovery.router, tags=['Configuration Discovery'], prefix='/v2')
application.include_router(crypto.router, tags=['Cryptographic Utilities'], prefix='/crypto')
application.include_router(admin.router, tags=['Debugging Endpoints'], prefix='/admin')
application.include_router(interface.router, tags=['User Interface'], prefix='/ui')
application.include_router(healthchecks.router, tags=['Healthchecks'])
application.add_middleware(RequestContextLogMiddleware)
application.add_middleware(LoggingMiddleware)
application.add_middleware(ScheduledTasksMiddleware)
if config.sentry_dsn and sentry_sdk:
sentry_sdk.init(config.sentry_dsn)
application.add_middleware(SentryAsgiMiddleware)
LOG.info('Sentry middleware enabled')
@application.exception_handler(500)
async def exception_handler(_, exc: Exception) -> json_response_class:
"""
We cannot incur the execution of this function from unit tests
because the starlette test client simply returns exceptions and does
not run them through the exception handler.
Ergo, this is a facade function for `generic_error_response`
"""
return generic_error_response(exc) # pragma: no cover
@application.get('/')
def redirect_to_docs():
return RedirectResponse('/ui')
@application.get('/static/{filename}', summary='Return a static asset')
def static(filename: str):
return FileResponse(resource_filename('sovereign', f'static/{filename}'))
return application
def sources_refresh():
"""
All source data is stored in ``sovereign.sources._source_data``.
Since the variable is outside this functions scope, we can only make
in-place modifications to it via its methods.
This function retrieves all sources, puts them into a temporary list,
and then clears and re-fills ``_source_data`` with the new data.
The process is done in two steps to avoid ``_source_data`` being empty
for any significant amount of time.
"""
stats.increment('sources.attempt')
try:
new_source_data = SourceData()
for configured_source in config.sources:
source = setup_sources(configured_source)
new_source_data.scopes[source.scope].extend(source.get())
except Exception as e:
LOG.error(
'Error while refreshing sources',
traceback=[line for line in traceback.format_exc().split('\n')],
error=e.__class__.__name__,
detail=getattr(e, 'detail', '-'),
request_id=get_request_id()
)
stats.increment('sources.error')
raise
if new_source_data == _source_data:
stats.increment('sources.unchanged')
source_metadata.update_date()
return
else:
stats.increment('sources.refreshed')
_source_data.scopes.clear()
_source_data.scopes.update(new_source_data.scopes)
source_metadata.update_date()
source_metadata.update_count([
instance
for scope in _source_data.scopes.values()
for instance in scope
])
async def dispatch(self, request: Request,
call_next: RequestResponseEndpoint):
start_time = time.time()
response = Response("Internal server error", status_code=500)
new_log_context()
add_log_context(env=config.environment,
site=request.headers.get('host', '-'),
method=request.method,
uri_path=request.url.path,
uri_query=dict(request.query_params.items()),
src_ip=request.client.host,
src_port=request.client.port,
pid=os.getpid(),
user_agent=request.headers.get('user-agent', '-'),
bytes_in=request.headers.get('content-length', '-'))
try:
response: Response = await call_next(request)
finally:
duration = time.time() - start_time
LOG.info(bytes_out=response.headers.get('content-length', '-'),
status=response.status_code,
duration=duration,
request_id=response.headers.get('X-Request-Id', '-'))
if 'discovery' in str(request.url):
tags = {
'path':
request.url.path,
'xds_type':
response.headers.get("X-Sovereign-Requested-Type"),
'client_version':
response.headers.get("X-Sovereign-Client-Build"),
'response_code':
response.status_code,
}
tags = [
':'.join(map(str, [k, v])) for k, v in tags.items()
if v is not None
]
stats.increment('discovery.rq_total', tags=tags)
stats.timing('discovery.rq_ms',
value=duration * 1000,
tags=tags)
return response
application.add_middleware(SentryAsgiMiddleware)
LOG.info('Sentry middleware enabled')
@application.exception_handler(500)
async def exception_handler(_, exc: Exception) -> json_response_class:
"""
We cannot incur the execution of this function from unit tests
because the starlette test client simply returns exceptions and does
not run them through the exception handler.
Ergo, this is a facade function for `generic_error_response`
"""
return generic_error_response(exc) # pragma: no cover
@application.get('/')
def redirect_to_docs():
return RedirectResponse('/ui')
@application.get('/static/{filename}', summary='Return a static asset')
def static(filename: str):
return FileResponse(resource_filename('sovereign', f'static/{filename}'))
return application
app = init_app()
LOG.info(f'Sovereign started and listening on {asgi_config.host}:{asgi_config.port}')
if __name__ == '__main__': # pragma: no cover
uvicorn.run(app, host='0.0.0.0', port=8000, access_log=False)
from sovereign import config
from sovereign.logs import LOG
disabled_suite = namedtuple('DisabledSuite', ['encrypt', 'decrypt'])
disabled_suite.encrypt = lambda x: 'Unavailable (No Secret Key)'
disabled_suite.decrypt = lambda x: 'Unavailable (No Secret Key)'
try:
_cipher_suite = Fernet(config.encryption_key)
KEY_AVAILABLE = True
except TypeError:
KEY_AVAILABLE = False
_cipher_suite = disabled_suite
except ValueError as e:
if config.encryption_key != '':
LOG.warn(
f'Fernet key was provided, but appears to be invalid: {repr(e)}')
_cipher_suite = disabled_suite
KEY_AVAILABLE = False
def encrypt(data: str, key=None) -> str:
_local_cipher_suite = _cipher_suite
if key is not None:
_local_cipher_suite = Fernet(key)
try:
encrypted: bytes = _local_cipher_suite.encrypt(data.encode())
except (InvalidToken, AttributeError):
raise HTTPException(status_code=400, detail='Encryption failed')
return encrypted.decode()