async def get_data_with_input(self, path: str, input: BaseModel) -> Dict:
"""
evaluates a data document against an input.
that is how OPA "runs queries".
see explanation how opa evaluate documents:
https://www.openpolicyagent.org/docs/latest/philosophy/#the-opa-document-model
see api reference:
https://www.openpolicyagent.org/docs/latest/rest-api/#get-a-document-with-input
"""
# opa data api format needs the input to sit under "input"
opa_input = {
"input": input.dict()
}
if path.startswith("/"):
path = path[1:]
try:
async with aiohttp.ClientSession() as session:
async with session.post(
f"{self._opa_url}/data/{path}",
data=json.dumps(opa_input)
) as opa_response:
return await proxy_response(opa_response)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def _fetch_policy_bundle(
self,
directories: List[str] = ['.'],
base_hash: Optional[str] = None) -> Optional[PolicyBundle]:
"""
Fetches the bundle. May throw, in which case we retry again.
"""
params = {"path": directories}
if base_hash is not None:
params["base_hash"] = base_hash
async with aiohttp.ClientSession() as session:
try:
async with session.get(f"{self._backend_url}/policy",
headers={
'content-type': 'text/plain',
**self._auth_headers
},
params=params) as response:
if response.status == status.HTTP_404_NOT_FOUND:
logger.warning("requested paths not found: {paths}",
paths=directories)
return None
# may throw ValueError
await throw_if_bad_status_code(
response, expected=[status.HTTP_200_OK])
# may throw Validation Error
bundle = await response.json()
return force_valid_bundle(bundle)
except aiohttp.ClientError as e:
logger.warning("server connection error: {err}", err=repr(e))
raise
def policy_bundle_or_none(bundle) -> Optional[PolicyBundle]:
try:
return PolicyBundle(**bundle)
except ValidationError as e:
logger.warning("server returned invalid bundle: {err}",
bundle=bundle,
err=e)
return None
def force_valid_bundle(bundle) -> PolicyBundle:
try:
return PolicyBundle(**bundle)
except ValidationError as e:
logger.warning("server returned invalid bundle: {err}",
bundle=bundle,
err=repr(e))
raise
async def delete_policy(self, policy_id: str):
async with aiohttp.ClientSession() as session:
try:
async with session.delete(
f"{self._opa_url}/policies/{policy_id}",
) as opa_response:
return await proxy_response(opa_response)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def get_policy(self, policy_id: str) -> Optional[str]:
async with aiohttp.ClientSession() as session:
try:
async with session.get(f"{self._opa_url}/policies/{policy_id}",
) as opa_response:
result = await opa_response.json()
return result.get("result", {}).get("raw", None)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=repr(e))
raise
async def get_policy_module_ids(self) -> List[str]:
async with aiohttp.ClientSession() as session:
try:
async with session.get(
f"{self._opa_url}/policies", ) as opa_response:
result = await opa_response.json()
return OpaClient._extract_module_ids_from_policies_json(
result)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=repr(e))
raise
async def delete_policy_data(self, path: str = ""):
if not path:
return await self.set_policy_data({})
async with aiohttp.ClientSession() as session:
try:
async with session.delete(
f"{self._opa_url}/data{path}", ) as opa_response:
return await proxy_response(opa_response)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def set_policy(self, policy_id: str, policy_code: str):
self._cached_policies[policy_id] = policy_code
async with aiohttp.ClientSession() as session:
try:
async with session.put(f"{self._opa_url}/policies/{policy_id}",
data=policy_code,
headers={'content-type':
'text/plain'}) as opa_response:
return await proxy_response(opa_response)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def set_policy(self, policy_id: str, policy_code: str, transaction_id:Optional[str]=None):
self._cached_policies[policy_id] = policy_code
async with aiohttp.ClientSession() as session:
try:
async with session.put(
f"{self._opa_url}/policies/{policy_id}",
data=policy_code,
headers={'content-type': 'text/plain'}
) as opa_response:
return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[status.HTTP_200_OK])
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def throw_if_bad_status_code(
response: aiohttp.ClientResponse,
expected: List[int]) -> aiohttp.ClientResponse:
if response.status in expected:
return response
# else, bad status code
details = await response.json()
logger.warning("Unexpected response code {status}: {details}",
status=response.status,
details=details)
raise ValueError(
f"unexpected response code while fetching bundle: {response.status}")
async def fetch_policy_bundle(
self,
directories: List[str] = ['.'],
base_hash: Optional[str] = None) -> Optional[PolicyBundle]:
attempter = retry(**self._retry_config)(self._fetch_policy_bundle)
try:
return await attempter(directories=directories,
base_hash=base_hash)
except Exception as err:
logger.warning(
"Failed all attempts to fetch bundle, got error: {err}",
err=repr(err))
return None
async def delete_policy(self, policy_id: str, transaction_id:Optional[str]=None):
async with aiohttp.ClientSession() as session:
try:
async with session.delete(
f"{self._opa_url}/policies/{policy_id}",
) as opa_response:
return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[
status.HTTP_200_OK,
status.HTTP_404_NOT_FOUND
])
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def end_transcation(self,
exc_type=None,
exc=None,
tb=None,
transaction_id: str = None,
actions: List[str] = None):
"""
PolicyStoreTranscationContextManager calls here on __aexit__
Complete a series of operations with the policy store
Args:
exc_type: The exception type (if raised). Defaults to None.
exc: The exception type (if raised). Defaults to None.
tb: The traceback (if raised). Defaults to None.
transaction_id (str, optional): The transaction id. Defaults to None.
actions (List[str], optional): All the methods called in the transaction. Defaults to None.
"""
if transaction_id is None or not actions:
return # skip, nothing to do if we have no data to log
if exc is not None:
try:
error_message = repr(exc)
except: # maybe repr throws here
error_message = None
transaction = StoreTransaction(id=transaction_id,
actions=actions,
success=False,
error=error_message)
logger.warning(
"OPA transaction failed, transaction id={id}, actions={actions}, error={err}",
id=transaction_id,
actions=repr(actions),
err=error_message)
else:
transaction = StoreTransaction(id=transaction_id,
actions=actions,
success=True)
if not opal_client_config.OPA_HEALTH_CHECK_POLICY_ENABLED:
return # skip persisting the transaction, healthcheck policy is disabled
try:
await self.persist_transaction(transaction)
except Exception as e:
# The writes to transaction log in OPA cache are not done a protected
# transaction context. If they fail, we do nothing special.
logger.error(
"Cannot write to OPAL transaction log, transaction id={id}, error={err}",
id=transaction.id,
err=repr(e))
async def set_policy_data(self,
policy_data: Dict[str, Any],
path: str = ""):
self._policy_data = policy_data
async with aiohttp.ClientSession() as session:
try:
async with session.put(
f"{self._opa_url}/data{path}",
data=json.dumps(self._policy_data),
) as opa_response:
return await proxy_response(opa_response)
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def set_policy_data(self, policy_data: JsonableValue, path: str = "", transaction_id:Optional[str]=None):
path = self._safe_data_module_path(path)
self._policy_data = policy_data
async with aiohttp.ClientSession() as session:
try:
async with session.put(
f"{self._opa_url}/data{path}",
data=json.dumps(self._policy_data),
) as opa_response:
return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[
status.HTTP_204_NO_CONTENT,
status.HTTP_304_NOT_MODIFIED
])
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def patch_data(self, path: str, patch_document: JSONPatchDocument, transaction_id:Optional[str]=None):
path = self._safe_data_module_path(path)
# a patch document is a list of actions
# we render each action with pydantic, and then dump the doc into json
json_document = json.dumps([action.dict() for action in patch_document])
async with aiohttp.ClientSession() as session:
try:
async with session.patch(
f"{self._opa_url}/data{path}",
data=json_document,
) as opa_response:
return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[status.HTTP_204_NO_CONTENT])
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def delete_policy_data(self, path: str = "", transaction_id:Optional[str]=None):
path = self._safe_data_module_path(path)
if not path:
return await self.set_policy_data({})
async with aiohttp.ClientSession() as session:
try:
async with session.delete(
f"{self._opa_url}/data{path}",
) as opa_response:
return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[
status.HTTP_204_NO_CONTENT,
status.HTTP_404_NOT_FOUND
])
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def get_data(self, path: str) -> Dict:
"""
wraps opa's "GET /data" api that extracts base data documents from opa cache.
NOTE: opa always returns 200 and empty dict (for valid input) even if the data does not exist.
returns a dict (parsed json).
"""
# function accepts paths that start with / and also path that do not start with /
if path.startswith("/"):
path = path[1:]
try:
async with aiohttp.ClientSession() as session:
async with session.get(f"{self._opa_url}/data/{path}") as opa_response:
return await opa_response.json()
except aiohttp.ClientError as e:
logger.warning("Opa connection error: {err}", err=e)
raise
async def _set_policy_data_from_bundle_data_module(self,
module: DataModule,
hash: Optional[
str] = None):
module_path = self._safe_data_module_path(module.path)
try:
module_data = json.loads(module.data)
return await self.set_policy_data(
policy_data=module_data,
path=module_path,
)
except json.JSONDecodeError as e:
logger.warning(
"bundle contains non-json data module: {module_path}",
err=e,
module_path=module_path,
bundle_hash=hash)
async def _update_policy_callback(self, data: dict = None, topic: str = "", **kwargs):
"""
Pub/Sub callback - triggering policy updates
will run when we get notifications on the policy topic.
i.e: when the source repository changes (new commits)
"""
if topic.startswith(POLICY_PREFIX):
directories = [remove_prefix(topic, prefix=POLICY_PREFIX)]
logger.info(
"Received policy update: affected directories={directories}, new commit hash='{new_hash}'",
directories=directories,
topic=topic,
new_hash=data
)
else:
directories = default_subscribed_policy_directories()
logger.warning("Received policy updated (invalid topic): {topic}", topic=topic)
await self.update_policy(directories)
async def fetch_policy_bundle(
self,
directories: List[str] = ['.'],
base_hash: Optional[str] = None) -> Optional[PolicyBundle]:
params = {"path": directories}
if base_hash is not None:
params["base_hash"] = base_hash
async with aiohttp.ClientSession() as session:
try:
async with session.get(f"{self._backend_url}/policy",
headers={
'content-type': 'text/plain',
**self._auth_headers
},
params=params) as response:
if response.status == status.HTTP_404_NOT_FOUND:
logger.warning("requested paths not found: {paths}",
paths=directories)
return None
bundle = await response.json()
return policy_bundle_or_none(bundle)
except aiohttp.ClientError as e:
logger.warning("server connection error: {err}", err=e)
raise
