async def get_data_with_input(self, path: str, input: BaseModel) -> Dict: """ evaluates a data document against an input. that is how OPA "runs queries". see explanation how opa evaluate documents: https://www.openpolicyagent.org/docs/latest/philosophy/#the-opa-document-model see api reference: https://www.openpolicyagent.org/docs/latest/rest-api/#get-a-document-with-input """ # opa data api format needs the input to sit under "input" opa_input = { "input": input.dict() } if path.startswith("/"): path = path[1:] try: async with aiohttp.ClientSession() as session: async with session.post( f"{self._opa_url}/data/{path}", data=json.dumps(opa_input) ) as opa_response: return await proxy_response(opa_response) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def _fetch_policy_bundle( self, directories: List[str] = ['.'], base_hash: Optional[str] = None) -> Optional[PolicyBundle]: """ Fetches the bundle. May throw, in which case we retry again. """ params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash async with aiohttp.ClientSession() as session: try: async with session.get(f"{self._backend_url}/policy", headers={ 'content-type': 'text/plain', **self._auth_headers }, params=params) as response: if response.status == status.HTTP_404_NOT_FOUND: logger.warning("requested paths not found: {paths}", paths=directories) return None # may throw ValueError await throw_if_bad_status_code( response, expected=[status.HTTP_200_OK]) # may throw Validation Error bundle = await response.json() return force_valid_bundle(bundle) except aiohttp.ClientError as e: logger.warning("server connection error: {err}", err=repr(e)) raise
def policy_bundle_or_none(bundle) -> Optional[PolicyBundle]: try: return PolicyBundle(**bundle) except ValidationError as e: logger.warning("server returned invalid bundle: {err}", bundle=bundle, err=e) return None
def force_valid_bundle(bundle) -> PolicyBundle: try: return PolicyBundle(**bundle) except ValidationError as e: logger.warning("server returned invalid bundle: {err}", bundle=bundle, err=repr(e)) raise
async def delete_policy(self, policy_id: str): async with aiohttp.ClientSession() as session: try: async with session.delete( f"{self._opa_url}/policies/{policy_id}", ) as opa_response: return await proxy_response(opa_response) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def get_policy(self, policy_id: str) -> Optional[str]: async with aiohttp.ClientSession() as session: try: async with session.get(f"{self._opa_url}/policies/{policy_id}", ) as opa_response: result = await opa_response.json() return result.get("result", {}).get("raw", None) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=repr(e)) raise
async def get_policy_module_ids(self) -> List[str]: async with aiohttp.ClientSession() as session: try: async with session.get( f"{self._opa_url}/policies", ) as opa_response: result = await opa_response.json() return OpaClient._extract_module_ids_from_policies_json( result) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=repr(e)) raise
async def delete_policy_data(self, path: str = ""): if not path: return await self.set_policy_data({}) async with aiohttp.ClientSession() as session: try: async with session.delete( f"{self._opa_url}/data{path}", ) as opa_response: return await proxy_response(opa_response) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def set_policy(self, policy_id: str, policy_code: str): self._cached_policies[policy_id] = policy_code async with aiohttp.ClientSession() as session: try: async with session.put(f"{self._opa_url}/policies/{policy_id}", data=policy_code, headers={'content-type': 'text/plain'}) as opa_response: return await proxy_response(opa_response) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def set_policy(self, policy_id: str, policy_code: str, transaction_id:Optional[str]=None): self._cached_policies[policy_id] = policy_code async with aiohttp.ClientSession() as session: try: async with session.put( f"{self._opa_url}/policies/{policy_id}", data=policy_code, headers={'content-type': 'text/plain'} ) as opa_response: return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[status.HTTP_200_OK]) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def throw_if_bad_status_code( response: aiohttp.ClientResponse, expected: List[int]) -> aiohttp.ClientResponse: if response.status in expected: return response # else, bad status code details = await response.json() logger.warning("Unexpected response code {status}: {details}", status=response.status, details=details) raise ValueError( f"unexpected response code while fetching bundle: {response.status}")
async def fetch_policy_bundle( self, directories: List[str] = ['.'], base_hash: Optional[str] = None) -> Optional[PolicyBundle]: attempter = retry(**self._retry_config)(self._fetch_policy_bundle) try: return await attempter(directories=directories, base_hash=base_hash) except Exception as err: logger.warning( "Failed all attempts to fetch bundle, got error: {err}", err=repr(err)) return None
async def delete_policy(self, policy_id: str, transaction_id:Optional[str]=None): async with aiohttp.ClientSession() as session: try: async with session.delete( f"{self._opa_url}/policies/{policy_id}", ) as opa_response: return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[ status.HTTP_200_OK, status.HTTP_404_NOT_FOUND ]) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def end_transcation(self, exc_type=None, exc=None, tb=None, transaction_id: str = None, actions: List[str] = None): """ PolicyStoreTranscationContextManager calls here on __aexit__ Complete a series of operations with the policy store Args: exc_type: The exception type (if raised). Defaults to None. exc: The exception type (if raised). Defaults to None. tb: The traceback (if raised). Defaults to None. transaction_id (str, optional): The transaction id. Defaults to None. actions (List[str], optional): All the methods called in the transaction. Defaults to None. """ if transaction_id is None or not actions: return # skip, nothing to do if we have no data to log if exc is not None: try: error_message = repr(exc) except: # maybe repr throws here error_message = None transaction = StoreTransaction(id=transaction_id, actions=actions, success=False, error=error_message) logger.warning( "OPA transaction failed, transaction id={id}, actions={actions}, error={err}", id=transaction_id, actions=repr(actions), err=error_message) else: transaction = StoreTransaction(id=transaction_id, actions=actions, success=True) if not opal_client_config.OPA_HEALTH_CHECK_POLICY_ENABLED: return # skip persisting the transaction, healthcheck policy is disabled try: await self.persist_transaction(transaction) except Exception as e: # The writes to transaction log in OPA cache are not done a protected # transaction context. If they fail, we do nothing special. logger.error( "Cannot write to OPAL transaction log, transaction id={id}, error={err}", id=transaction.id, err=repr(e))
async def set_policy_data(self, policy_data: Dict[str, Any], path: str = ""): self._policy_data = policy_data async with aiohttp.ClientSession() as session: try: async with session.put( f"{self._opa_url}/data{path}", data=json.dumps(self._policy_data), ) as opa_response: return await proxy_response(opa_response) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def set_policy_data(self, policy_data: JsonableValue, path: str = "", transaction_id:Optional[str]=None): path = self._safe_data_module_path(path) self._policy_data = policy_data async with aiohttp.ClientSession() as session: try: async with session.put( f"{self._opa_url}/data{path}", data=json.dumps(self._policy_data), ) as opa_response: return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[ status.HTTP_204_NO_CONTENT, status.HTTP_304_NOT_MODIFIED ]) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def patch_data(self, path: str, patch_document: JSONPatchDocument, transaction_id:Optional[str]=None): path = self._safe_data_module_path(path) # a patch document is a list of actions # we render each action with pydantic, and then dump the doc into json json_document = json.dumps([action.dict() for action in patch_document]) async with aiohttp.ClientSession() as session: try: async with session.patch( f"{self._opa_url}/data{path}", data=json_document, ) as opa_response: return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[status.HTTP_204_NO_CONTENT]) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def delete_policy_data(self, path: str = "", transaction_id:Optional[str]=None): path = self._safe_data_module_path(path) if not path: return await self.set_policy_data({}) async with aiohttp.ClientSession() as session: try: async with session.delete( f"{self._opa_url}/data{path}", ) as opa_response: return await proxy_response_unless_invalid(opa_response, accepted_status_codes=[ status.HTTP_204_NO_CONTENT, status.HTTP_404_NOT_FOUND ]) except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def get_data(self, path: str) -> Dict: """ wraps opa's "GET /data" api that extracts base data documents from opa cache. NOTE: opa always returns 200 and empty dict (for valid input) even if the data does not exist. returns a dict (parsed json). """ # function accepts paths that start with / and also path that do not start with / if path.startswith("/"): path = path[1:] try: async with aiohttp.ClientSession() as session: async with session.get(f"{self._opa_url}/data/{path}") as opa_response: return await opa_response.json() except aiohttp.ClientError as e: logger.warning("Opa connection error: {err}", err=e) raise
async def _set_policy_data_from_bundle_data_module(self, module: DataModule, hash: Optional[ str] = None): module_path = self._safe_data_module_path(module.path) try: module_data = json.loads(module.data) return await self.set_policy_data( policy_data=module_data, path=module_path, ) except json.JSONDecodeError as e: logger.warning( "bundle contains non-json data module: {module_path}", err=e, module_path=module_path, bundle_hash=hash)
async def _update_policy_callback(self, data: dict = None, topic: str = "", **kwargs): """ Pub/Sub callback - triggering policy updates will run when we get notifications on the policy topic. i.e: when the source repository changes (new commits) """ if topic.startswith(POLICY_PREFIX): directories = [remove_prefix(topic, prefix=POLICY_PREFIX)] logger.info( "Received policy update: affected directories={directories}, new commit hash='{new_hash}'", directories=directories, topic=topic, new_hash=data ) else: directories = default_subscribed_policy_directories() logger.warning("Received policy updated (invalid topic): {topic}", topic=topic) await self.update_policy(directories)
async def fetch_policy_bundle( self, directories: List[str] = ['.'], base_hash: Optional[str] = None) -> Optional[PolicyBundle]: params = {"path": directories} if base_hash is not None: params["base_hash"] = base_hash async with aiohttp.ClientSession() as session: try: async with session.get(f"{self._backend_url}/policy", headers={ 'content-type': 'text/plain', **self._auth_headers }, params=params) as response: if response.status == status.HTTP_404_NOT_FOUND: logger.warning("requested paths not found: {paths}", paths=directories) return None bundle = await response.json() return policy_bundle_or_none(bundle) except aiohttp.ClientError as e: logger.warning("server connection error: {err}", err=e) raise