async def report_update_results(self, update: DataUpdate,
reports: List[DataEntryReport],
data_fetcher: DataFetcher):
try:
whole_report = DataUpdateReport(update_id=update.id,
reports=reports)
callbacks = update.callback.callbacks or opal_client_config.DEFAULT_UPDATE_CALLBACKS.callbacks
urls = []
for callback in callbacks:
if isinstance(callback, str):
url = callback
callback_config = opal_client_config.DEFAULT_UPDATE_CALLBACK_CONFIG.copy(
)
else:
url, callback_config = callback
callback_config.data = whole_report.json()
urls.append((url, callback_config))
logger.info("Reporting the update to requested callbacks",
urls=repr(urls))
report_results = await data_fetcher.handle_urls(urls)
# log reports which we failed to send
for (url, config), result in zip(urls, report_results):
if isinstance(result, Exception):
logger.error(
"Failed to send report to {url} with config {config}",
url=url,
config=config,
exc_info=result)
except:
logger.exception("Failed to excute report_update_results")
async def get_policy_data_config(self,
url: str = None) -> DataSourceConfig:
"""
Get the configuration for
Args:
url: the URL to query for the config, Defaults to self._data_sources_config_url
Returns:
DataSourceConfig: the data sources config
"""
if url is None:
url = self._data_sources_config_url
logger.info("Getting data-sources configuration from '{source}'",
source=url)
try:
async with ClientSession(headers=self._extra_headers) as session:
response = await session.get(url)
if response.status == 200:
return DataSourceConfig.parse_obj(await response.json())
else:
error_details = await response.json()
raise ClientError(
f"Fetch data sources failed with status code {response.status}, error: {error_details}"
)
except:
logger.exception(f"Failed to load data sources config")
raise
def calc_hash(data):
"""
Calculate an hash (sah256) on the given data, if data isn't a string, it will be converted to JSON.
String are encoded as 'utf-8' prior to hash calculation.
Returns:
the hash of the given data (as a a hexdigit string) or '' on failure to process.
"""
try:
if not isinstance(data, str):
data = json.dumps(data)
return hashlib.sha256(data.encode('utf-8')).hexdigest()
except:
logger.exception("Failed to calculate hash for data {data}", data=data)
return ""
async def get_policy_data_config(self, url: str = None) -> DataSourceConfig:
"""
Get the configuration for
Args:
url: the URL to query for the config, Defaults to self._data_sources_config_url
Returns:
DataSourceConfig: the data sources config
"""
if url is None:
url = self._data_sources_config_url
logger.info("Getting data-sources configuration from '{source}'", source=url)
try:
async with ClientSession(headers=self._extra_headers) as session:
res = await session.get(url)
return DataSourceConfig.parse_obj(await res.json())
except:
logger.exception(f"Failed to load data sources config")
raise
async def update_policy_data(self, update: DataUpdate = None, policy_store: BasePolicyStoreClient = None, data_fetcher=None):
"""
fetches policy data (policy configuration) from backend and updates it into policy-store (i.e. OPA)
"""
policy_store = policy_store or DEFAULT_POLICY_STORE_GETTER()
if data_fetcher is None:
data_fetcher = DataFetcher()
# types / defaults
urls: List[Tuple[str, FetcherConfig]] = None
entries: List[DataSourceEntry] = []
# track the result of each url in order to report back
reports: List[DataEntryReport] = []
# if we have an actual specification for the update
if update is not None:
entries = update.entries
urls = [(entry.url, entry.config) for entry in entries]
# get the data for the update
logger.info("Fetching policy data", urls=urls)
# Urls may be None - handle_urls has a default for None
policy_data_with_urls = await data_fetcher.handle_urls(urls)
# Save the data from the update
# We wrap our interaction with the policy store with a transaction
async with policy_store.transaction_context(update.id) as store_transaction:
# for intelisense treat store_transaction as a PolicyStoreClient (which it proxies)
store_transaction: BasePolicyStoreClient
for (url, fetch_config, result), entry in itertools.zip_longest(policy_data_with_urls, entries):
if not isinstance(result, Exception):
# get path to store the URL data (default mode (None) is as "" - i.e. as all the data at root)
policy_store_path = "" if entry is None else entry.dst_path
# None is not valid - use "" (protect from missconfig)
if policy_store_path is None:
policy_store_path = ""
# fix opa_path (if not empty must start with "/" to be nested under data)
if policy_store_path != "" and not policy_store_path.startswith("/"):
policy_store_path = f"/{policy_store_path}"
policy_data = result
# Create a report on the data-fetching
report = DataEntryReport(entry=entry, hash=self.calc_hash(policy_data), fetched=True)
logger.info(
"Saving fetched data to policy-store: source url='{url}', destination path='{path}'",
url=url,
path=policy_store_path or '/'
)
try:
await store_transaction.set_policy_data(policy_data, path=policy_store_path)
# No exception we we're able to save to the policy-store
report.saved = True
# save the report for the entry
reports.append(report)
except:
logger.exception("Failed to save data update to policy-store")
# we failed to save to policy-store
report.saved = False
# save the report for the entry
reports.append(report)
# re-raise so the context manager will be aware of the failure
raise
else:
report = DataEntryReport(entry=entry, fetched=False, saved=False)
# save the report for the entry
reports.append(report)
# should we send a report to defined callbackers?
if self._should_send_reports:
# spin off reporting (no need to wait on it)
asyncio.create_task(self.report_update_results(update, reports, data_fetcher))