def open_cached_render_result( crr: CachedRenderResult) -> ContextManager[RenderResult]: """ Yield a RenderResult equivalent to the one passed to `cache_render_result()`. Raise CorruptCacheError if the cached data does not match `crr`. That can mean: * The cached Parquet file is corrupt * The cached Parquet file is missing * `crr` is stale -- the cached result is for a different delta. This could be detected by a `Workflow.cooperative_lock()`, too, should the caller want to distinguish this error from the others. The returned RenderResult is backed by an mmapped file on disk, so it doesn't require much physical RAM. """ if not crr.table_metadata.columns: # Zero-column tables aren't written to cache yield RenderResult( ArrowTable.from_zero_column_metadata( TableMetadata(crr.table_metadata.n_rows, [])), crr.errors, crr.json, ) return with tempfile_context(prefix="cached-render-result") as arrow_path: # raise CorruptCacheError (deleting `arrow_path` in the process) result = load_cached_render_result(crr, arrow_path) yield result
def load_cached_render_result(crr: CachedRenderResult, path: Path) -> RenderResult: """ Return a RenderResult equivalent to the one passed to `cache_render_result()`. Raise CorruptCacheError if the cached data does not match `crr`. That can mean: * The cached Parquet file is corrupt * The cached Parquet file is missing * `crr` is stale -- the cached result is for a different delta. This could be detected by a `Workflow.cooperative_lock()`, too, should the caller want to distinguish this error from the others. The returned RenderResult is backed by an mmapped file on disk -- the one supplied as `path`. It doesn't require much physical RAM: the Linux kernel may page out data we aren't using. """ if not crr.table_metadata.columns: # Zero-column tables aren't written to cache return RenderResult( ArrowTable.from_zero_column_metadata( TableMetadata(crr.table_metadata.n_rows, [])), crr.errors, crr.json, ) # raises CorruptCacheError with downloaded_parquet_file(crr) as parquet_path: try: # raises ArrowIOError cjwparquet.convert_parquet_file_to_arrow_file(parquet_path, path) except pyarrow.ArrowIOError as err: raise CorruptCacheError from err # TODO handle validation errors => CorruptCacheError arrow_table = ArrowTable.from_trusted_file(path, crr.table_metadata) return RenderResult(arrow_table, crr.errors, crr.json)