class BufferingInMemoryCache(object): """PValueCache wrapper for buffering bundles until a PValue is fully computed. BufferingInMemoryCache keeps an in memory cache of (applied_ptransform, tag) tuples. It accepts appending to existing cache entries until it is finalized. finalize() will make all the existing cached entries visible to the underyling PValueCache in their entirety, clean the in memory cache and stop accepting new cache entries. """ def __init__(self): self._cache = collections.defaultdict(list) self._pvalue_cache = PValueCache() self._finalized = False @property def pvalue_cache(self): return self._pvalue_cache def append(self, applied_ptransform, tag, elements): assert not self._finalized assert elements is not None self._cache[(applied_ptransform, tag)].extend(elements) def finalize(self): """Make buffered cache elements visible to the underlying PValueCache.""" assert not self._finalized for key, value in self._cache.iteritems(): applied_ptransform, tag = key self._pvalue_cache.cache_output(applied_ptransform, tag, value) self._cache = None
def __init__(self): self._cache = collections.defaultdict(list) self._pvalue_cache = PValueCache() self._finalized = False
def __init__(self): self._cache = collections.defaultdict(list) self._pvalue_cache = PValueCache() self._finalized = False
def __init__(self, cache=None): # Cache of CloudWorkflowStep protos generated while the runner # "executes" a pipeline. self._cache = cache if cache is not None else PValueCache() self._unique_step_id = 0