class OnDemandRepositoryView(RepositoryView): def __init__(self, repository, name, version): if version is not None: self._version = version else: self._version = repository.store.getVersion() self._exclusive = ThreadSemaphore() self._hooks = [] super(OnDemandRepositoryView, self).__init__(repository, name, version) def isNew(self): return self._version == 0 def _setLoading(self, loading, runHooks=False): if not loading and self.isLoading() and runHooks: try: for hook in self._hooks: hook(self) finally: self._hooks = [] return super(OnDemandRepositoryView, self)._setLoading(loading, runHooks) def _readItem(self, itemReader): try: release = False loading = self.isLoading() debug = self.isDebug() if not loading: release = self._exclusive.acquire() self._setLoading(True) self._hooks = [] exception = None if debug: self.logger.debug('loading item %s', itemReader.getUUID()) item = itemReader.readItem(self, self._hooks) if debug: self.logger.debug("loaded version %d of %s", item._version, item.itsPath) except: if not loading: self._setLoading(False, False) self._hooks = [] if release: self._exclusive.release() raise else: if not loading: self._setLoading(False, True) if release: self._exclusive.release() return item def _loadItem(self, uuid): if uuid in self._loadingRegistry: raise RecursiveLoadItemError, uuid if not uuid in self._deletedRegistry: itemReader = self.repository.store.loadItem(self._version, uuid) if itemReader is not None: try: self._loadingRegistry.add(uuid) self.logger.debug("loading item %s", uuid) return self._readItem(itemReader) finally: self._loadingRegistry.remove(uuid) return None def _findSchema(self, spec, withSchema): if withSchema: return self.find(spec, load=False) # when crossing the schema boundary, reset loading status so that # hooks get called before resuming regular loading try: hooks = self._hooks loading = self._setLoading(False) return self.find(spec) finally: self._hooks = hooks self._setLoading(loading) def _addItem(self, item, previous=None, next=None): super(OnDemandRepositoryView, self)._addItem(item, previous, next) item.setPinned(True) return item def _removeItem(self, item): super(OnDemandRepositoryView, self)._removeItem(item) item.setPinned(False) def prune(self, size): registry = self._registry if len(registry) > size * 1.1: gc.collect() heap = [(item._lastAccess, item._uuid) for item in registry.itervalues() if not item._status & (item.PINNED | item.DIRTY)] heapq.heapify(heap) count = len(heap) - int(size * 0.9) if count > 0: self.logger.info('pruning %d items', count) if self.isRefCounted(): for i in xrange(count): item = registry[heapq.heappop(heap)[1]] itemRefs = item._refCount() pythonRefs = sys.getrefcount(item) if pythonRefs - itemRefs <= 3: item._unloadItem(False, self) else: self.logger.warn('not pruning %s (refCount %d)', item._repr_(), pythonRefs - itemRefs) else: for i in xrange(count): registry[heapq.heappop(heap)[1]]._unloadItem( False, self)
class OnDemandRepositoryView(RepositoryView): def __init__(self, repository, name, version): if version is not None: self._version = version else: self._version = repository.store.getVersion() self._exclusive = ThreadSemaphore() self._hooks = [] super(OnDemandRepositoryView, self).__init__(repository, name, version) def isNew(self): return self._version == 0 def _setLoading(self, loading, runHooks=False): if not loading and self.isLoading() and runHooks: try: for hook in self._hooks: hook(self) finally: self._hooks = [] return super(OnDemandRepositoryView, self)._setLoading(loading, runHooks) def _readItem(self, itemReader): try: release = False loading = self.isLoading() debug = self.isDebug() if not loading: release = self._exclusive.acquire() self._setLoading(True) self._hooks = [] exception = None if debug: self.logger.debug('loading item %s', itemReader.getUUID()) item = itemReader.readItem(self, self._hooks) if debug: self.logger.debug("loaded version %d of %s", item._version, item.itsPath) except: if not loading: self._setLoading(False, False) self._hooks = [] if release: self._exclusive.release() raise else: if not loading: self._setLoading(False, True) if release: self._exclusive.release() return item def _loadItem(self, uuid): if uuid in self._loadingRegistry: raise RecursiveLoadItemError, uuid if not uuid in self._deletedRegistry: itemReader = self.repository.store.loadItem(self._version, uuid) if itemReader is not None: try: self._loadingRegistry.add(uuid) self.logger.debug("loading item %s", uuid) return self._readItem(itemReader) finally: self._loadingRegistry.remove(uuid) return None def _findSchema(self, spec, withSchema): if withSchema: return self.find(spec, load=False) # when crossing the schema boundary, reset loading status so that # hooks get called before resuming regular loading try: hooks = self._hooks loading = self._setLoading(False) return self.find(spec) finally: self._hooks = hooks self._setLoading(loading) def _addItem(self, item, previous=None, next=None): super(OnDemandRepositoryView, self)._addItem(item, previous, next) item.setPinned(True) return item def _removeItem(self, item): super(OnDemandRepositoryView, self)._removeItem(item) item.setPinned(False) def prune(self, size): registry = self._registry if len(registry) > size * 1.1: gc.collect() heap = [(item._lastAccess, item._uuid) for item in registry.itervalues() if not item._status & (item.PINNED | item.DIRTY)] heapq.heapify(heap) count = len(heap) - int(size * 0.9) if count > 0: self.logger.info('pruning %d items', count) if self.isRefCounted(): for i in xrange(count): item = registry[heapq.heappop(heap)[1]] itemRefs = item._refCount() pythonRefs = sys.getrefcount(item) if pythonRefs - itemRefs <= 3: item._unloadItem(False, self) else: self.logger.warn('not pruning %s (refCount %d)', item._repr_(), pythonRefs - itemRefs) else: for i in xrange(count): registry[heapq.heappop(heap)[1]]._unloadItem(False, self)