async def __anit__(self): self.loop = asyncio.get_running_loop() if __debug__: import synapse.lib.threads as s_threads # avoid import cycle self.tid = s_threads.iden() self.call_stack = traceback.format_stack() # For cleanup debugging self.isfini = False self.anitted = True # For assertion purposes self.finievt = None self.entered = False self.exitinfo = None self.exitok = None self.entered = False self.exitinfo = None # hold a weak ref to other bases we should fini if they # are still around when we go down... self.tofini = weakref.WeakSet() self._syn_funcs = collections.defaultdict(list) self._syn_refs = 1 # one ref for the ctor self._syn_links = [] self._fini_funcs = [] self._fini_atexit = False self._active_tasks = set() # the free running tasks associated with me
async def __anit__(self): self.loop = asyncio.get_running_loop() if __debug__: import synapse.lib.threads as s_threads # avoid import cycle self.tid = s_threads.iden() self.call_stack = traceback.format_stack() # For cleanup debugging if object.__getattribute__(self, 'anitted') is True: # The Base has already been anitted. This allows a class to treat # multiple Base objects as a mixin and __anit__ themselves without # smashing fini or event handlers from the others. return self.isfini = False self.anitted = True # For assertion purposes self.finievt = None self.entered = False self.exitinfo = None self.exitok = None self.entered = False self.exitinfo = None # hold a weak ref to other bases we should fini if they # are still around when we go down... self.tofini = weakref.WeakSet() self._syn_funcs = collections.defaultdict(list) self._syn_refs = 1 # one ref for the ctor self._syn_links = [] self._fini_funcs = [] self._fini_atexit = False self._active_tasks = set() # the free running tasks associated with me
def schedCoro(self, coro): ''' Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd. It does not pend on coroutine completion. Precondition: This function is *not* threadsafe and must be run on the Base's event loop Returns: An asyncio.Task ''' if __debug__: assert s_coro.iscoro(coro) import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() == self.tid task = self.loop.create_task(coro) def taskDone(task): self._active_tasks.remove(task) try: task.result() except asyncio.CancelledError: pass except Exception: logger.exception( 'Task scheduled through Base.schedCoro raised exception') self._active_tasks.add(task) task.add_done_callback(taskDone) return task
def getCoreXact(self, size=1000, core=None): ''' Get a Storage transaction context for use in a with block. This object allows bulk storage layer optimization and proper ordering of events. Args: size (int): Number of transactions to cache before starting to execute storage layer events. core: Cortex to attach to the StoreXact. Required for splice event support. Examples: Get a StoreXact object and use it:: with store.getCoreXact() as xact: store.dostuff() Returns: s_xact.StoreXact: A StoreXact object for the current thread. ''' iden = s_threads.iden() xact = self._store_xacts.get(iden) if xact is not None: return xact xact = self.getStoreXact(size, core=core) self._store_xacts[iden] = xact return xact
def __enter__(self): ''' Convenience function to enable using Proxy objects as synchronous context managers. Note: This must not be used from async code, and it should never be used in core synapse code. ''' if s_threads.iden() == self.tid: raise s_exc.SynErr('Use of synchronous context manager in async code') self._ctxobj = self.schedCoroSafePend(self.__aenter__()) return self
def __enter__(self): ''' Convenience function to enable using Proxy objects as synchronous context managers. Note: This should never be used by synapse core code. This is for sync client code convenience only. ''' if s_threads.iden() == self.tid: raise s_exc.SynErr('Use of synchronous context manager in async code') self._ctxobj = self.schedCoroSafePend(self.__aenter__()) return self
def schedCoroSafePend(self, coro): ''' Schedules a coroutine to run as soon as possible on the same event loop that this Base is running on Note: This method may *not* be run inside an event loop ''' if __debug__: import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() != self.tid task = asyncio.run_coroutine_threadsafe(coro, self.loop) return task.result()
def test_glob_inpool(self): iden = s_threads.iden() retn = {} evnt = threading.Event() @s_glob.inpool def woot(): retn['iden'] = s_threads.iden() evnt.set() woot() evnt.wait(timeout=1) self.ne(iden, retn.get('iden'))
async def fini(self): ''' Shut down the object and notify any onfini() coroutines. Returns: Remaining ref count ''' assert self.anitted, f'{self.__class__.__name__} initialized improperly. Must use Base.anit class method.' if self.isfini: return if __debug__: import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() == self.tid self._syn_refs -= 1 if self._syn_refs > 0: return self._syn_refs self.isfini = True for base in list(self.tofini): await base.fini() try: await self._kill_active_tasks() except Exception: logger.exception(f'{self} - Exception during _kill_active_tasks') for fini in self._fini_funcs: try: await s_coro.ornot(fini) except asyncio.CancelledError: raise except Exception: logger.exception(f'{self} - fini function failed: {fini}') self._syn_funcs.clear() self._fini_funcs.clear() fevt = self.finievt if fevt is not None: fevt.set() return 0
async def fini(self): ''' Shut down the object and notify any onfini() coroutines. Returns: Remaining ref count ''' assert self.anitted, f'{self.__class__.__name__} initialized improperly. Must use Base.anit class method.' if self.isfini: return if __debug__: import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() == self.tid self._syn_refs -= 1 if self._syn_refs > 0: return self._syn_refs self.isfini = True fevt = self.finievt if fevt is not None: fevt.set() for base in list(self.tofini): await base.fini() try: await self._kill_active_tasks() except: logger.exception(f'{self} - Exception during _kill_active_tasks') for fini in self._fini_funcs: try: await s_coro.ornot(fini) except asyncio.CancelledError: raise except Exception: logger.exception(f'{self} - fini function failed: {fini}') self._syn_funcs.clear() self._fini_funcs.clear() return 0
async def __anit__(self, link, name): await s_base.Base.__anit__(self) self.tid = s_threads.iden() self.link = link self.name = name self.tasks = {} self.shares = {} self.sharinfo = {} self.methinfo = {} self.sess = None self.links = collections.deque() self._link_poolsize = 4 self.synack = None self.syndone = asyncio.Event() self.handlers = { 'task:fini': self._onTaskFini, 'share:data': self._onShareData, 'share:fini': self._onShareFini, } async def fini(): for item in list(self.shares.values()): await item.fini() mesg = ('task:fini', {'retn': (False, ('IsFini', {}))}) for name, task in list(self.tasks.items()): task.reply(mesg) del self.tasks[name] for link in self.links: await link.fini() del self.syndone await self.link.fini() self.onfini(fini) self.link.onfini(self.fini)
async def __anit__(self, link, name): await s_base.Base.__anit__(self) self.tid = s_threads.iden() self.link = link self.name = name self.tasks = {} self.shares = {} self.sharinfo = {} self.methinfo = {} self.sess = None self.links = collections.deque() self.synack = None self.syndone = asyncio.Event() self.handlers = { 'task:fini': self._onTaskFini, 'share:data': self._onShareData, 'share:fini': self._onShareFini, } async def fini(): for item in list(self.shares.values()): await item.fini() mesg = ('task:fini', {'retn': (False, ('IsFini', {}))}) for name, task in list(self.tasks.items()): task.reply(mesg) del self.tasks[name] for link in self.links: await link.fini() del self.syndone await self.link.fini() self.onfini(fini) self.link.onfini(self.fini)
def schedCoro(self, coro): ''' Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd. It does not pend on coroutine completion. Precondition: This function is *not* threadsafe and must be run on the Base's event loop Returns: asyncio.Task: An asyncio.Task object. ''' import synapse.lib.provenance as s_provenance # avoid import cycle if __debug__: assert inspect.isawaitable(coro) import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() == self.tid task = self.loop.create_task(coro) # In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context if asyncio.current_task(): s_provenance.dupstack(task) def taskDone(task): self._active_tasks.remove(task) try: if not task.done(): task.result() except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only pass except Exception: logger.exception( 'Task %s scheduled through Base.schedCoro raised exception', task) self._active_tasks.add(task) task.add_done_callback(taskDone) return task
def _getTxn(self, write=False): ''' Acquires a transaction. LMDB doesn't have the concept of store access without a transaction, so figure out whether there's already one open and use that, else make one. If we found an existing transaction, this doesn't close it after leaving the context. If we made one and the context is exited without exception, the transaction is committed. ''' existing_xact = self._store_xacts.get(s_threads.iden()) if existing_xact is not None: yield existing_xact.txn else: if write: with self._write_lock: self._ensure_map_slack() with self.dbenv.begin(buffers=True, write=True) as txn: yield txn else: with self.dbenv.begin(buffers=True, write=False) as txn: yield txn
async def __anit__(self, dirn, readonly=False): await s_layer.Layer.__anit__(self, dirn, readonly=readonly) path = os.path.join(self.dirn, 'layer.lmdb') self.fresh = not os.path.exists(path) mapsize = self.conf.get('lmdb:mapsize') readahead = self.conf.get('lmdb:readahead') maxsize = self.conf.get('lmdb:maxsize') growsize = self.conf.get('lmdb:growsize') self.layrslab = await s_lmdbslab.Slab.anit(path, max_dbs=128, map_size=mapsize, maxsize=maxsize, growsize=growsize, writemap=True, readahead=readahead, readonly=readonly) self.onfini(self.layrslab.fini) self.dbs = {} self.utf8 = s_layer.Utf8er() self.encoder = s_layer.Encoder() self.tid = s_threads.iden() self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu> self.byprop = await self.initdb('byprop', dupsort=True ) # <form>00<prop>00<indx>=<buid> self.byuniv = await self.initdb('byuniv', dupsort=True) # <prop>00<indx>=<buid> offsdb = await self.initdb('offsets') self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb) self.splicedb = await self.initdb('splices') self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')
def schedCoro(self, coro): ''' Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd. It does not pend on coroutine completion. Precondition: This function is *not* threadsafe and must be run on the Base's event loop Returns: asyncio.Task: An asyncio.Task object. ''' import synapse.lib.provenance as s_provenance # avoid import cycle if __debug__: assert s_coro.iscoro(coro) import synapse.lib.threads as s_threads # avoid import cycle assert s_threads.iden() == self.tid task = self.loop.create_task(coro) # In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context if asyncio.current_task(): s_provenance.dupstack(task) def taskDone(task): self._active_tasks.remove(task) try: task.result() except asyncio.CancelledError: pass except Exception: logger.exception('Task scheduled through Base.schedCoro raised exception') self._active_tasks.add(task) task.add_done_callback(taskDone) return task
def _popCoreXact(self): # Used by the CoreXact fini routine self._store_xacts.pop(s_threads.iden(), None)
def woot(): retn['iden'] = s_threads.iden() evnt.set()