async def __anit__(self):
self.loop = asyncio.get_running_loop()
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
self.tid = s_threads.iden()
self.call_stack = traceback.format_stack() # For cleanup debugging
self.isfini = False
self.anitted = True # For assertion purposes
self.finievt = None
self.entered = False
self.exitinfo = None
self.exitok = None
self.entered = False
self.exitinfo = None
# hold a weak ref to other bases we should fini if they
# are still around when we go down...
self.tofini = weakref.WeakSet()
self._syn_funcs = collections.defaultdict(list)
self._syn_refs = 1 # one ref for the ctor
self._syn_links = []
self._fini_funcs = []
self._fini_atexit = False
self._active_tasks = set() # the free running tasks associated with me
async def __anit__(self):
self.loop = asyncio.get_running_loop()
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
self.tid = s_threads.iden()
self.call_stack = traceback.format_stack() # For cleanup debugging
if object.__getattribute__(self, 'anitted') is True:
# The Base has already been anitted. This allows a class to treat
# multiple Base objects as a mixin and __anit__ themselves without
# smashing fini or event handlers from the others.
return
self.isfini = False
self.anitted = True # For assertion purposes
self.finievt = None
self.entered = False
self.exitinfo = None
self.exitok = None
self.entered = False
self.exitinfo = None
# hold a weak ref to other bases we should fini if they
# are still around when we go down...
self.tofini = weakref.WeakSet()
self._syn_funcs = collections.defaultdict(list)
self._syn_refs = 1 # one ref for the ctor
self._syn_links = []
self._fini_funcs = []
self._fini_atexit = False
self._active_tasks = set() # the free running tasks associated with me
def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
An asyncio.Task
'''
if __debug__:
assert s_coro.iscoro(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
def taskDone(task):
self._active_tasks.remove(task)
try:
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception(
'Task scheduled through Base.schedCoro raised exception')
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
def getCoreXact(self, size=1000, core=None):
'''
Get a Storage transaction context for use in a with block.
This object allows bulk storage layer optimization and proper ordering
of events.
Args:
size (int): Number of transactions to cache before starting to
execute storage layer events.
core: Cortex to attach to the StoreXact. Required for splice
event support.
Examples:
Get a StoreXact object and use it::
with store.getCoreXact() as xact:
store.dostuff()
Returns:
s_xact.StoreXact: A StoreXact object for the current thread.
'''
iden = s_threads.iden()
xact = self._store_xacts.get(iden)
if xact is not None:
return xact
xact = self.getStoreXact(size, core=core)
self._store_xacts[iden] = xact
return xact
async def __anit__(self):
self.loop = asyncio.get_running_loop()
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
self.tid = s_threads.iden()
self.call_stack = traceback.format_stack() # For cleanup debugging
self.isfini = False
self.anitted = True # For assertion purposes
self.finievt = None
self.entered = False
self.exitinfo = None
self.exitok = None
self.entered = False
self.exitinfo = None
# hold a weak ref to other bases we should fini if they
# are still around when we go down...
self.tofini = weakref.WeakSet()
self._syn_funcs = collections.defaultdict(list)
self._syn_refs = 1 # one ref for the ctor
self._syn_links = []
self._fini_funcs = []
self._fini_atexit = False
self._active_tasks = set() # the free running tasks associated with me
def __enter__(self):
'''
Convenience function to enable using Proxy objects as synchronous context managers.
Note:
This must not be used from async code, and it should never be used in core synapse code.
'''
if s_threads.iden() == self.tid:
raise s_exc.SynErr('Use of synchronous context manager in async code')
self._ctxobj = self.schedCoroSafePend(self.__aenter__())
return self
def __enter__(self):
'''
Convenience function to enable using Proxy objects as synchronous context managers.
Note:
This must not be used from async code, and it should never be used in core synapse code.
'''
if s_threads.iden() == self.tid:
raise s_exc.SynErr('Use of synchronous context manager in async code')
self._ctxobj = self.schedCoroSafePend(self.__aenter__())
return self
def __enter__(self):
'''
Convenience function to enable using Proxy objects as synchronous context managers.
Note:
This should never be used by synapse core code. This is for sync client code convenience only.
'''
if s_threads.iden() == self.tid:
raise s_exc.SynErr('Use of synchronous context manager in async code')
self._ctxobj = self.schedCoroSafePend(self.__aenter__())
return self
def __enter__(self):
'''
Convenience function to enable using Proxy objects as synchronous context managers.
Note:
This should never be used by synapse core code. This is for sync client code convenience only.
'''
if s_threads.iden() == self.tid:
raise s_exc.SynErr('Use of synchronous context manager in async code')
self._ctxobj = self.schedCoroSafePend(self.__aenter__())
return self
def schedCoroSafePend(self, coro):
'''
Schedules a coroutine to run as soon as possible on the same event loop that this Base is running on
Note:
This method may *not* be run inside an event loop
'''
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() != self.tid
task = asyncio.run_coroutine_threadsafe(coro, self.loop)
return task.result()
def schedCoroSafePend(self, coro):
'''
Schedules a coroutine to run as soon as possible on the same event loop that this Base is running on
Note:
This method may *not* be run inside an event loop
'''
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() != self.tid
task = asyncio.run_coroutine_threadsafe(coro, self.loop)
return task.result()
def test_glob_inpool(self):
iden = s_threads.iden()
retn = {}
evnt = threading.Event()
@s_glob.inpool
def woot():
retn['iden'] = s_threads.iden()
evnt.set()
woot()
evnt.wait(timeout=1)
self.ne(iden, retn.get('iden'))
async def fini(self):
'''
Shut down the object and notify any onfini() coroutines.
Returns:
Remaining ref count
'''
assert self.anitted, f'{self.__class__.__name__} initialized improperly. Must use Base.anit class method.'
if self.isfini:
return
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
self._syn_refs -= 1
if self._syn_refs > 0:
return self._syn_refs
self.isfini = True
for base in list(self.tofini):
await base.fini()
try:
await self._kill_active_tasks()
except Exception:
logger.exception(f'{self} - Exception during _kill_active_tasks')
for fini in self._fini_funcs:
try:
await s_coro.ornot(fini)
except asyncio.CancelledError:
raise
except Exception:
logger.exception(f'{self} - fini function failed: {fini}')
self._syn_funcs.clear()
self._fini_funcs.clear()
fevt = self.finievt
if fevt is not None:
fevt.set()
return 0
async def fini(self):
'''
Shut down the object and notify any onfini() coroutines.
Returns:
Remaining ref count
'''
assert self.anitted, f'{self.__class__.__name__} initialized improperly. Must use Base.anit class method.'
if self.isfini:
return
if __debug__:
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
self._syn_refs -= 1
if self._syn_refs > 0:
return self._syn_refs
self.isfini = True
fevt = self.finievt
if fevt is not None:
fevt.set()
for base in list(self.tofini):
await base.fini()
try:
await self._kill_active_tasks()
except:
logger.exception(f'{self} - Exception during _kill_active_tasks')
for fini in self._fini_funcs:
try:
await s_coro.ornot(fini)
except asyncio.CancelledError:
raise
except Exception:
logger.exception(f'{self} - fini function failed: {fini}')
self._syn_funcs.clear()
self._fini_funcs.clear()
return 0
async def __anit__(self, link, name):
await s_base.Base.__anit__(self)
self.tid = s_threads.iden()
self.link = link
self.name = name
self.tasks = {}
self.shares = {}
self.sharinfo = {}
self.methinfo = {}
self.sess = None
self.links = collections.deque()
self._link_poolsize = 4
self.synack = None
self.syndone = asyncio.Event()
self.handlers = {
'task:fini': self._onTaskFini,
'share:data': self._onShareData,
'share:fini': self._onShareFini,
}
async def fini():
for item in list(self.shares.values()):
await item.fini()
mesg = ('task:fini', {'retn': (False, ('IsFini', {}))})
for name, task in list(self.tasks.items()):
task.reply(mesg)
del self.tasks[name]
for link in self.links:
await link.fini()
del self.syndone
await self.link.fini()
self.onfini(fini)
self.link.onfini(self.fini)
async def __anit__(self, link, name):
await s_base.Base.__anit__(self)
self.tid = s_threads.iden()
self.link = link
self.name = name
self.tasks = {}
self.shares = {}
self.sharinfo = {}
self.methinfo = {}
self.sess = None
self.links = collections.deque()
self.synack = None
self.syndone = asyncio.Event()
self.handlers = {
'task:fini': self._onTaskFini,
'share:data': self._onShareData,
'share:fini': self._onShareFini,
}
async def fini():
for item in list(self.shares.values()):
await item.fini()
mesg = ('task:fini', {'retn': (False, ('IsFini', {}))})
for name, task in list(self.tasks.items()):
task.reply(mesg)
del self.tasks[name]
for link in self.links:
await link.fini()
del self.syndone
await self.link.fini()
self.onfini(fini)
self.link.onfini(self.fini)
def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
asyncio.Task: An asyncio.Task object.
'''
import synapse.lib.provenance as s_provenance # avoid import cycle
if __debug__:
assert inspect.isawaitable(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
# In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
if asyncio.current_task():
s_provenance.dupstack(task)
def taskDone(task):
self._active_tasks.remove(task)
try:
if not task.done():
task.result()
except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only
pass
except Exception:
logger.exception(
'Task %s scheduled through Base.schedCoro raised exception',
task)
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
def _getTxn(self, write=False):
'''
Acquires a transaction.
LMDB doesn't have the concept of store access without a transaction, so figure out
whether there's already one open and use that, else make one. If we found an existing
transaction, this doesn't close it after leaving the context. If we made one and the
context is exited without exception, the transaction is committed.
'''
existing_xact = self._store_xacts.get(s_threads.iden())
if existing_xact is not None:
yield existing_xact.txn
else:
if write:
with self._write_lock:
self._ensure_map_slack()
with self.dbenv.begin(buffers=True, write=True) as txn:
yield txn
else:
with self.dbenv.begin(buffers=True, write=False) as txn:
yield txn
async def __anit__(self, dirn, readonly=False):
await s_layer.Layer.__anit__(self, dirn, readonly=readonly)
path = os.path.join(self.dirn, 'layer.lmdb')
self.fresh = not os.path.exists(path)
mapsize = self.conf.get('lmdb:mapsize')
readahead = self.conf.get('lmdb:readahead')
maxsize = self.conf.get('lmdb:maxsize')
growsize = self.conf.get('lmdb:growsize')
self.layrslab = await s_lmdbslab.Slab.anit(path,
max_dbs=128,
map_size=mapsize,
maxsize=maxsize,
growsize=growsize,
writemap=True,
readahead=readahead,
readonly=readonly)
self.onfini(self.layrslab.fini)
self.dbs = {}
self.utf8 = s_layer.Utf8er()
self.encoder = s_layer.Encoder()
self.tid = s_threads.iden()
self.bybuid = await self.initdb('bybuid') # <buid><prop>=<valu>
self.byprop = await self.initdb('byprop', dupsort=True
) # <form>00<prop>00<indx>=<buid>
self.byuniv = await self.initdb('byuniv',
dupsort=True) # <prop>00<indx>=<buid>
offsdb = await self.initdb('offsets')
self.offs = s_slaboffs.SlabOffs(self.layrslab, offsdb)
self.splicedb = await self.initdb('splices')
self.splicelog = s_slabseqn.SlabSeqn(self.layrslab, 'splices')
def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
asyncio.Task: An asyncio.Task object.
'''
import synapse.lib.provenance as s_provenance # avoid import cycle
if __debug__:
assert s_coro.iscoro(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
# In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
if asyncio.current_task():
s_provenance.dupstack(task)
def taskDone(task):
self._active_tasks.remove(task)
try:
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception('Task scheduled through Base.schedCoro raised exception')
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
def _popCoreXact(self):
# Used by the CoreXact fini routine
self._store_xacts.pop(s_threads.iden(), None)
def woot():
retn['iden'] = s_threads.iden()
evnt.set()