def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
An asyncio.Task
'''
if __debug__:
assert s_coro.iscoro(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
def taskDone(task):
self._active_tasks.remove(task)
try:
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception(
'Task scheduled through Base.schedCoro raised exception')
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
async def addFeedData(self, name, items, seqn=None):
func = self.core.getFeedFunc(name)
if func is None:
raise s_exc.NoSuchName(name=name)
logger.info(f'adding feed data ({name}): {len(items)} {seqn!r}')
retn = func(self, items)
# If the feed function is an async generator, run it...
if isinstance(retn, types.AsyncGeneratorType):
retn = [x async for x in retn]
elif s_coro.iscoro(retn):
await retn
if seqn is not None:
iden, offs = seqn
nextoff = offs + len(items)
await self.setOffset(iden, nextoff)
return nextoff
def _fini_atexit(): # pragma: no cover
for item in gc.get_objects():
if not isinstance(item, Base):
continue
if not item.anitted:
continue
if item.isfini:
continue
if not item._fini_atexit and not OMIT_FINI_WARNS:
if __debug__:
print(f'At exit: Missing fini for {item}')
for depth, call in enumerate(item.call_stack[:-2]):
print(f'{depth+1:3}: {call.strip()}')
continue
try:
if __debug__:
logger.debug('At exit: Calling fini for %r', item)
rv = item.fini()
if s_coro.iscoro(rv):
# Try to run the fini on its loop
loop = item.loop
if not loop.is_running():
continue
loop.create_task(rv)
except Exception:
logger.exception('atexit fini fail: %r' % (item,))
def _fini_atexit(): # pragma: no cover
for item in gc.get_objects():
if not isinstance(item, Base):
continue
if not item.anitted:
continue
if item.isfini:
continue
if not item._fini_atexit:
if __debug__:
print(f'At exit: Missing fini for {item}')
for depth, call in enumerate(item.call_stack[:-2]):
print(f'{depth+1:3}: {call.strip()}')
continue
try:
if __debug__:
logger.debug('At exit: Calling fini for %r', item)
rv = item.fini()
if s_coro.iscoro(rv):
# Try to run the fini on its loop
loop = item.loop
if not loop.is_running():
continue
loop.create_task(rv)
except Exception:
logger.exception('atexit fini fail: %r' % (item,))
async def addFeedData(self, name, items, seqn=None):
func = self.core.getFeedFunc(name)
if func is None:
raise s_exc.NoSuchName(name=name)
logger.info(f'adding feed data ({name}): {len(items)} {seqn!r}')
retn = func(self, items)
# If the feed function is an async generator, run it...
if isinstance(retn, types.AsyncGeneratorType):
retn = [x async for x in retn]
elif s_coro.iscoro(retn):
await retn
if seqn is not None:
iden, offs = seqn
nextoff = offs + len(items)
await self.setOffset(iden, nextoff)
return nextoff
async def test_coro_iscoro(self):
async def agen():
yield 42
def genr():
yield 'woot'
async def woot():
return 10
item = woot()
self.true(s_coro.iscoro(item))
await item
self.false(s_coro.iscoro(genr()))
self.false(s_coro.iscoro(agen()))
async def test_coro_iscoro(self):
async def agen():
yield 42
def genr():
yield 'woot'
async def woot():
return 10
item = woot()
self.true(s_coro.iscoro(item))
await item
self.false(s_coro.iscoro(genr()))
self.false(s_coro.iscoro(agen()))
async def _runTodoMeth(self, link, meth, args, kwargs):
valu = meth(*args, **kwargs)
for wraptype, wrapctor in dmonwrap:
if isinstance(valu, wraptype):
return await wrapctor.anit(link, valu)
if s_coro.iscoro(valu):
valu = await valu
return valu
async def _runTodoMeth(self, link, meth, args, kwargs):
valu = meth(*args, **kwargs)
for wraptype, wrapctor in dmonwrap:
if isinstance(valu, wraptype):
return await wrapctor.anit(link, valu)
if s_coro.iscoro(valu):
valu = await valu
return valu
async def wasDeleted(self, node):
'''
Fire the onDel() callbacks for node deletion.
'''
for func in self.ondels:
try:
retn = func(node)
if s_coro.iscoro(retn):
await retn
except asyncio.CancelledError:
raise
except Exception:
logger.exception('error on ondel for %s' % (self.name, ))
async def addFeedData(self, name, items):
func = self.core.getFeedFunc(name)
if func is None:
raise s_exc.NoSuchName(name=name)
logger.info(f'adding feed data ({name}): {len(items)}')
retn = func(self, items)
# If the feed function is an async generator, run it...
if isinstance(retn, types.AsyncGeneratorType):
retn = [x async for x in retn]
elif s_coro.iscoro(retn):
await retn
async def wasAdded(self, node):
'''
Fire the onAdd() callbacks for node creation.
'''
for func in self.onadds:
try:
retn = func(node)
if s_coro.iscoro(retn):
await retn
except asyncio.CancelledError:
raise
except Exception:
logger.exception('error on onadd for %s' % (self.name, ))
await node.snap.core.triggers.runNodeAdd(node)
async def wasDeleted(self, node):
'''
Fire the onDel() callbacks for node deletion.
'''
for func in self.ondels:
try:
retn = func(node)
if s_coro.iscoro(retn):
await retn
except asyncio.CancelledError:
raise
except Exception:
logger.exception('error on ondel for %s' % (self.name,))
await node.snap.core.triggers.runNodeDel(node)
async def wasAdded(self, node):
'''
Fire the onAdd() callbacks for node creation.
'''
waits = self.waits.pop(node.buid, None)
if waits is not None:
[e.set() for e in waits]
for func in self.onadds:
try:
retn = func(node)
if s_coro.iscoro(retn):
await retn
except asyncio.CancelledError:
raise
except Exception:
logger.exception('error on onadd for %s' % (self.name, ))
await node.snap.view.runNodeAdd(node)
async def wasAdded(self, node):
'''
Fire the onAdd() callbacks for node creation.
'''
waits = self.waits.pop(node.buid, None)
if waits is not None:
[e.set() for e in waits]
for func in self.onadds:
try:
retn = func(node)
if s_coro.iscoro(retn):
await retn
except asyncio.CancelledError:
raise
except Exception:
logger.exception('error on onadd for %s' % (self.name,))
await node.snap.core.triggers.runNodeAdd(node)
def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
asyncio.Task: An asyncio.Task object.
'''
import synapse.lib.provenance as s_provenance # avoid import cycle
if __debug__:
assert s_coro.iscoro(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
# In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
if asyncio.current_task():
s_provenance.dupstack(task)
def taskDone(task):
self._active_tasks.remove(task)
try:
if not task.done():
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception(
'Task %s scheduled through Base.schedCoro raised exception',
task)
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
def schedCoro(self, coro):
'''
Schedules a free-running coroutine to run on this base's event loop. Kills the coroutine if Base is fini'd.
It does not pend on coroutine completion.
Precondition:
This function is *not* threadsafe and must be run on the Base's event loop
Returns:
asyncio.Task: An asyncio.Task object.
'''
import synapse.lib.provenance as s_provenance # avoid import cycle
if __debug__:
assert s_coro.iscoro(coro)
import synapse.lib.threads as s_threads # avoid import cycle
assert s_threads.iden() == self.tid
task = self.loop.create_task(coro)
# In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
if asyncio.current_task():
s_provenance.dupstack(task)
def taskDone(task):
self._active_tasks.remove(task)
try:
task.result()
except asyncio.CancelledError:
pass
except Exception:
logger.exception('Task scheduled through Base.schedCoro raised exception')
self._active_tasks.add(task)
task.add_done_callback(taskDone)
return task
async def _onTaskV2Init(self, link, mesg):
# t2:init is used by the pool sockets on the client
name = mesg[1].get('name')
sidn = mesg[1].get('sess')
todo = mesg[1].get('todo')
try:
if sidn is None or todo is None:
raise s_exc.NoSuchObj(name=name)
sess = self.sessions.get(sidn)
if sess is None:
raise s_exc.NoSuchObj(name=name)
item = sess.getSessItem(name)
if item is None:
raise s_exc.NoSuchObj(name=name)
s_scope.set('sess', sess)
# TODO set user....
methname, args, kwargs = todo
if methname[0] == '_':
raise s_exc.NoSuchMeth(name=methname)
meth = getattr(item, methname, None)
if meth is None:
logger.warning(f'{item!r} has no method: {methname}')
raise s_exc.NoSuchMeth(name=methname)
valu = meth(*args, **kwargs)
if s_coro.iscoro(valu):
valu = await valu
if isinstance(valu, types.AsyncGeneratorType):
try:
await link.tx(('t2:genr', {}))
async for item in valu:
await link.tx(('t2:yield', {'retn': (True, item)}))
await link.tx(('t2:yield', {'retn': None}))
except Exception as e:
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:yield', {'retn': retn}))
return
if isinstance(valu, types.GeneratorType):
try:
await link.tx(('t2:genr', {}))
for item in valu:
await link.tx(('t2:yield', {'retn': (True, item)}))
await link.tx(('t2:yield', {'retn': None}))
except Exception as e:
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:yield', {'retn': (False, retn)}))
return
if isinstance(valu, s_share.Share):
iden = s_common.guid()
sess.setSessItem(iden, valu)
await link.tx(('t2:share', {'iden': iden}))
return
await link.tx(('t2:fini', {'retn': (True, valu)}))
except Exception as e:
logger.exception('on task:init: %r' % (mesg, ))
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:fini', {'retn': retn}))
async def _onTaskV2Init(self, link, mesg):
# t2:init is used by the pool sockets on the client
name = mesg[1].get('name')
sidn = mesg[1].get('sess')
todo = mesg[1].get('todo')
try:
if sidn is None or todo is None:
raise s_exc.NoSuchObj(name=name)
sess = self.sessions.get(sidn)
if sess is None:
raise s_exc.NoSuchObj(name=name)
item = sess.getSessItem(name)
if item is None:
raise s_exc.NoSuchObj(name=name)
s_scope.set('sess', sess)
# TODO set user....
methname, args, kwargs = todo
if methname[0] == '_':
raise s_exc.NoSuchMeth(name=methname)
meth = getattr(item, methname, None)
if meth is None:
logger.warning(f'{item!r} has no method: {methname}')
raise s_exc.NoSuchMeth(name=methname)
valu = meth(*args, **kwargs)
if s_coro.iscoro(valu):
valu = await valu
try:
if isinstance(valu, types.AsyncGeneratorType):
desc = 'async generator'
await link.tx(('t2:genr', {}))
async for item in valu:
await link.tx(('t2:yield', {'retn': (True, item)}))
await link.tx(('t2:yield', {'retn': None}))
return
elif isinstance(valu, types.GeneratorType):
desc = 'generator'
await link.tx(('t2:genr', {}))
for item in valu:
await link.tx(('t2:yield', {'retn': (True, item)}))
await link.tx(('t2:yield', {'retn': None}))
return
except Exception as e:
logger.exception(f'error during {desc} task: {methname}')
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:yield', {'retn': retn}))
return
if isinstance(valu, s_share.Share):
sess.onfini(valu)
iden = s_common.guid()
sess.setSessItem(iden, valu)
info = s_reflect.getShareInfo(valu)
await link.tx(('t2:share', {'iden': iden, 'sharinfo': info}))
return
await link.tx(('t2:fini', {'retn': (True, valu)}))
except Exception as e:
logger.exception('on t2:init: %r' % (mesg,))
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:fini', {'retn': retn}))
async def t2call(link, meth, args, kwargs):
'''
Call the given ``meth(*args, **kwargs)`` and handle the response to provide
telepath task v2 events to the given link.
'''
try:
valu = meth(*args, **kwargs)
if s_coro.iscoro(valu):
valu = await valu
try:
first = True
if isinstance(valu, types.AsyncGeneratorType):
async for item in valu:
if first:
await link.tx(('t2:genr', {}))
first = False
await link.tx(('t2:yield', {'retn': (True, item)}))
if first:
await link.tx(('t2:genr', {}))
await link.tx(('t2:yield', {'retn': None}))
return
elif isinstance(valu, types.GeneratorType):
for item in valu:
if first:
await link.tx(('t2:genr', {}))
first = False
await link.tx(('t2:yield', {'retn': (True, item)}))
if first:
await link.tx(('t2:genr', {}))
await link.tx(('t2:yield', {'retn': None}))
return
except s_exc.DmonSpawn as e:
context = e.__context__
if context:
if not isinstance(context, asyncio.CancelledError):
logger.error('Error during DmonSpawn call: %r', context)
await link.fini()
return
except (asyncio.CancelledError, Exception) as e:
if isinstance(e, asyncio.CancelledError):
logger.info('t2call task %s cancelled', meth.__name__)
else:
logger.exception('error during task %s', meth.__name__)
if isinstance(valu, types.AsyncGeneratorType):
await valu.aclose()
elif isinstance(valu, types.GeneratorType):
valu.close()
if not link.isfini:
if first:
await link.tx(('t2:genr', {}))
retn = s_common.retnexc(e)
await link.tx(('t2:yield', {'retn': retn}))
return
if isinstance(valu, s_share.Share):
info = s_reflect.getShareInfo(valu)
await link.tx(('t2:share', {'iden': valu.iden, 'sharinfo': info}))
return valu
await link.tx(('t2:fini', {'retn': (True, valu)}))
except s_exc.DmonSpawn as e:
context = e.__context__
if context:
logger.error('Error during DmonSpawn call: %r', context)
await link.fini()
return
except (asyncio.CancelledError, Exception) as e:
logger.exception('error during task: %s', meth.__name__)
if not link.isfini:
retn = s_common.retnexc(e)
await link.tx(('t2:fini', {'retn': retn}))
