예제 #1
0
    def schedCoro(self, coro):
        '''
        Schedules a free-running coroutine to run on this base's event loop.  Kills the coroutine if Base is fini'd.
        It does not pend on coroutine completion.

        Precondition:
            This function is *not* threadsafe and must be run on the Base's event loop

        Returns:
            An asyncio.Task

        '''
        if __debug__:
            assert s_coro.iscoro(coro)
            import synapse.lib.threads as s_threads  # avoid import cycle
            assert s_threads.iden() == self.tid

        task = self.loop.create_task(coro)

        def taskDone(task):
            self._active_tasks.remove(task)
            try:
                task.result()
            except asyncio.CancelledError:
                pass
            except Exception:
                logger.exception(
                    'Task scheduled through Base.schedCoro raised exception')

        self._active_tasks.add(task)
        task.add_done_callback(taskDone)

        return task
예제 #2
0
파일: snap.py 프로젝트: vivisect/synapse
    async def addFeedData(self, name, items, seqn=None):

        func = self.core.getFeedFunc(name)
        if func is None:
            raise s_exc.NoSuchName(name=name)

        logger.info(f'adding feed data ({name}): {len(items)} {seqn!r}')

        retn = func(self, items)

        # If the feed function is an async generator, run it...
        if isinstance(retn, types.AsyncGeneratorType):
            retn = [x async for x in retn]
        elif s_coro.iscoro(retn):
            await retn

        if seqn is not None:

            iden, offs = seqn

            nextoff = offs + len(items)

            await self.setOffset(iden, nextoff)

            return nextoff
예제 #3
0
def _fini_atexit(): # pragma: no cover

    for item in gc.get_objects():

        if not isinstance(item, Base):
            continue

        if not item.anitted:
            continue

        if item.isfini:
            continue

        if not item._fini_atexit and not OMIT_FINI_WARNS:
            if __debug__:
                print(f'At exit: Missing fini for {item}')
                for depth, call in enumerate(item.call_stack[:-2]):
                    print(f'{depth+1:3}: {call.strip()}')
            continue

        try:
            if __debug__:
                logger.debug('At exit: Calling fini for %r', item)
            rv = item.fini()
            if s_coro.iscoro(rv):
                # Try to run the fini on its loop
                loop = item.loop
                if not loop.is_running():
                    continue
                loop.create_task(rv)

        except Exception:
            logger.exception('atexit fini fail: %r' % (item,))
예제 #4
0
파일: base.py 프로젝트: vivisect/synapse
def _fini_atexit(): # pragma: no cover

    for item in gc.get_objects():

        if not isinstance(item, Base):
            continue

        if not item.anitted:
            continue

        if item.isfini:
            continue

        if not item._fini_atexit:
            if __debug__:
                print(f'At exit: Missing fini for {item}')
                for depth, call in enumerate(item.call_stack[:-2]):
                    print(f'{depth+1:3}: {call.strip()}')
            continue

        try:
            if __debug__:
                logger.debug('At exit: Calling fini for %r', item)
            rv = item.fini()
            if s_coro.iscoro(rv):
                # Try to run the fini on its loop
                loop = item.loop
                if not loop.is_running():
                    continue
                loop.create_task(rv)

        except Exception:
            logger.exception('atexit fini fail: %r' % (item,))
예제 #5
0
    async def addFeedData(self, name, items, seqn=None):

        func = self.core.getFeedFunc(name)
        if func is None:
            raise s_exc.NoSuchName(name=name)

        logger.info(f'adding feed data ({name}): {len(items)} {seqn!r}')

        retn = func(self, items)

        # If the feed function is an async generator, run it...
        if isinstance(retn, types.AsyncGeneratorType):
            retn = [x async for x in retn]
        elif s_coro.iscoro(retn):
            await retn

        if seqn is not None:

            iden, offs = seqn

            nextoff = offs + len(items)

            await self.setOffset(iden, nextoff)

            return nextoff
예제 #6
0
    async def test_coro_iscoro(self):
        async def agen():
            yield 42

        def genr():
            yield 'woot'

        async def woot():
            return 10

        item = woot()
        self.true(s_coro.iscoro(item))

        await item

        self.false(s_coro.iscoro(genr()))
        self.false(s_coro.iscoro(agen()))
예제 #7
0
    async def test_coro_iscoro(self):

        async def agen():
            yield 42

        def genr():
            yield 'woot'

        async def woot():
            return 10

        item = woot()
        self.true(s_coro.iscoro(item))

        await item

        self.false(s_coro.iscoro(genr()))
        self.false(s_coro.iscoro(agen()))
예제 #8
0
파일: daemon.py 프로젝트: vivisect/synapse
    async def _runTodoMeth(self, link, meth, args, kwargs):

        valu = meth(*args, **kwargs)

        for wraptype, wrapctor in dmonwrap:
            if isinstance(valu, wraptype):
                return await wrapctor.anit(link, valu)

        if s_coro.iscoro(valu):
            valu = await valu

        return valu
예제 #9
0
파일: daemon.py 프로젝트: rjammala/synapse
    async def _runTodoMeth(self, link, meth, args, kwargs):

        valu = meth(*args, **kwargs)

        for wraptype, wrapctor in dmonwrap:
            if isinstance(valu, wraptype):
                return await wrapctor.anit(link, valu)

        if s_coro.iscoro(valu):
            valu = await valu

        return valu
예제 #10
0
 async def wasDeleted(self, node):
     '''
     Fire the onDel() callbacks for node deletion.
     '''
     for func in self.ondels:
         try:
             retn = func(node)
             if s_coro.iscoro(retn):
                 await retn
         except asyncio.CancelledError:
             raise
         except Exception:
             logger.exception('error on ondel for %s' % (self.name, ))
예제 #11
0
파일: snap.py 프로젝트: freecamel/synapse
    async def addFeedData(self, name, items):

        func = self.core.getFeedFunc(name)
        if func is None:
            raise s_exc.NoSuchName(name=name)

        logger.info(f'adding feed data ({name}): {len(items)}')

        retn = func(self, items)

        # If the feed function is an async generator, run it...
        if isinstance(retn, types.AsyncGeneratorType):
            retn = [x async for x in retn]
        elif s_coro.iscoro(retn):
            await retn
예제 #12
0
    async def wasAdded(self, node):
        '''
        Fire the onAdd() callbacks for node creation.
        '''
        for func in self.onadds:
            try:
                retn = func(node)
                if s_coro.iscoro(retn):
                    await retn
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception('error on onadd for %s' % (self.name, ))

        await node.snap.core.triggers.runNodeAdd(node)
예제 #13
0
    async def wasDeleted(self, node):
        '''
        Fire the onDel() callbacks for node deletion.
        '''
        for func in self.ondels:
            try:
                retn = func(node)
                if s_coro.iscoro(retn):
                    await retn
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception('error on ondel for %s' % (self.name,))

        await node.snap.core.triggers.runNodeDel(node)
예제 #14
0
    async def wasAdded(self, node):
        '''
        Fire the onAdd() callbacks for node creation.
        '''
        waits = self.waits.pop(node.buid, None)
        if waits is not None:
            [e.set() for e in waits]

        for func in self.onadds:
            try:
                retn = func(node)
                if s_coro.iscoro(retn):
                    await retn
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception('error on onadd for %s' % (self.name, ))

        await node.snap.view.runNodeAdd(node)
예제 #15
0
    async def wasAdded(self, node):
        '''
        Fire the onAdd() callbacks for node creation.
        '''
        waits = self.waits.pop(node.buid, None)
        if waits is not None:
            [e.set() for e in waits]

        for func in self.onadds:
            try:
                retn = func(node)
                if s_coro.iscoro(retn):
                    await retn
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception('error on onadd for %s' % (self.name,))

        await node.snap.core.triggers.runNodeAdd(node)
예제 #16
0
    def schedCoro(self, coro):
        '''
        Schedules a free-running coroutine to run on this base's event loop.  Kills the coroutine if Base is fini'd.
        It does not pend on coroutine completion.

        Precondition:
            This function is *not* threadsafe and must be run on the Base's event loop

        Returns:
            asyncio.Task: An asyncio.Task object.

        '''
        import synapse.lib.provenance as s_provenance  # avoid import cycle

        if __debug__:
            assert s_coro.iscoro(coro)
            import synapse.lib.threads as s_threads  # avoid import cycle
            assert s_threads.iden() == self.tid

        task = self.loop.create_task(coro)

        # In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
        if asyncio.current_task():
            s_provenance.dupstack(task)

        def taskDone(task):
            self._active_tasks.remove(task)
            try:
                if not task.done():
                    task.result()
            except asyncio.CancelledError:
                pass
            except Exception:
                logger.exception(
                    'Task %s scheduled through Base.schedCoro raised exception',
                    task)

        self._active_tasks.add(task)
        task.add_done_callback(taskDone)

        return task
예제 #17
0
파일: base.py 프로젝트: vivisect/synapse
    def schedCoro(self, coro):
        '''
        Schedules a free-running coroutine to run on this base's event loop.  Kills the coroutine if Base is fini'd.
        It does not pend on coroutine completion.

        Precondition:
            This function is *not* threadsafe and must be run on the Base's event loop

        Returns:
            asyncio.Task: An asyncio.Task object.

        '''
        import synapse.lib.provenance as s_provenance  # avoid import cycle

        if __debug__:
            assert s_coro.iscoro(coro)
            import synapse.lib.threads as s_threads  # avoid import cycle
            assert s_threads.iden() == self.tid

        task = self.loop.create_task(coro)

        # In rare cases, (Like this function being triggered from call_soon_threadsafe), there's no task context
        if asyncio.current_task():
            s_provenance.dupstack(task)

        def taskDone(task):
            self._active_tasks.remove(task)
            try:
                task.result()
            except asyncio.CancelledError:
                pass
            except Exception:
                logger.exception('Task scheduled through Base.schedCoro raised exception')

        self._active_tasks.add(task)
        task.add_done_callback(taskDone)

        return task
예제 #18
0
파일: daemon.py 프로젝트: rjammala/synapse
    async def _onTaskV2Init(self, link, mesg):

        # t2:init is used by the pool sockets on the client

        name = mesg[1].get('name')
        sidn = mesg[1].get('sess')
        todo = mesg[1].get('todo')

        try:

            if sidn is None or todo is None:
                raise s_exc.NoSuchObj(name=name)

            sess = self.sessions.get(sidn)
            if sess is None:
                raise s_exc.NoSuchObj(name=name)

            item = sess.getSessItem(name)
            if item is None:
                raise s_exc.NoSuchObj(name=name)

            s_scope.set('sess', sess)
            # TODO set user....

            methname, args, kwargs = todo

            if methname[0] == '_':
                raise s_exc.NoSuchMeth(name=methname)

            meth = getattr(item, methname, None)
            if meth is None:
                logger.warning(f'{item!r} has no method: {methname}')
                raise s_exc.NoSuchMeth(name=methname)

            valu = meth(*args, **kwargs)

            if s_coro.iscoro(valu):
                valu = await valu

            if isinstance(valu, types.AsyncGeneratorType):

                try:

                    await link.tx(('t2:genr', {}))

                    async for item in valu:
                        await link.tx(('t2:yield', {'retn': (True, item)}))

                    await link.tx(('t2:yield', {'retn': None}))

                except Exception as e:
                    if not link.isfini:
                        retn = s_common.retnexc(e)
                        await link.tx(('t2:yield', {'retn': retn}))

                return

            if isinstance(valu, types.GeneratorType):

                try:

                    await link.tx(('t2:genr', {}))

                    for item in valu:
                        await link.tx(('t2:yield', {'retn': (True, item)}))

                    await link.tx(('t2:yield', {'retn': None}))

                except Exception as e:
                    if not link.isfini:
                        retn = s_common.retnexc(e)
                        await link.tx(('t2:yield', {'retn': (False, retn)}))

                return

            if isinstance(valu, s_share.Share):
                iden = s_common.guid()
                sess.setSessItem(iden, valu)
                await link.tx(('t2:share', {'iden': iden}))
                return

            await link.tx(('t2:fini', {'retn': (True, valu)}))

        except Exception as e:

            logger.exception('on task:init: %r' % (mesg, ))

            if not link.isfini:
                retn = s_common.retnexc(e)
                await link.tx(('t2:fini', {'retn': retn}))
예제 #19
0
파일: daemon.py 프로젝트: vivisect/synapse
    async def _onTaskV2Init(self, link, mesg):

        # t2:init is used by the pool sockets on the client

        name = mesg[1].get('name')
        sidn = mesg[1].get('sess')
        todo = mesg[1].get('todo')

        try:

            if sidn is None or todo is None:
                raise s_exc.NoSuchObj(name=name)

            sess = self.sessions.get(sidn)
            if sess is None:
                raise s_exc.NoSuchObj(name=name)

            item = sess.getSessItem(name)
            if item is None:
                raise s_exc.NoSuchObj(name=name)

            s_scope.set('sess', sess)
            # TODO set user....

            methname, args, kwargs = todo

            if methname[0] == '_':
                raise s_exc.NoSuchMeth(name=methname)

            meth = getattr(item, methname, None)
            if meth is None:
                logger.warning(f'{item!r} has no method: {methname}')
                raise s_exc.NoSuchMeth(name=methname)

            valu = meth(*args, **kwargs)

            if s_coro.iscoro(valu):
                valu = await valu

            try:
                if isinstance(valu, types.AsyncGeneratorType):
                    desc = 'async generator'

                    await link.tx(('t2:genr', {}))

                    async for item in valu:
                        await link.tx(('t2:yield', {'retn': (True, item)}))

                    await link.tx(('t2:yield', {'retn': None}))
                    return

                elif isinstance(valu, types.GeneratorType):
                    desc = 'generator'

                    await link.tx(('t2:genr', {}))

                    for item in valu:
                        await link.tx(('t2:yield', {'retn': (True, item)}))

                    await link.tx(('t2:yield', {'retn': None}))
                    return

            except Exception as e:
                logger.exception(f'error during {desc} task: {methname}')
                if not link.isfini:
                    retn = s_common.retnexc(e)
                    await link.tx(('t2:yield', {'retn': retn}))

                return

            if isinstance(valu, s_share.Share):
                sess.onfini(valu)
                iden = s_common.guid()
                sess.setSessItem(iden, valu)
                info = s_reflect.getShareInfo(valu)
                await link.tx(('t2:share', {'iden': iden, 'sharinfo': info}))
                return

            await link.tx(('t2:fini', {'retn': (True, valu)}))

        except Exception as e:

            logger.exception('on t2:init: %r' % (mesg,))

            if not link.isfini:
                retn = s_common.retnexc(e)
                await link.tx(('t2:fini', {'retn': retn}))
예제 #20
0
async def t2call(link, meth, args, kwargs):
    '''
    Call the given ``meth(*args, **kwargs)`` and handle the response to provide
    telepath task v2 events to the given link.
    '''
    try:

        valu = meth(*args, **kwargs)

        if s_coro.iscoro(valu):
            valu = await valu

        try:

            first = True
            if isinstance(valu, types.AsyncGeneratorType):

                async for item in valu:

                    if first:
                        await link.tx(('t2:genr', {}))
                        first = False

                    await link.tx(('t2:yield', {'retn': (True, item)}))

                if first:
                    await link.tx(('t2:genr', {}))

                await link.tx(('t2:yield', {'retn': None}))
                return

            elif isinstance(valu, types.GeneratorType):

                for item in valu:

                    if first:
                        await link.tx(('t2:genr', {}))
                        first = False

                    await link.tx(('t2:yield', {'retn': (True, item)}))

                if first:
                    await link.tx(('t2:genr', {}))

                await link.tx(('t2:yield', {'retn': None}))
                return

        except s_exc.DmonSpawn as e:
            context = e.__context__
            if context:
                if not isinstance(context, asyncio.CancelledError):
                    logger.error('Error during DmonSpawn call: %r', context)
                await link.fini()
            return

        except (asyncio.CancelledError, Exception) as e:

            if isinstance(e, asyncio.CancelledError):
                logger.info('t2call task %s cancelled', meth.__name__)
            else:
                logger.exception('error during task %s', meth.__name__)

            if isinstance(valu, types.AsyncGeneratorType):
                await valu.aclose()
            elif isinstance(valu, types.GeneratorType):
                valu.close()

            if not link.isfini:

                if first:
                    await link.tx(('t2:genr', {}))

                retn = s_common.retnexc(e)
                await link.tx(('t2:yield', {'retn': retn}))

            return

        if isinstance(valu, s_share.Share):

            info = s_reflect.getShareInfo(valu)
            await link.tx(('t2:share', {'iden': valu.iden, 'sharinfo': info}))
            return valu

        await link.tx(('t2:fini', {'retn': (True, valu)}))

    except s_exc.DmonSpawn as e:
        context = e.__context__
        if context:
            logger.error('Error during DmonSpawn call: %r', context)
            await link.fini()
        return

    except (asyncio.CancelledError, Exception) as e:
        logger.exception('error during task: %s', meth.__name__)
        if not link.isfini:
            retn = s_common.retnexc(e)
            await link.tx(('t2:fini', {'retn': retn}))