Ejemplo n.º 1
0
class DSession(Session):
    """ 
        Session drives the collection and running of tests
        and generates test events for reporters. 
    """ 
    MAXITEMSPERHOST = 15
    
    def __init__(self, config):
        self.queue = queue.Queue()
        self.node2pending = {}
        self.item2nodes = {}
        super(DSession, self).__init__(config=config)

    #def pytest_configure(self, __multicall__, config):
    #    __multicall__.execute()
    #    try:
    #        config.getxspecs()
    #    except config.Error:
    #        print
    #        raise config.Error("dist mode %r needs test execution environments, "
    #                           "none found." %(config.option.dist))

    def main(self, colitems=None):
        colitems = self.getinitialitems(colitems)
        self.sessionstarts()
        self.setup()
        exitstatus = self.loop(colitems)
        self.teardown()
        self.sessionfinishes(exitstatus=exitstatus) 
        return exitstatus

    def loop_once(self, loopstate):
        if loopstate.shuttingdown:
            return self.loop_once_shutdown(loopstate)
        colitems = loopstate.colitems 
        if loopstate.dowork and colitems:
            self.triggertesting(loopstate.colitems) 
            colitems[:] = []
        # we use a timeout here so that control-C gets through 
        while 1:
            try:
                eventcall = self.queue.get(timeout=2.0)
                break
            except queue.Empty:
                continue
        loopstate.dowork = True 
          
        callname, args, kwargs = eventcall
        if callname is not None:
            call = getattr(self.config.hook, callname)
            assert not args
            call(**kwargs)

        # termination conditions
        if ((loopstate.testsfailed and self.config.option.exitfirst) or 
            (not self.item2nodes and not colitems and not self.queue.qsize())):
            self.triggershutdown()
            loopstate.shuttingdown = True
        elif not self.node2pending:
            loopstate.exitstatus = outcome.EXIT_NOHOSTS
           
    def loop_once_shutdown(self, loopstate):
        # once we are in shutdown mode we dont send 
        # events other than HostDown upstream 
        eventname, args, kwargs = self.queue.get()
        if eventname == "pytest_testnodedown":
            self.config.hook.pytest_testnodedown(**kwargs)
            self.removenode(kwargs['node'])
        elif eventname == "pytest_runtest_logreport":
            # might be some teardown report
            self.config.hook.pytest_runtest_logreport(**kwargs)
        if not self.node2pending:
            # finished
            if loopstate.testsfailed:
                loopstate.exitstatus = outcome.EXIT_TESTSFAILED
            else:
                loopstate.exitstatus = outcome.EXIT_OK
        #self.config.pluginmanager.unregister(loopstate)

    def _initloopstate(self, colitems):
        loopstate = LoopState(self, colitems)
        self.config.pluginmanager.register(loopstate)
        return loopstate

    def loop(self, colitems):
        try:
            loopstate = self._initloopstate(colitems)
            loopstate.dowork = False # first receive at least one HostUp events
            while 1:
                self.loop_once(loopstate)
                if loopstate.exitstatus is not None:
                    exitstatus = loopstate.exitstatus
                    break 
        except KeyboardInterrupt:
            excinfo = py.code.ExceptionInfo()
            self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
            exitstatus = outcome.EXIT_INTERRUPTED
        except:
            self.config.pluginmanager.notify_exception()
            exitstatus = outcome.EXIT_INTERNALERROR
        self.config.pluginmanager.unregister(loopstate)
        if exitstatus == 0 and self._testsfailed:
            exitstatus = outcome.EXIT_TESTSFAILED
        return exitstatus

    def triggershutdown(self):
        for node in self.node2pending:
            node.shutdown()

    def addnode(self, node):
        assert node not in self.node2pending
        self.node2pending[node] = []

    def removenode(self, node):
        try:
            pending = self.node2pending.pop(node)
        except KeyError:
            # this happens if we didn't receive a testnodeready event yet
            return []
        for item in pending:
            l = self.item2nodes[item]
            l.remove(node)
            if not l:
                del self.item2nodes[item]
        return pending

    def triggertesting(self, colitems):
        colitems = self.filteritems(colitems)
        senditems = []
        for next in colitems:
            if isinstance(next, py.test.collect.Item):
                senditems.append(next)
            else:
                self.config.hook.pytest_collectstart(collector=next)
                colrep = self.config.hook.pytest_make_collect_report(collector=next)
                self.queueevent("pytest_collectreport", report=colrep)
        if self.config.option.dist == "each":
            self.senditems_each(senditems)
        else:
            # XXX assert self.config.option.dist == "load"
            self.senditems_load(senditems)

    def queueevent(self, eventname, **kwargs):
        self.queue.put((eventname, (), kwargs)) 

    def senditems_each(self, tosend):
        if not tosend:
            return 
        room = self.MAXITEMSPERHOST
        for node, pending in self.node2pending.items():
            room = min(self.MAXITEMSPERHOST - len(pending), room)
        sending = tosend[:room]
        for node, pending in self.node2pending.items():
            node.sendlist(sending)
            pending.extend(sending)
            for item in sending:
                nodes = self.item2nodes.setdefault(item, [])
                assert node not in nodes
                nodes.append(node)
                self.config.hook.pytest_itemstart(item=item, node=node)
        tosend[:] = tosend[room:]  # update inplace
        if tosend:
            # we have some left, give it to the main loop
            self.queueevent("pytest_rescheduleitems", items=tosend)

    def senditems_load(self, tosend):
        if not tosend:
            return 
        for node, pending in self.node2pending.items():
            room = self.MAXITEMSPERHOST - len(pending)
            if room > 0:
                sending = tosend[:room]
                node.sendlist(sending)
                for item in sending:
                    #assert item not in self.item2node, (
                    #    "sending same item %r to multiple "
                    #    "not implemented" %(item,))
                    self.item2nodes.setdefault(item, []).append(node)
                    self.config.hook.pytest_itemstart(item=item, node=node)
                pending.extend(sending)
                tosend[:] = tosend[room:]  # update inplace
                if not tosend:
                    break
        if tosend:
            # we have some left, give it to the main loop
            self.queueevent("pytest_rescheduleitems", items=tosend)

    def removeitem(self, item, node):
        if item not in self.item2nodes:
            raise AssertionError(item, self.item2nodes)
        nodes = self.item2nodes[item]
        if node in nodes: # the node might have gone down already
            nodes.remove(node)
        if not nodes:
            del self.item2nodes[item]
        pending = self.node2pending[node]
        pending.remove(item)

    def handle_crashitem(self, item, node):
        runner = item.config.pluginmanager.getplugin("runner") 
        info = "!!! Node %r crashed during running of test %r" %(node, item)
        rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
        rep.node = node
        self.config.hook.pytest_runtest_logreport(report=rep)

    def setup(self):
        """ setup any neccessary resources ahead of the test run. """
        self.nodemanager = NodeManager(self.config)
        self.nodemanager.setup_nodes(putevent=self.queue.put)
        if self.config.option.dist == "each":
            self.nodemanager.wait_nodesready(5.0)

    def teardown(self):
        """ teardown any resources after a test run. """ 
        self.nodemanager.teardown_nodes()
Ejemplo n.º 2
0
class DSession(Session):
    """ 
        Session drives the collection and running of tests
        and generates test events for reporters. 
    """
    MAXITEMSPERHOST = 15

    def __init__(self, config):
        self.queue = queue.Queue()
        self.node2pending = {}
        self.item2nodes = {}
        super(DSession, self).__init__(config=config)

    #def pytest_configure(self, __multicall__, config):
    #    __multicall__.execute()
    #    try:
    #        config.getxspecs()
    #    except config.Error:
    #        print
    #        raise config.Error("dist mode %r needs test execution environments, "
    #                           "none found." %(config.option.dist))

    def main(self, colitems=None):
        colitems = self.getinitialitems(colitems)
        self.sessionstarts()
        self.setup()
        exitstatus = self.loop(colitems)
        self.teardown()
        self.sessionfinishes(exitstatus=exitstatus)
        return exitstatus

    def loop_once(self, loopstate):
        if loopstate.shuttingdown:
            return self.loop_once_shutdown(loopstate)
        colitems = loopstate.colitems
        if loopstate.dowork and colitems:
            self.triggertesting(loopstate.colitems)
            colitems[:] = []
        # we use a timeout here so that control-C gets through
        while 1:
            try:
                eventcall = self.queue.get(timeout=2.0)
                break
            except queue.Empty:
                continue
        loopstate.dowork = True

        callname, args, kwargs = eventcall
        if callname is not None:
            call = getattr(self.config.hook, callname)
            assert not args
            call(**kwargs)

        # termination conditions
        if ((loopstate.testsfailed and self.config.option.exitfirst) or
            (not self.item2nodes and not colitems and not self.queue.qsize())):
            self.triggershutdown()
            loopstate.shuttingdown = True
        elif not self.node2pending:
            loopstate.exitstatus = outcome.EXIT_NOHOSTS

    def loop_once_shutdown(self, loopstate):
        # once we are in shutdown mode we dont send
        # events other than HostDown upstream
        eventname, args, kwargs = self.queue.get()
        if eventname == "pytest_testnodedown":
            self.config.hook.pytest_testnodedown(**kwargs)
            self.removenode(kwargs['node'])
        elif eventname == "pytest_runtest_logreport":
            # might be some teardown report
            self.config.hook.pytest_runtest_logreport(**kwargs)
        if not self.node2pending:
            # finished
            if loopstate.testsfailed:
                loopstate.exitstatus = outcome.EXIT_TESTSFAILED
            else:
                loopstate.exitstatus = outcome.EXIT_OK
        #self.config.pluginmanager.unregister(loopstate)

    def _initloopstate(self, colitems):
        loopstate = LoopState(self, colitems)
        self.config.pluginmanager.register(loopstate)
        return loopstate

    def loop(self, colitems):
        try:
            loopstate = self._initloopstate(colitems)
            loopstate.dowork = False  # first receive at least one HostUp events
            while 1:
                self.loop_once(loopstate)
                if loopstate.exitstatus is not None:
                    exitstatus = loopstate.exitstatus
                    break
        except KeyboardInterrupt:
            excinfo = py.code.ExceptionInfo()
            self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
            exitstatus = outcome.EXIT_INTERRUPTED
        except:
            self.config.pluginmanager.notify_exception()
            exitstatus = outcome.EXIT_INTERNALERROR
        self.config.pluginmanager.unregister(loopstate)
        if exitstatus == 0 and self._testsfailed:
            exitstatus = outcome.EXIT_TESTSFAILED
        return exitstatus

    def triggershutdown(self):
        for node in self.node2pending:
            node.shutdown()

    def addnode(self, node):
        assert node not in self.node2pending
        self.node2pending[node] = []

    def removenode(self, node):
        try:
            pending = self.node2pending.pop(node)
        except KeyError:
            # this happens if we didn't receive a testnodeready event yet
            return []
        for item in pending:
            l = self.item2nodes[item]
            l.remove(node)
            if not l:
                del self.item2nodes[item]
        return pending

    def triggertesting(self, colitems):
        colitems = self.filteritems(colitems)
        senditems = []
        for next in colitems:
            if isinstance(next, py.test.collect.Item):
                senditems.append(next)
            else:
                self.config.hook.pytest_collectstart(collector=next)
                colrep = self.config.hook.pytest_make_collect_report(
                    collector=next)
                self.queueevent("pytest_collectreport", report=colrep)
        if self.config.option.dist == "each":
            self.senditems_each(senditems)
        else:
            # XXX assert self.config.option.dist == "load"
            self.senditems_load(senditems)

    def queueevent(self, eventname, **kwargs):
        self.queue.put((eventname, (), kwargs))

    def senditems_each(self, tosend):
        if not tosend:
            return
        room = self.MAXITEMSPERHOST
        for node, pending in self.node2pending.items():
            room = min(self.MAXITEMSPERHOST - len(pending), room)
        sending = tosend[:room]
        for node, pending in self.node2pending.items():
            node.sendlist(sending)
            pending.extend(sending)
            for item in sending:
                nodes = self.item2nodes.setdefault(item, [])
                assert node not in nodes
                nodes.append(node)
                self.config.hook.pytest_itemstart(item=item, node=node)
        tosend[:] = tosend[room:]  # update inplace
        if tosend:
            # we have some left, give it to the main loop
            self.queueevent("pytest_rescheduleitems", items=tosend)

    def senditems_load(self, tosend):
        if not tosend:
            return
        for node, pending in self.node2pending.items():
            room = self.MAXITEMSPERHOST - len(pending)
            if room > 0:
                sending = tosend[:room]
                node.sendlist(sending)
                for item in sending:
                    #assert item not in self.item2node, (
                    #    "sending same item %r to multiple "
                    #    "not implemented" %(item,))
                    self.item2nodes.setdefault(item, []).append(node)
                    self.config.hook.pytest_itemstart(item=item, node=node)
                pending.extend(sending)
                tosend[:] = tosend[room:]  # update inplace
                if not tosend:
                    break
        if tosend:
            # we have some left, give it to the main loop
            self.queueevent("pytest_rescheduleitems", items=tosend)

    def removeitem(self, item, node):
        if item not in self.item2nodes:
            raise AssertionError(item, self.item2nodes)
        nodes = self.item2nodes[item]
        if node in nodes:  # the node might have gone down already
            nodes.remove(node)
        if not nodes:
            del self.item2nodes[item]
        pending = self.node2pending[node]
        pending.remove(item)

    def handle_crashitem(self, item, node):
        runner = item.config.pluginmanager.getplugin("runner")
        info = "!!! Node %r crashed during running of test %r" % (node, item)
        rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
        rep.node = node
        self.config.hook.pytest_runtest_logreport(report=rep)

    def setup(self):
        """ setup any neccessary resources ahead of the test run. """
        self.nodemanager = NodeManager(self.config)
        self.nodemanager.setup_nodes(putevent=self.queue.put)
        if self.config.option.dist == "each":
            self.nodemanager.wait_nodesready(5.0)

    def teardown(self):
        """ teardown any resources after a test run. """
        self.nodemanager.teardown_nodes()
Ejemplo n.º 3
0
 def setup(self):
     """ setup any neccessary resources ahead of the test run. """
     self.nodemanager = NodeManager(self.config)
     self.nodemanager.setup_nodes(putevent=self.queue.put)
     if self.config.option.dist == "each":
         self.nodemanager.wait_nodesready(5.0)
Ejemplo n.º 4
0
 def setup(self):
     """ setup any neccessary resources ahead of the test run. """
     self.nodemanager = NodeManager(self.config)
     self.nodemanager.setup_nodes(putevent=self.queue.put)
     if self.config.option.dist == "each":
         self.nodemanager.wait_nodesready(5.0)