def test_optimise_popen(self, source, dest):
specs = ["popen"] * 3
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = py.test.config._reparse([source])
nodemanager = NodeManager(config, specs)
nodemanager.rsync_roots()
for gwspec in nodemanager.gwmanager.specs:
assert gwspec._samefilesystem()
assert not gwspec.chdir
def test_optimise_popen(self, mysetup):
source, dest = mysetup.source, mysetup.dest
specs = ["popen"] * 3
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = py.test.config._reparse([source])
nodemanager = NodeManager(config, specs)
nodemanager.rsync_roots()
for gwspec in nodemanager.gwmanager.specs:
assert gwspec._samefilesystem()
assert not gwspec.chdir
def test_rsync_roots_no_roots(self, source, dest):
source.ensure("dir1", "file1").write("hello")
config = py.test.config._reparse([source])
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
assert nodemanager.config.topdir == source == config.topdir
nodemanager.rsync_roots()
p, = nodemanager.gwmanager.multi_exec("import os ; channel.send(os.getcwd())").receive_each()
p = py.path.local(p)
print "remote curdir", p
assert p == dest.join(config.topdir.basename)
assert p.join("dir1").check()
assert p.join("dir1", "file1").check()
def test_rsync_roots_no_roots(self, mysetup):
mysetup.source.ensure("dir1", "file1").write("hello")
config = py.test.config._reparse([source])
nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
assert nodemanager.config.topdir == source == config.topdir
nodemanager.rsync_roots()
p, = nodemanager.gwmanager.multi_exec(
"import os ; channel.send(os.getcwd())").receive_each()
p = py.path.local(p)
print "remote curdir", p
assert p == mysetup.dest.join(config.topdir.basename)
assert p.join("dir1").check()
assert p.join("dir1", "file1").check()
def test_setup_DEBUG(self, mysetup, testdir):
source = mysetup.source
specs = ["popen"] * 2
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = py.test.config._reparse([source, '--debug'])
assert config.option.debug
nodemanager = NodeManager(config, specs)
reprec = testdir.getreportrecorder(config).hookrecorder
nodemanager.setup_nodes(putevent=[].append)
for spec in nodemanager.gwmanager.specs:
l = reprec.getcalls("pytest_trace")
assert l
nodemanager.teardown_nodes()
def test_init_rsync_roots(self, source, dest):
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir1", "somefile", dir=1)
dir2.ensure("hello")
source.ensure("bogusdir", "file")
source.join("conftest.py").write(py.code.Source("""
rsyncdirs = ['dir1/dir2']
"""))
session = py.test.config._reparse([source]).initsession()
nodemanager = NodeManager(session.config, ["popen//chdir=%s" % dest])
nodemanager.rsync_roots()
assert dest.join("dir2").check()
assert not dest.join("dir1").check()
assert not dest.join("bogus").check()
def test_setup_DEBUG(self, source, EventRecorder):
specs = ["popen"] * 2
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = py.test.config._reparse([source, '--debug'])
assert config.option.debug
nodemanager = NodeManager(config, specs)
evrec = EventRecorder(config.bus, debug=True)
nodemanager.setup_nodes(putevent=[].append)
for spec in nodemanager.gwmanager.specs:
l = evrec.getnamed("trace")
print evrec.events
assert l
nodemanager.teardown_nodes()
def test_init_rsync_roots(self, mysetup):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir1", "somefile", dir=1)
dir2.ensure("hello")
source.ensure("bogusdir", "file")
source.join("conftest.py").write(
py.code.Source("""
rsyncdirs = ['dir1/dir2']
"""))
session = py.test.config._reparse([source]).initsession()
nodemanager = NodeManager(session.config, ["popen//chdir=%s" % dest])
nodemanager.rsync_roots()
assert dest.join("dir2").check()
assert not dest.join("dir1").check()
assert not dest.join("bogus").check()
def test_rsyncignore(self, mysetup):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
dir5 = source.ensure("dir5", "dir6", "bogus")
dirf = source.ensure("dir5", "file")
dir2.ensure("hello")
source.join("conftest.py").write(py.code.Source("""
rsyncdirs = ['dir1', 'dir5']
rsyncignore = ['dir1/dir2', 'dir5/dir6']
"""))
session = py.test.config._reparse([source]).initsession()
nodemanager = NodeManager(session.config,
["popen//chdir=%s" % dest])
nodemanager.rsync_roots()
assert dest.join("dir1").check()
assert not dest.join("dir1", "dir2").check()
assert dest.join("dir5","file").check()
assert not dest.join("dir6").check()
def test_popen_rsync_subdir(self, testdir, source, dest):
dir1 = source.mkdir("dir1")
dir2 = dir1.mkdir("dir2")
dir2.ensure("hello")
for rsyncroot in (dir1, source):
dest.remove()
nodemanager = NodeManager(testdir.parseconfig(
"--tx", "popen//chdir=%s" % dest,
"--rsyncdir", rsyncroot,
source,
))
assert nodemanager.config.topdir == source
nodemanager.rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
nodemanager.gwmanager.exit()
def test_popen_rsync_subdir(self, testdir, mysetup):
source, dest = mysetup.source, mysetup.dest
dir1 = mysetup.source.mkdir("dir1")
dir2 = dir1.mkdir("dir2")
dir2.ensure("hello")
for rsyncroot in (dir1, source):
dest.remove()
nodemanager = NodeManager(
testdir.parseconfig(
"--tx",
"popen//chdir=%s" % dest,
"--rsyncdir",
rsyncroot,
source,
))
assert nodemanager.config.topdir == source
nodemanager.rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
nodemanager.gwmanager.exit()
def test_setup_DEBUG(self, mysetup, testdir):
source = mysetup.source
specs = ["popen"] * 2
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = py.test.config._reparse([source, '--debug'])
assert config.option.debug
nodemanager = NodeManager(config, specs)
reprec = testdir.getreportrecorder(config).hookrecorder
nodemanager.setup_nodes(putevent=[].append)
for spec in nodemanager.gwmanager.specs:
l = reprec.getcalls("pytest_trace")
assert l
nodemanager.teardown_nodes()
class DSession(Session):
"""
Session drives the collection and running of tests
and generates test events for reporters.
"""
MAXITEMSPERHOST = 15
def __init__(self, config):
self.queue = Queue.Queue()
self.node2pending = {}
self.item2nodes = {}
super(DSession, self).__init__(config=config)
#def pytest_configure(self, __multicall__, config):
# __multicall__.execute()
# try:
# config.getxspecs()
# except config.Error:
# print
# raise config.Error("dist mode %r needs test execution environments, "
# "none found." %(config.option.dist))
def main(self, colitems=None):
colitems = self.getinitialitems(colitems)
self.sessionstarts()
self.setup()
exitstatus = self.loop(colitems)
self.teardown()
self.sessionfinishes(exitstatus=exitstatus)
return exitstatus
def loop_once(self, loopstate):
if loopstate.shuttingdown:
return self.loop_once_shutdown(loopstate)
colitems = loopstate.colitems
if loopstate.dowork and colitems:
self.triggertesting(loopstate.colitems)
colitems[:] = []
# we use a timeout here so that control-C gets through
while 1:
try:
eventcall = self.queue.get(timeout=2.0)
break
except Queue.Empty:
continue
loopstate.dowork = True
callname, args, kwargs = eventcall
if callname is not None:
call = getattr(self.config.hook, callname)
assert not args
call(**kwargs)
# termination conditions
if ((loopstate.testsfailed and self.config.option.exitfirst) or
(not self.item2nodes and not colitems and not self.queue.qsize())):
self.triggershutdown()
loopstate.shuttingdown = True
elif not self.node2pending:
loopstate.exitstatus = outcome.EXIT_NOHOSTS
def loop_once_shutdown(self, loopstate):
# once we are in shutdown mode we dont send
# events other than HostDown upstream
eventname, args, kwargs = self.queue.get()
if eventname == "pytest_testnodedown":
self.config.hook.pytest_testnodedown(**kwargs)
self.removenode(kwargs['node'])
elif eventname == "pytest_runtest_logreport":
# might be some teardown report
self.config.hook.pytest_runtest_logreport(**kwargs)
if not self.node2pending:
# finished
if loopstate.testsfailed:
loopstate.exitstatus = outcome.EXIT_TESTSFAILED
else:
loopstate.exitstatus = outcome.EXIT_OK
#self.config.pluginmanager.unregister(loopstate)
def _initloopstate(self, colitems):
loopstate = LoopState(self, colitems)
self.config.pluginmanager.register(loopstate)
return loopstate
def loop(self, colitems):
try:
loopstate = self._initloopstate(colitems)
loopstate.dowork = False # first receive at least one HostUp events
while 1:
self.loop_once(loopstate)
if loopstate.exitstatus is not None:
exitstatus = loopstate.exitstatus
break
except KeyboardInterrupt:
excinfo = py.code.ExceptionInfo()
self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
exitstatus = outcome.EXIT_INTERRUPTED
except:
self.config.pluginmanager.notify_exception()
exitstatus = outcome.EXIT_INTERNALERROR
self.config.pluginmanager.unregister(loopstate)
if exitstatus == 0 and self._testsfailed:
exitstatus = outcome.EXIT_TESTSFAILED
return exitstatus
def triggershutdown(self):
for node in self.node2pending:
node.shutdown()
def addnode(self, node):
assert node not in self.node2pending
self.node2pending[node] = []
def removenode(self, node):
try:
pending = self.node2pending.pop(node)
except KeyError:
# this happens if we didn't receive a testnodeready event yet
return []
for item in pending:
l = self.item2nodes[item]
l.remove(node)
if not l:
del self.item2nodes[item]
return pending
def triggertesting(self, colitems):
colitems = self.filteritems(colitems)
senditems = []
for next in colitems:
if isinstance(next, py.test.collect.Item):
senditems.append(next)
else:
self.config.hook.pytest_collectstart(collector=next)
colrep = self.config.hook.pytest_make_collect_report(collector=next)
self.queueevent("pytest_collectreport", report=colrep)
if self.config.option.dist == "each":
self.senditems_each(senditems)
else:
# XXX assert self.config.option.dist == "load"
self.senditems_load(senditems)
def queueevent(self, eventname, **kwargs):
self.queue.put((eventname, (), kwargs))
def senditems_each(self, tosend):
if not tosend:
return
room = self.MAXITEMSPERHOST
for node, pending in self.node2pending.items():
room = min(self.MAXITEMSPERHOST - len(pending), room)
sending = tosend[:room]
for node, pending in self.node2pending.items():
node.sendlist(sending)
pending.extend(sending)
for item in sending:
nodes = self.item2nodes.setdefault(item, [])
assert node not in nodes
nodes.append(node)
self.config.hook.pytest_itemstart(item=item, node=node)
tosend[:] = tosend[room:] # update inplace
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
def senditems_load(self, tosend):
if not tosend:
return
for node, pending in self.node2pending.items():
room = self.MAXITEMSPERHOST - len(pending)
if room > 0:
sending = tosend[:room]
node.sendlist(sending)
for item in sending:
#assert item not in self.item2node, (
# "sending same item %r to multiple "
# "not implemented" %(item,))
self.item2nodes.setdefault(item, []).append(node)
self.config.hook.pytest_itemstart(item=item, node=node)
pending.extend(sending)
tosend[:] = tosend[room:] # update inplace
if not tosend:
break
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
def removeitem(self, item, node):
if item not in self.item2nodes:
raise AssertionError(item, self.item2nodes)
nodes = self.item2nodes[item]
if node in nodes: # the node might have gone down already
nodes.remove(node)
if not nodes:
del self.item2nodes[item]
pending = self.node2pending[node]
pending.remove(item)
def handle_crashitem(self, item, node):
runner = item.config.pluginmanager.getplugin("runner")
info = "!!! Node %r crashed during running of test %r" %(node, item)
rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
rep.node = node
self.config.hook.pytest_runtest_logreport(report=rep)
def setup(self):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def teardown(self):
""" teardown any resources after a test run. """
self.nodemanager.teardown_nodes()
def test_popen_nodes_are_ready(self, testdir):
nodemanager = NodeManager(testdir.parseconfig(
"--tx", "3*popen"))
nodemanager.setup_nodes([].append)
nodemanager.wait_nodesready(timeout=2.0)
class DSession(Session):
"""
Session drives the collection and running of tests
and generates test events for reporters.
"""
MAXITEMSPERHOST = 15
def __init__(self, config):
self.queue = Queue.Queue()
self.node2pending = {}
self.item2nodes = {}
super(DSession, self).__init__(config=config)
#def pytest_configure(self, __multicall__, config):
# __multicall__.execute()
# try:
# config.getxspecs()
# except config.Error:
# print
# raise config.Error("dist mode %r needs test execution environments, "
# "none found." %(config.option.dist))
def main(self, colitems=None):
colitems = self.getinitialitems(colitems)
self.sessionstarts()
self.setup()
exitstatus = self.loop(colitems)
self.teardown()
self.sessionfinishes(exitstatus=exitstatus)
return exitstatus
def loop_once(self, loopstate):
if loopstate.shuttingdown:
return self.loop_once_shutdown(loopstate)
colitems = loopstate.colitems
if loopstate.dowork and colitems:
self.triggertesting(loopstate.colitems)
colitems[:] = []
# we use a timeout here so that control-C gets through
while 1:
try:
eventcall = self.queue.get(timeout=2.0)
break
except Queue.Empty:
continue
loopstate.dowork = True
callname, args, kwargs = eventcall
if callname is not None:
call = getattr(self.config.hook, callname)
assert not args
call(**kwargs)
# termination conditions
if ((loopstate.testsfailed and self.config.option.exitfirst) or
(not self.item2nodes and not colitems and not self.queue.qsize())):
self.triggershutdown()
loopstate.shuttingdown = True
elif not self.node2pending:
loopstate.exitstatus = outcome.EXIT_NOHOSTS
def loop_once_shutdown(self, loopstate):
# once we are in shutdown mode we dont send
# events other than HostDown upstream
eventname, args, kwargs = self.queue.get()
if eventname == "pytest_testnodedown":
self.config.hook.pytest_testnodedown(**kwargs)
self.removenode(kwargs['node'])
elif eventname == "pytest_runtest_logreport":
# might be some teardown report
self.config.hook.pytest_runtest_logreport(**kwargs)
if not self.node2pending:
# finished
if loopstate.testsfailed:
loopstate.exitstatus = outcome.EXIT_TESTSFAILED
else:
loopstate.exitstatus = outcome.EXIT_OK
#self.config.pluginmanager.unregister(loopstate)
def _initloopstate(self, colitems):
loopstate = LoopState(self, colitems)
self.config.pluginmanager.register(loopstate)
return loopstate
def loop(self, colitems):
try:
loopstate = self._initloopstate(colitems)
loopstate.dowork = False # first receive at least one HostUp events
while 1:
self.loop_once(loopstate)
if loopstate.exitstatus is not None:
exitstatus = loopstate.exitstatus
break
except KeyboardInterrupt:
excinfo = py.code.ExceptionInfo()
self.config.hook.pytest_keyboard_interrupt(excinfo=excinfo)
exitstatus = outcome.EXIT_INTERRUPTED
except:
self.config.pluginmanager.notify_exception()
exitstatus = outcome.EXIT_INTERNALERROR
self.config.pluginmanager.unregister(loopstate)
if exitstatus == 0 and self._testsfailed:
exitstatus = outcome.EXIT_TESTSFAILED
return exitstatus
def triggershutdown(self):
for node in self.node2pending:
node.shutdown()
def addnode(self, node):
assert node not in self.node2pending
self.node2pending[node] = []
def removenode(self, node):
try:
pending = self.node2pending.pop(node)
except KeyError:
# this happens if we didn't receive a testnodeready event yet
return []
for item in pending:
l = self.item2nodes[item]
l.remove(node)
if not l:
del self.item2nodes[item]
return pending
def triggertesting(self, colitems):
colitems = self.filteritems(colitems)
senditems = []
for next in colitems:
if isinstance(next, py.test.collect.Item):
senditems.append(next)
else:
self.config.hook.pytest_collectstart(collector=next)
colrep = self.config.hook.pytest_make_collect_report(
collector=next)
self.queueevent("pytest_collectreport", report=colrep)
if self.config.option.dist == "each":
self.senditems_each(senditems)
else:
# XXX assert self.config.option.dist == "load"
self.senditems_load(senditems)
def queueevent(self, eventname, **kwargs):
self.queue.put((eventname, (), kwargs))
def senditems_each(self, tosend):
if not tosend:
return
room = self.MAXITEMSPERHOST
for node, pending in self.node2pending.items():
room = min(self.MAXITEMSPERHOST - len(pending), room)
sending = tosend[:room]
for node, pending in self.node2pending.items():
node.sendlist(sending)
pending.extend(sending)
for item in sending:
nodes = self.item2nodes.setdefault(item, [])
assert node not in nodes
nodes.append(node)
self.config.hook.pytest_itemstart(item=item, node=node)
tosend[:] = tosend[room:] # update inplace
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
def senditems_load(self, tosend):
if not tosend:
return
for node, pending in self.node2pending.items():
room = self.MAXITEMSPERHOST - len(pending)
if room > 0:
sending = tosend[:room]
node.sendlist(sending)
for item in sending:
#assert item not in self.item2node, (
# "sending same item %r to multiple "
# "not implemented" %(item,))
self.item2nodes.setdefault(item, []).append(node)
self.config.hook.pytest_itemstart(item=item, node=node)
pending.extend(sending)
tosend[:] = tosend[room:] # update inplace
if not tosend:
break
if tosend:
# we have some left, give it to the main loop
self.queueevent("pytest_rescheduleitems", items=tosend)
def removeitem(self, item, node):
if item not in self.item2nodes:
raise AssertionError(item, self.item2nodes)
nodes = self.item2nodes[item]
if node in nodes: # the node might have gone down already
nodes.remove(node)
if not nodes:
del self.item2nodes[item]
pending = self.node2pending[node]
pending.remove(item)
def handle_crashitem(self, item, node):
runner = item.config.pluginmanager.getplugin("runner")
info = "!!! Node %r crashed during running of test %r" % (node, item)
rep = runner.ItemTestReport(item=item, excinfo=info, when="???")
rep.node = node
self.config.hook.pytest_runtest_logreport(report=rep)
def setup(self):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def teardown(self):
""" teardown any resources after a test run. """
self.nodemanager.teardown_nodes()
def setup(self):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def setup(self):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def test_popen_nodes_are_ready(self, testdir):
nodemanager = NodeManager(testdir.parseconfig("--tx", "3*popen"))
nodemanager.setup_nodes([].append)
nodemanager.wait_nodesready(timeout=2.0)