def test_getrsyncdirs(self, testdir):
config = testdir.parseconfigure('--rsyncdir=' + str(testdir.tmpdir))
nm = NodeManager(config, specs=[execnet.XSpec("popen")])
assert not nm._getrsyncdirs()
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert nm.roots
assert testdir.tmpdir in nm.roots
def test_popens_rsync(self, config, mysetup, slavecontroller):
source = mysetup.source
hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None)
assert len(hm.group) == 2
for gw in hm.group:
class pseudoexec:
args = []
def __init__(self, *args):
self.args.extend(args)
def waitclose(self):
pass
gw.remote_exec = pseudoexec
notifications = []
for gw in hm.group:
hm.rsync(gw,
source,
notify=lambda *args: notifications.append(args))
assert not notifications
hm.teardown_nodes()
assert not len(hm.group)
assert "sys.path.insert" in gw.remote_exec.args[0]
def test_getxspecs(self, testdir):
config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz")
nodemanager = NodeManager(config)
xspecs = nodemanager._getxspecs()
assert len(xspecs) == 2
print(xspecs)
assert xspecs[0].popen
assert xspecs[1].ssh == "xyz"
def pytest_sessionstart(self, session):
"""Creates and starts the nodes.
The nodes are setup to put their events onto self.queue. As
soon as nodes start they will emit the slave_slaveready event.
"""
self.nodemanager = NodeManager(self.config)
nodes = self.nodemanager.setup_nodes(putevent=self.queue.put)
self._active_nodes.update(nodes)
self._session = session
def test_optimise_popen(self, testdir, mysetup, slavecontroller):
source = mysetup.source
specs = ["popen"] * 3
source.join("conftest.py").write("rsyncdirs = ['a']")
source.ensure('a', dir=1)
config = testdir.parseconfig(source)
nodemanager = NodeManager(config, specs)
nodemanager.setup_nodes(None) # calls .rysnc_roots()
for gwspec in nodemanager.specs:
assert gwspec._samefilesystem()
assert not gwspec.chdir
def test_rsync_roots_no_roots(self, testdir, mysetup):
mysetup.source.ensure("dir1", "file1").write("hello")
config = testdir.parseconfig(mysetup.source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest])
# assert nodemanager.config.topdir == source == config.topdir
nodemanager.makegateways()
nodemanager.rsync_roots()
p, = nodemanager.gwmanager.multi_exec(
"import os ; channel.send(os.getcwd())").receive_each()
p = py.path.local(p)
py.builtin.print_("remote curdir", p)
assert p == mysetup.dest.join(config.topdir.basename)
assert p.join("dir1").check()
assert p.join("dir1", "file1").check()
def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder,
slavecontroller):
source, dest = mysetup.source, mysetup.dest
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2)
hm.roots = []
hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello")
gw = hm.group[0]
hm.rsync(gw, source)
call = hookrecorder.popcall("pytest_xdist_rsyncstart")
assert call.source == source
assert len(call.gateways) == 1
assert call.gateways[0] in hm.group
call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
def test_rsync_popen_with_path(self, config, mysetup, slavecontroller):
source, dest = mysetup.source, mysetup.dest
hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1)
hm.setup_nodes(None)
source.ensure("dir1", "dir2", "hello")
l = []
for gw in hm.group:
hm.rsync(gw, source, notify=lambda *args: l.append(args))
assert len(l) == 1
assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source)
hm.teardown_nodes()
dest = dest.join(source.basename)
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
def test_popen_makegateway_events(self, config, hookrecorder,
slavecontroller):
hm = NodeManager(config, ["popen"] * 2)
hm.setup_nodes(None)
call = hookrecorder.popcall("pytest_xdist_setupnodes")
assert len(call.specs) == 2
call = hookrecorder.popcall("pytest_xdist_newgateway")
assert call.gateway.spec == execnet.XSpec("popen")
assert call.gateway.id == "gw0"
call = hookrecorder.popcall("pytest_xdist_newgateway")
assert call.gateway.id == "gw1"
assert len(hm.group) == 2
hm.teardown_nodes()
assert not len(hm.group)
def test_init_rsync_roots(self, testdir, mysetup, slavecontroller):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir1", "somefile", dir=1)
dir2.ensure("hello")
source.ensure("bogusdir", "file")
source.join("tox.ini").write(py.std.textwrap.dedent("""
[pytest]
rsyncdirs=dir1/dir2
"""))
config = testdir.parseconfig(source)
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir2").check()
assert not dest.join("dir1").check()
assert not dest.join("bogus").check()
def test_getrsyncdirs_with_conftest(self, testdir):
p = py.path.local()
for bn in 'x y z'.split():
p.mkdir(bn)
testdir.makeini("""
[pytest]
rsyncdirs= x
""")
config = testdir.parseconfigure(testdir.tmpdir, '--rsyncdir=y',
'--rsyncdir=z')
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")])
roots = nm._getrsyncdirs()
# assert len(roots) == 3 + 1 # pylib
assert py.path.local('y') in roots
assert py.path.local('z') in roots
assert testdir.tmpdir.join('x') in roots
def test_popen_rsync_subdir(self, testdir, mysetup, slavecontroller):
source, dest = mysetup.source, mysetup.dest
dir1 = mysetup.source.mkdir("dir1")
dir2 = dir1.mkdir("dir2")
dir2.ensure("hello")
for rsyncroot in (dir1, source):
dest.remove()
nodemanager = NodeManager(testdir.parseconfig(
"--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot,
source, ))
nodemanager.setup_nodes(None) # calls .rsync_roots()
if rsyncroot == source:
dest = dest.join("source")
assert dest.join("dir1").check()
assert dest.join("dir1", "dir2").check()
assert dest.join("dir1", "dir2", 'hello').check()
nodemanager.teardown_nodes()
def test_rsyncignore(self, testdir, mysetup, slavecontroller):
source, dest = mysetup.source, mysetup.dest
dir2 = source.ensure("dir1", "dir2", dir=1)
source.ensure("dir5", "dir6", "bogus")
source.ensure("dir5", "file")
dir2.ensure("hello")
source.ensure("foo", "bar")
source.ensure("bar", "foo")
source.join("tox.ini").write(py.std.textwrap.dedent("""
[pytest]
rsyncdirs = dir1 dir5
rsyncignore = dir1/dir2 dir5/dir6 foo*
"""))
config = testdir.parseconfig(source)
config.option.rsyncignore = ['bar']
nodemanager = NodeManager(config, ["popen//chdir=%s" % dest])
nodemanager.setup_nodes(None) # calls .rsync_roots()
assert dest.join("dir1").check()
assert not dest.join("dir1", "dir2").check()
assert dest.join("dir5", "file").check()
assert not dest.join("dir6").check()
assert not dest.join('foo').check()
assert not dest.join('bar').check()
def get_nodes_specs(nodes,
python=None,
chdir=None,
virtualenv_path=None,
mem_per_process=None,
max_processes=None,
rsync_max_processes=None,
rsync_bandwidth_limit=None,
config=None):
"""Get nodes specs.
Get list of node names, connect to each of them, get the system information, produce the list of node specs out of
that information filtering non-connectable nodes and nodes which don't comply the requirements.
Executed on the master node side.
:param nodes: `list` of node names in form [[<username>@]<hostname>, ...]
:type nodes: list
:param python: python executable name to use on the remote side
:type python: str
:param chdir: relative path where to run (and sync) tests on the remote side
:type chdir: str
:param virtualenv_path: relative path to the virtualenv to activate on the remote test node
:type virtualenv_path: str
:param mem_per_process: optional amount of memory per process needed, in megabytest
:type mem_per_process: int
:param max_processes: optional maximum number of processes per test node
:type max_processes: int
:param rsync_max_processes: optional maximum number of rsync processes
:type rsync_max_processes: int
:param rsync_bandwidth_limit: optional bandwidth limit per rsync process in kilobytes per second
:type rsync_bandwidth_limit: int
:param config: pytest config object
:type config: pytest.Config
:return: `list` of test gateway specs for all test nodes which confirm given requirements
in form ['1*ssh=<node>//id=<hostname>:<index>', ...]
:rtype: list
"""
group = execnet.Group()
try:
if virtualenv_path:
nm = NodeManager(config, specs=[])
virtualenv_path = os.path.relpath(virtualenv_path)
node_specs = []
node_caps = {}
root_dir = getrootdir(config, '')
nodes = list(unique_everseen(nodes))
print('Detected root dir: {0}'.format(root_dir))
rsync = RSync(root_dir,
chdir,
includes=config.getini("rsyncdirs"),
jobs=rsync_max_processes or len(nodes),
bwlimit=rsync_bandwidth_limit,
bandwidth_limit=rsync_bandwidth_limit,
**nm.rsyncoptions)
print('Detecting connectable test nodes...')
for node in nodes:
host = node.split('@')[1] if '@' in node else node
spec = 'ssh={node}//id={host}//chdir={chdir}//python={python}'.format(
node=node, host=host, chdir=chdir, python=python)
try:
make_gateway(group, spec)
except Exception: # pylint: disable=W0703
continue
rsync.add_target_host(node)
node_specs.append((node, host))
if node_specs:
print('Found {0} connectable test nodes: {1}'.format(
len(node_specs), rsync.targets))
else:
pytest.exit('None of the given test nodes are connectable')
print('RSyncing directory structure')
rsync.send()
print('RSync finished')
develop_eggs = get_develop_eggs(root_dir, config)
group.remote_exec(patches.activate_env,
virtualenv_path=virtualenv_path,
develop_eggs=develop_eggs).waitclose()
multi_channel = group.remote_exec(get_node_capabilities)
try:
caps = multi_channel.receive_each(True)
for ch, cap in caps:
node_caps[ch.gateway.id] = cap
finally:
multi_channel.waitclose()
return list(
chain.from_iterable(
get_node_specs(
node,
hst,
node_caps[hst],
python=os.path.join(chdir, virtualenv_path, 'bin', python),
chdir=chdir,
mem_per_process=mem_per_process,
max_processes=max_processes) for node, hst in node_specs))
finally:
try:
group.terminate()
except Exception: # pylint: disable=W0703
pass
def test_default_chdir(self, config):
l = ["ssh=noco", "socket=xyz"]
for spec in NodeManager(config, l).specs:
assert spec.chdir == "pyexecnetcache"
for spec in NodeManager(config, l, defaultchdir="abc").specs:
assert spec.chdir == "abc"
def test_popen_no_default_chdir(self, config):
gm = NodeManager(config, ["popen"])
assert gm.specs[0].chdir is None
def pytest_sessionstart(self, session):
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
def test_getrsyncignore(self, testdir):
config = testdir.parseconfigure('--rsyncignore=fo*')
nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")])
assert 'fo*' in nm.rsyncoptions['ignores']
def test_xspecs_multiplied(self, testdir):
config = testdir.parseconfigure("--tx=3*popen", )
xspecs = NodeManager(config)._getxspecs()
assert len(xspecs) == 3
assert xspecs[1].popen
