def test_getrsyncdirs(self, testdir): config = testdir.parseconfigure('--rsyncdir=' + str(testdir.tmpdir)) nm = NodeManager(config, specs=[execnet.XSpec("popen")]) assert not nm._getrsyncdirs() nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) assert nm.roots assert testdir.tmpdir in nm.roots
def test_popens_rsync(self, config, mysetup, slavecontroller): source = mysetup.source hm = NodeManager(config, ["popen"] * 2) hm.setup_nodes(None) assert len(hm.group) == 2 for gw in hm.group: class pseudoexec: args = [] def __init__(self, *args): self.args.extend(args) def waitclose(self): pass gw.remote_exec = pseudoexec notifications = [] for gw in hm.group: hm.rsync(gw, source, notify=lambda *args: notifications.append(args)) assert not notifications hm.teardown_nodes() assert not len(hm.group) assert "sys.path.insert" in gw.remote_exec.args[0]
def test_getxspecs(self, testdir): config = testdir.parseconfigure("--tx=popen", "--tx", "ssh=xyz") nodemanager = NodeManager(config) xspecs = nodemanager._getxspecs() assert len(xspecs) == 2 print(xspecs) assert xspecs[0].popen assert xspecs[1].ssh == "xyz"
def pytest_sessionstart(self, session): """Creates and starts the nodes. The nodes are setup to put their events onto self.queue. As soon as nodes start they will emit the slave_slaveready event. """ self.nodemanager = NodeManager(self.config) nodes = self.nodemanager.setup_nodes(putevent=self.queue.put) self._active_nodes.update(nodes) self._session = session
def test_optimise_popen(self, testdir, mysetup, slavecontroller): source = mysetup.source specs = ["popen"] * 3 source.join("conftest.py").write("rsyncdirs = ['a']") source.ensure('a', dir=1) config = testdir.parseconfig(source) nodemanager = NodeManager(config, specs) nodemanager.setup_nodes(None) # calls .rysnc_roots() for gwspec in nodemanager.specs: assert gwspec._samefilesystem() assert not gwspec.chdir
def test_rsync_roots_no_roots(self, testdir, mysetup): mysetup.source.ensure("dir1", "file1").write("hello") config = testdir.parseconfig(mysetup.source) nodemanager = NodeManager(config, ["popen//chdir=%s" % mysetup.dest]) # assert nodemanager.config.topdir == source == config.topdir nodemanager.makegateways() nodemanager.rsync_roots() p, = nodemanager.gwmanager.multi_exec( "import os ; channel.send(os.getcwd())").receive_each() p = py.path.local(p) py.builtin.print_("remote curdir", p) assert p == mysetup.dest.join(config.topdir.basename) assert p.join("dir1").check() assert p.join("dir1", "file1").check()
def test_rsync_same_popen_twice(self, config, mysetup, hookrecorder, slavecontroller): source, dest = mysetup.source, mysetup.dest hm = NodeManager(config, ["popen//chdir=%s" % dest] * 2) hm.roots = [] hm.setup_nodes(None) source.ensure("dir1", "dir2", "hello") gw = hm.group[0] hm.rsync(gw, source) call = hookrecorder.popcall("pytest_xdist_rsyncstart") assert call.source == source assert len(call.gateways) == 1 assert call.gateways[0] in hm.group call = hookrecorder.popcall("pytest_xdist_rsyncfinish")
def test_rsync_popen_with_path(self, config, mysetup, slavecontroller): source, dest = mysetup.source, mysetup.dest hm = NodeManager(config, ["popen//chdir=%s" % dest] * 1) hm.setup_nodes(None) source.ensure("dir1", "dir2", "hello") l = [] for gw in hm.group: hm.rsync(gw, source, notify=lambda *args: l.append(args)) assert len(l) == 1 assert l[0] == ("rsyncrootready", hm.group['gw0'].spec, source) hm.teardown_nodes() dest = dest.join(source.basename) assert dest.join("dir1").check() assert dest.join("dir1", "dir2").check() assert dest.join("dir1", "dir2", 'hello').check()
def test_popen_makegateway_events(self, config, hookrecorder, slavecontroller): hm = NodeManager(config, ["popen"] * 2) hm.setup_nodes(None) call = hookrecorder.popcall("pytest_xdist_setupnodes") assert len(call.specs) == 2 call = hookrecorder.popcall("pytest_xdist_newgateway") assert call.gateway.spec == execnet.XSpec("popen") assert call.gateway.id == "gw0" call = hookrecorder.popcall("pytest_xdist_newgateway") assert call.gateway.id == "gw1" assert len(hm.group) == 2 hm.teardown_nodes() assert not len(hm.group)
def test_init_rsync_roots(self, testdir, mysetup, slavecontroller): source, dest = mysetup.source, mysetup.dest dir2 = source.ensure("dir1", "dir2", dir=1) source.ensure("dir1", "somefile", dir=1) dir2.ensure("hello") source.ensure("bogusdir", "file") source.join("tox.ini").write(py.std.textwrap.dedent(""" [pytest] rsyncdirs=dir1/dir2 """)) config = testdir.parseconfig(source) nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager.setup_nodes(None) # calls .rsync_roots() assert dest.join("dir2").check() assert not dest.join("dir1").check() assert not dest.join("bogus").check()
def test_getrsyncdirs_with_conftest(self, testdir): p = py.path.local() for bn in 'x y z'.split(): p.mkdir(bn) testdir.makeini(""" [pytest] rsyncdirs= x """) config = testdir.parseconfigure(testdir.tmpdir, '--rsyncdir=y', '--rsyncdir=z') nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=xyz")]) roots = nm._getrsyncdirs() # assert len(roots) == 3 + 1 # pylib assert py.path.local('y') in roots assert py.path.local('z') in roots assert testdir.tmpdir.join('x') in roots
def test_popen_rsync_subdir(self, testdir, mysetup, slavecontroller): source, dest = mysetup.source, mysetup.dest dir1 = mysetup.source.mkdir("dir1") dir2 = dir1.mkdir("dir2") dir2.ensure("hello") for rsyncroot in (dir1, source): dest.remove() nodemanager = NodeManager(testdir.parseconfig( "--tx", "popen//chdir=%s" % dest, "--rsyncdir", rsyncroot, source, )) nodemanager.setup_nodes(None) # calls .rsync_roots() if rsyncroot == source: dest = dest.join("source") assert dest.join("dir1").check() assert dest.join("dir1", "dir2").check() assert dest.join("dir1", "dir2", 'hello').check() nodemanager.teardown_nodes()
def test_rsyncignore(self, testdir, mysetup, slavecontroller): source, dest = mysetup.source, mysetup.dest dir2 = source.ensure("dir1", "dir2", dir=1) source.ensure("dir5", "dir6", "bogus") source.ensure("dir5", "file") dir2.ensure("hello") source.ensure("foo", "bar") source.ensure("bar", "foo") source.join("tox.ini").write(py.std.textwrap.dedent(""" [pytest] rsyncdirs = dir1 dir5 rsyncignore = dir1/dir2 dir5/dir6 foo* """)) config = testdir.parseconfig(source) config.option.rsyncignore = ['bar'] nodemanager = NodeManager(config, ["popen//chdir=%s" % dest]) nodemanager.setup_nodes(None) # calls .rsync_roots() assert dest.join("dir1").check() assert not dest.join("dir1", "dir2").check() assert dest.join("dir5", "file").check() assert not dest.join("dir6").check() assert not dest.join('foo').check() assert not dest.join('bar').check()
def get_nodes_specs(nodes, python=None, chdir=None, virtualenv_path=None, mem_per_process=None, max_processes=None, rsync_max_processes=None, rsync_bandwidth_limit=None, config=None): """Get nodes specs. Get list of node names, connect to each of them, get the system information, produce the list of node specs out of that information filtering non-connectable nodes and nodes which don't comply the requirements. Executed on the master node side. :param nodes: `list` of node names in form [[<username>@]<hostname>, ...] :type nodes: list :param python: python executable name to use on the remote side :type python: str :param chdir: relative path where to run (and sync) tests on the remote side :type chdir: str :param virtualenv_path: relative path to the virtualenv to activate on the remote test node :type virtualenv_path: str :param mem_per_process: optional amount of memory per process needed, in megabytest :type mem_per_process: int :param max_processes: optional maximum number of processes per test node :type max_processes: int :param rsync_max_processes: optional maximum number of rsync processes :type rsync_max_processes: int :param rsync_bandwidth_limit: optional bandwidth limit per rsync process in kilobytes per second :type rsync_bandwidth_limit: int :param config: pytest config object :type config: pytest.Config :return: `list` of test gateway specs for all test nodes which confirm given requirements in form ['1*ssh=<node>//id=<hostname>:<index>', ...] :rtype: list """ group = execnet.Group() try: if virtualenv_path: nm = NodeManager(config, specs=[]) virtualenv_path = os.path.relpath(virtualenv_path) node_specs = [] node_caps = {} root_dir = getrootdir(config, '') nodes = list(unique_everseen(nodes)) print('Detected root dir: {0}'.format(root_dir)) rsync = RSync(root_dir, chdir, includes=config.getini("rsyncdirs"), jobs=rsync_max_processes or len(nodes), bwlimit=rsync_bandwidth_limit, bandwidth_limit=rsync_bandwidth_limit, **nm.rsyncoptions) print('Detecting connectable test nodes...') for node in nodes: host = node.split('@')[1] if '@' in node else node spec = 'ssh={node}//id={host}//chdir={chdir}//python={python}'.format( node=node, host=host, chdir=chdir, python=python) try: make_gateway(group, spec) except Exception: # pylint: disable=W0703 continue rsync.add_target_host(node) node_specs.append((node, host)) if node_specs: print('Found {0} connectable test nodes: {1}'.format( len(node_specs), rsync.targets)) else: pytest.exit('None of the given test nodes are connectable') print('RSyncing directory structure') rsync.send() print('RSync finished') develop_eggs = get_develop_eggs(root_dir, config) group.remote_exec(patches.activate_env, virtualenv_path=virtualenv_path, develop_eggs=develop_eggs).waitclose() multi_channel = group.remote_exec(get_node_capabilities) try: caps = multi_channel.receive_each(True) for ch, cap in caps: node_caps[ch.gateway.id] = cap finally: multi_channel.waitclose() return list( chain.from_iterable( get_node_specs( node, hst, node_caps[hst], python=os.path.join(chdir, virtualenv_path, 'bin', python), chdir=chdir, mem_per_process=mem_per_process, max_processes=max_processes) for node, hst in node_specs)) finally: try: group.terminate() except Exception: # pylint: disable=W0703 pass
def test_default_chdir(self, config): l = ["ssh=noco", "socket=xyz"] for spec in NodeManager(config, l).specs: assert spec.chdir == "pyexecnetcache" for spec in NodeManager(config, l, defaultchdir="abc").specs: assert spec.chdir == "abc"
def test_popen_no_default_chdir(self, config): gm = NodeManager(config, ["popen"]) assert gm.specs[0].chdir is None
def pytest_sessionstart(self, session): self.nodemanager = NodeManager(self.config) self.nodemanager.setup_nodes(putevent=self.queue.put)
def test_getrsyncignore(self, testdir): config = testdir.parseconfigure('--rsyncignore=fo*') nm = NodeManager(config, specs=[execnet.XSpec("popen//chdir=qwe")]) assert 'fo*' in nm.rsyncoptions['ignores']
def test_xspecs_multiplied(self, testdir): config = testdir.parseconfigure("--tx=3*popen", ) xspecs = NodeManager(config)._getxspecs() assert len(xspecs) == 3 assert xspecs[1].popen