def test_run_in_subprocess_exception():
def fn(v):
raise v
v = ValueError(uuid4())
with pytest.raises(execnet.RemoteError) as exc: # @UndefinedVariable
cmdline.run_in_subprocess(fn)(v)
assert str(v) in str(exc.value)
def test_run_in_subprocess_timeout():
with patch.multiple('pkglib_testing.cmdline', cPickle=DEFAULT, execnet=DEFAULT) as mocks:
cmdline.run_in_subprocess(Mock(__name__='fn'), python='sentinel.python',
timeout=sentinel.timeout)(sentinel.arg, kw=sentinel.kw)
gw = mocks['execnet'].makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.assert_called_with(sentinel.timeout)
def test_run_in_subprocess_unbound_method_on_unpickleable_class():
class C(object):
def fn(self, *args, **kwargs):
return self, args, kwargs
with patch('pkglib_testing.cmdline.execnet'):
with pytest.raises(cPickle.PicklingError):
cmdline.run_in_subprocess(C.fn, python='sentinel.python')(C(), ARG, kw=KW)
def test_run_in_subprocess_nested_function():
def fn(*args, **kwargs):
return args, kwargs
source = """def fn(*args, **kwargs):
return args, kwargs
"""
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW)
((s, ), _) = chan.send.call_args
assert cPickle.loads(s) == (cmdline._evaluate_fn_source, (
source,
ARG,
), {
'kw': KW
})
((remote_fn, ), _) = gw.remote_exec.call_args
((chan.receive.return_value, ), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(
cPickle.dumps(((ARG, ), {
'kw': KW
}), protocol=0))
def test_run_in_subprocess_staticmethod_on_unpickleable_class():
class C(object):
@staticmethod
def fn(*args, **kwargs):
return args, kwargs
source = """@staticmethod
def fn(*args, **kwargs):
return args, kwargs
"""
C.__name__ = 'C_' + str(uuid4()).replace('-', '_')
C.fn.__module__ = 'pkglib_testing.cmdline'
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
with patch.object(cmdline, C.__name__, C, create=True):
cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG,
kw=KW)
((s, ), _) = chan.send.call_args
assert cPickle.loads(s) == (cmdline._evaluate_fn_source, (
source,
ARG,
), {
'kw': KW
})
((remote_fn, ), _) = gw.remote_exec.call_args
((chan.receive.return_value, ), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(
cPickle.dumps(((ARG, ), {
'kw': KW
}), protocol=0))
def test_run_in_subprocess_classmethod():
class C(object):
@classmethod
def fn(cls, *args, **kwargs):
return cls, args, kwargs
C.__name__ = 'C_' + str(uuid4()).replace('-', '_')
C.__module__ = 'pkglib_testing.cmdline'
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
c = C()
with patch.object(cmdline, C.__name__, C, create=True):
cmdline.run_in_subprocess(c.fn, python='sentinel.python')(ARG,
kw=KW)
((s, ), _) = chan.send.call_args
assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG),
{
'kw': KW
})
((remote_fn, ), _) = gw.remote_exec.call_args
((chan.receive.return_value, ), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(
cPickle.dumps((C, (ARG, ), {
'kw': KW
}), protocol=0))
def test_run_in_subprocess_exception():
def fn(v):
raise v
v = ValueError(uuid4())
with pytest.raises(execnet.RemoteError) as exc: # @UndefinedVariable
cmdline.run_in_subprocess(fn)(v)
assert str(v) in str(exc.value)
def test_run_in_subprocess_bound_method_on_unpickleable_class():
class C(object):
def fn(self, *args, **kwargs):
return self, args, kwargs
with patch('pkglib_testing.cmdline.execnet'):
with pytest.raises(cPickle.PicklingError):
cmdline.run_in_subprocess(C().fn, python='sentinel.python')(ARG,
kw=KW)
def test_run_in_subprocess_cd():
with patch.multiple('pkglib_testing.cmdline',
cPickle=DEFAULT,
execnet=DEFAULT) as mocks:
cmdline.run_in_subprocess(Mock(__name__='fn'),
python='sentinel.python',
cd='sentinel.cd')(sentinel.arg,
kw=sentinel.kw)
mocks['execnet'].makegateway.assert_called_once_with(
'popen//python=sentinel.python//chdir=sentinel.cd')
def test_run_in_subprocess_timeout():
with patch.multiple('pkglib_testing.cmdline',
cPickle=DEFAULT,
execnet=DEFAULT) as mocks:
cmdline.run_in_subprocess(Mock(__name__='fn'),
python='sentinel.python',
timeout=sentinel.timeout)(sentinel.arg,
kw=sentinel.kw)
gw = mocks['execnet'].makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.assert_called_with(sentinel.timeout)
def test_run_in_subprocess_str():
source = """def fn(*args, **kwargs):
return args, kwargs
"""
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
cmdline.run_in_subprocess(source, python='sentinel.python')(ARG, kw=KW)
((s,), _) = chan.send.call_args
assert cPickle.loads(s) == (cmdline._evaluate_fn_source, (source, ARG,), {'kw': KW})
((remote_fn,), _) = gw.remote_exec.call_args
((chan.receive.return_value,), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def bootstrap_tagup(self, python):
work_dir = os.path.join(self.workspace, 'pkglib-deps')
if not os.path.exists(work_dir):
os.makedirs(work_dir)
with open(os.path.join(work_dir, '.pypirc'), 'wt') as rc_file:
self.get_rc().write(rc_file)
# XXX find a better way to pass in credentials
new_env = copy.copy(dict(os.environ))
new_env['HOME'] = work_dir
if "PYTHONPATH" in new_env:
del new_env["PYTHONPATH"]
def get_pkglib_reqs():
from pkglib.setuptools.dependency import get_all_requirements
return [(dist.project_name, dist.version)
for dist in get_all_requirements(['pkglib', 'pytest', 'pytest-cov'], ignore_explicit_builtins=True)
if dist.project_name not in ['virtualenv', 'setuptools']]
for name, version in run_in_subprocess(get_pkglib_reqs, python=python, cd=self.workspace)():
# Quick hack to get the built eggs into the test PyPi instance.
# We register with an empty package file then copy the files in manually
# We may need pip and distribute if virtualenv installed old versions.
# (should only occur when upgrading to new virtualenv).
with open(os.path.join(work_dir, 'setup.py'), 'wb') as fp:
setup_py = SETUP_TMPL % {'name': name, 'version': version}
fp.write(setup_py.encode('utf-8'))
cmd = 'cd %s; %s setup.py register' % (work_dir, python)
out = self.run(cmd, capture=True, env=new_env)
logger.debug(out)
assert '200' in out
self.upload_requirement(work_dir, Requirement.parse('%s==%s' % (name, version)), python)
def upload_requirement(self, work_dir, req, python):
dest_dir = self.get_file_dir(req.project_name).strip()
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
def fetch_requirement(req, dest_dir, force_download):
from setuptools.package_index import PackageIndex # @Reimport
from pkg_resources import working_set # @Reimport # NOQA
i = PackageIndex()
if force_download:
[i.remove(i[req.key][0]) for _ in xrange(len(i[req.key]))]
d = i.download(req, dest_dir)
else:
d = i.fetch_distribution(req, dest_dir, force_scan=True)
d = getattr(d, 'location', d) if d else ''
return (d if d else working_set.resolve([req])[0].location)
with set_env(COVERAGE_DISABLE_WARNINGS="1"):
fetched = run_in_subprocess(fetch_requirement, python=python, cd=self.workspace
)(req, dest_dir, force_download=False)
if not fetched or not os.path.exists(fetched):
err_msg = "Unable to find requirement: %r\n%s" % (str(req), fetched)
raise RuntimeError(err_msg)
if os.path.isdir(fetched):
fetched = self.create_egg_for_package(fetched, work_dir, python)
print("Fetched %r" % fetched)
return fetched
def test_run_in_subprocess():
with patch.multiple('pkglib_testing.cmdline', cPickle=DEFAULT, execnet=DEFAULT) as mocks:
fn = Mock(__name__='fn')
res = cmdline.run_in_subprocess(fn, python='sentinel.python')(sentinel.arg, kw=sentinel.kw)
mocks['execnet'].makegateway.assert_called_once_with('popen//python=sentinel.python')
gw = mocks['execnet'].makegateway.return_value
((remote_fn,), _) = gw.remote_exec.call_args
chan = gw.remote_exec.return_value
mocks['cPickle'].dumps.assert_called_with((fn, (sentinel.arg,), {'kw': sentinel.kw}), protocol=0)
chan.send.assert_called_with(mocks['cPickle'].dumps.return_value)
chan.receive.assert_has_calls([call(None) for _i in range(gw.remote_exec.call_count)])
mocks['cPickle'].loads.assert_called_once_with(chan.receive.return_value)
assert res is mocks['cPickle'].loads.return_value
chan.close.assert_has_calls([call() for _i in range(gw.remote_exec.call_count)])
gw.exit.assert_called_once_with()
with patch('pkglib_util.six.moves.cPickle') as cPickle:
channel, fn = Mock(), Mock()
cPickle.loads.return_value = (fn, (sentinel.arg,), {'kw': sentinel.kw})
remote_fn(channel)
channel.receive.assert_called_once_with(None)
cPickle.loads.assert_called_once_with(channel.receive.return_value)
fn.assert_called_once_with(sentinel.arg, kw=sentinel.kw)
cPickle.dumps.assert_called_once_with(fn.return_value, protocol=0)
channel.send.assert_called_once_with(cPickle.dumps.return_value)
def test_run_in_subprocess_uses_passed_python():
def fn():
import sys # @Reimport
return sys.executable
python = cmdline.run_in_subprocess(fn, python=sys.executable)()
assert python == sys.executable
def test_run_in_subprocess_pickleable_function():
def fn(*args, **kwargs):
return args, kwargs
fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_')
fn.__module__ = 'pkglib_testing.cmdline'
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
with patch.object(cmdline, fn.__name__, fn, create=True):
cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW)
((s,), _) = chan.send.call_args
assert cPickle.loads(s) == (fn, (ARG,), {'kw': KW})
((remote_fn,), _) = gw.remote_exec.call_args
((chan.receive.return_value,), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_staticmethod():
class C(object):
@staticmethod
def fn(*args, **kwargs):
return args, kwargs
C.__name__ = 'C_' + str(uuid4()).replace('-', '_')
C.__module__ = C.fn.__module__ = 'pkglib_testing.cmdline'
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
with patch.object(cmdline, C.__name__, C, create=True):
cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG, kw=KW)
((s,), _) = chan.send.call_args
assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG,), {'kw': KW})
((remote_fn,), _) = gw.remote_exec.call_args
((chan.receive.return_value,), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_pickleable_function():
def fn(*args, **kwargs):
return args, kwargs
fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_')
fn.__module__ = 'pkglib_testing.cmdline'
with patch('pkglib_testing.cmdline.execnet') as execnet:
gw = execnet.makegateway.return_value
chan = gw.remote_exec.return_value
chan.receive.return_value = cPickle.dumps(sentinel.ret)
with patch.object(cmdline, fn.__name__, fn, create=True):
cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW)
((s, ), _) = chan.send.call_args
assert cPickle.loads(s) == (fn, (ARG, ), {'kw': KW})
((remote_fn, ), _) = gw.remote_exec.call_args
((chan.receive.return_value, ), _) = chan.send.call_args
remote_fn(chan)
chan.send.assert_called_with(
cPickle.dumps(((ARG, ), {
'kw': KW
}), protocol=0))
def test_run_in_subprocess():
with patch.multiple('pkglib_testing.cmdline',
cPickle=DEFAULT,
execnet=DEFAULT) as mocks:
fn = Mock(__name__='fn')
res = cmdline.run_in_subprocess(fn, python='sentinel.python')(
sentinel.arg, kw=sentinel.kw)
mocks['execnet'].makegateway.assert_called_once_with(
'popen//python=sentinel.python')
gw = mocks['execnet'].makegateway.return_value
((remote_fn, ), _) = gw.remote_exec.call_args
chan = gw.remote_exec.return_value
mocks['cPickle'].dumps.assert_called_with((fn, (sentinel.arg, ), {
'kw': sentinel.kw
}),
protocol=0)
chan.send.assert_called_with(mocks['cPickle'].dumps.return_value)
chan.receive.assert_has_calls(
[call(None) for _i in range(gw.remote_exec.call_count)])
mocks['cPickle'].loads.assert_called_once_with(
chan.receive.return_value)
assert res is mocks['cPickle'].loads.return_value
chan.close.assert_has_calls(
[call() for _i in range(gw.remote_exec.call_count)])
gw.exit.assert_called_once_with()
with patch('pkglib_util.six.moves.cPickle') as cPickle:
channel, fn = Mock(), Mock()
cPickle.loads.return_value = (fn, (sentinel.arg, ), {
'kw': sentinel.kw
})
remote_fn(channel)
channel.receive.assert_called_once_with(None)
cPickle.loads.assert_called_once_with(channel.receive.return_value)
fn.assert_called_once_with(sentinel.arg, kw=sentinel.kw)
cPickle.dumps.assert_called_once_with(fn.return_value, protocol=0)
channel.send.assert_called_once_with(cPickle.dumps.return_value)
def test_run_in_subprocess():
def fn():
return None
res = cmdline.run_in_subprocess(fn)()
assert res is None
def test_run_in_subprocess_uses_passed_python():
def fn():
import sys # @Reimport
return sys.executable
python = cmdline.run_in_subprocess(fn, python=sys.executable)()
assert python == sys.executable
def test_run_in_subprocess_cd():
with workspace.Workspace() as ws:
cwd = cmdline.run_in_subprocess(os.getcwd, cd=ws.workspace)()
assert cwd == ws.workspace
def test_run_in_subprocess_timeout():
with pytest.raises(execnet.TimeoutError) as exc: # @UndefinedVariable
cmdline.run_in_subprocess(time.sleep, timeout=0)(1)
assert 'no item after 0 seconds' in str(exc.value)
def test_run_in_subprocess():
def fn():
return None
res = cmdline.run_in_subprocess(fn)()
assert res is None
def test_run_in_subprocess_is_a_subprocess():
pid = cmdline.run_in_subprocess(os.getpid)()
assert pid != os.getpid()
def test_run_in_subprocess_cd():
with workspace.Workspace() as ws:
cwd = cmdline.run_in_subprocess(os.getcwd, cd=ws.workspace)()
assert cwd == ws.workspace
def test_run_in_subprocess_timeout():
with pytest.raises(execnet.TimeoutError) as exc: # @UndefinedVariable
cmdline.run_in_subprocess(time.sleep, timeout=0)(1)
assert "no item after 0 seconds" in str(exc.value)
def test_run_in_subprocess_cd():
with patch.multiple('pkglib_testing.cmdline', cPickle=DEFAULT, execnet=DEFAULT) as mocks:
cmdline.run_in_subprocess(Mock(__name__='fn'), python='sentinel.python',
cd='sentinel.cd')(sentinel.arg, kw=sentinel.kw)
mocks['execnet'].makegateway.assert_called_once_with('popen//python=sentinel.python//chdir=sentinel.cd')
def test_run_in_subprocess_is_a_subprocess():
pid = cmdline.run_in_subprocess(os.getpid)()
assert pid != os.getpid()
