def test_run_in_subprocess_classmethod(): class C(object): @classmethod def fn(cls, *args, **kwargs): return cls, args, kwargs C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) c = C() with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(c.fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps((C, (ARG, ), { 'kw': KW }), protocol=0))
def test_run_in_subprocess_nested_function(): def fn(*args, **kwargs): return args, kwargs source = """def fn(*args, **kwargs): return args, kwargs """ with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, ( source, ARG, ), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def test_run_in_subprocess_staticmethod_on_unpickleable_class(): class C(object): @staticmethod def fn(*args, **kwargs): return args, kwargs source = """@staticmethod def fn(*args, **kwargs): return args, kwargs """ C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, ( source, ARG, ), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def _make_pickleable(fn): # return a pickleable function followed by a tuple of initial arguments # could use partial but this is more efficient try: cPickle.dumps(fn, protocol=0) except TypeError: pass else: return fn, () if inspect.ismethod(fn): name, self_ = fn.__name__, fn.__self__ if self_ is None: # Python 2 unbound method self_ = fn.im_class return _invoke_method, (self_, name) elif inspect.isfunction(fn) and fn.__module__ in sys.modules: cls, name = _find_class_from_staticmethod(fn) if (cls, name) != (None, None): try: cPickle.dumps((cls, name), protocol=0) except cPickle.PicklingError: pass else: return _invoke_method, (cls, name) # Fall back to sending the source code return _evaluate_fn_source, (textwrap.dedent(inspect.getsource(fn)),)
def _make_pickleable(fn): # return a pickleable function followed by a tuple of initial arguments # could use partial but this is more efficient try: cPickle.dumps(fn, protocol=0) except TypeError: pass else: return fn, () if inspect.ismethod(fn): name, self_ = fn.__name__, fn.__self__ if self_ is None: # Python 2 unbound method self_ = fn.im_class return _invoke_method, (self_, name) elif inspect.isfunction(fn) and fn.__module__ in sys.modules: cls, name = _find_class_from_staticmethod(fn) if (cls, name) != (None, None): try: cPickle.dumps((cls, name), protocol=0) except cPickle.PicklingError: pass else: return _invoke_method, (cls, name) # Fall back to sending the source code return _evaluate_fn_source, (textwrap.dedent(inspect.getsource(fn)), )
def test_run_in_subprocess_str(): source = """def fn(*args, **kwargs): return args, kwargs """ with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) cmdline.run_in_subprocess(source, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, (source, ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_pickleable_function(): def fn(*args, **kwargs): return args, kwargs fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_') fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, fn.__name__, fn, create=True): cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (fn, (ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def inner(*args, **kwargs): # execnet sends stdout to /dev/null :( fix_stdout = sys.version_info < (3, 0, 0) # Python 3 passes close_fds=True to subprocess.Popen with ExitStack() as stack: with ExitStack() as stack2: if fix_stdout: fd = os.dup(1) stack2.callback(os.close, fd) gw = execnet.makegateway(spec) # @UndefinedVariable stack.callback(gw.exit) if fix_stdout: with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: chan.send(cPickle.dumps((_run_in_subprocess_redirect_stdout, (fd,), {}), protocol=0)) chan.receive(None) with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: payload = (pkl_fn, tuple(i for t in (preargs, args) for i in t), kwargs) chan.send(cPickle.dumps(payload, protocol=0)) return cPickle.loads(chan.receive(timeout))
def test_run_in_subprocess_staticmethod(): class C(object): @staticmethod def fn(*args, **kwargs): return args, kwargs C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.__module__ = C.fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_pickleable_function(): def fn(*args, **kwargs): return args, kwargs fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_') fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, fn.__name__, fn, create=True): cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (fn, (ARG, ), {'kw': KW}) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def inner(*args, **kwargs): # execnet sends stdout to /dev/null :( fix_stdout = sys.version_info < ( 3, 0, 0) # Python 3 passes close_fds=True to subprocess.Popen with ExitStack() as stack: with ExitStack() as stack2: if fix_stdout: fd = os.dup(1) stack2.callback(os.close, fd) gw = execnet.makegateway(spec) # @UndefinedVariable stack.callback(gw.exit) if fix_stdout: with closing( gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: chan.send( cPickle.dumps( (_run_in_subprocess_redirect_stdout, (fd, ), {}), protocol=0)) chan.receive(None) with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: payload = (pkl_fn, tuple(i for t in (preargs, args) for i in t), kwargs) chan.send(cPickle.dumps(payload, protocol=0)) return cPickle.loads(chan.receive(timeout))
def pickled(self): return cPickle.dumps(self)
def _run_in_subprocess_remote_fn(channel): from pkglib_util.six.moves import cPickle # @UnresolvedImport @Reimport # NOQA fn, args, kwargs = cPickle.loads(channel.receive(None)) channel.send(cPickle.dumps(fn(*args, **kwargs), protocol=0))