def test_run_in_subprocess_classmethod(): class C(object): @classmethod def fn(cls, *args, **kwargs): return cls, args, kwargs C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) c = C() with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(c.fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps((C, (ARG, ), { 'kw': KW }), protocol=0))
def test_run_in_subprocess_nested_function(): def fn(*args, **kwargs): return args, kwargs source = """def fn(*args, **kwargs): return args, kwargs """ with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, ( source, ARG, ), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def test_run_in_subprocess_staticmethod_on_unpickleable_class(): class C(object): @staticmethod def fn(*args, **kwargs): return args, kwargs source = """@staticmethod def fn(*args, **kwargs): return args, kwargs """ C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, ( source, ARG, ), { 'kw': KW }) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def get_data(self): """ pops the latest off the queue, or None is there is none """ try: data = self.queue.pop() except IndexError: return None, None try: data = cPickle.loads(data) except: try: data = data.decode('utf-8') except: pass if DEBUG: logger.info('got %s' % str(data)) t = None if isinstance(data, TimedMsg): d = data.value t = data.time elif isinstance(data, string_types): try: d = json.loads(data) except: d = data else: d = data return d, t
def test_run_in_subprocess_str(): source = """def fn(*args, **kwargs): return args, kwargs """ with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) cmdline.run_in_subprocess(source, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._evaluate_fn_source, (source, ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_pickleable_function(): def fn(*args, **kwargs): return args, kwargs fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_') fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, fn.__name__, fn, create=True): cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (fn, (ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def inner(*args, **kwargs): # execnet sends stdout to /dev/null :( fix_stdout = sys.version_info < (3, 0, 0) # Python 3 passes close_fds=True to subprocess.Popen with ExitStack() as stack: with ExitStack() as stack2: if fix_stdout: fd = os.dup(1) stack2.callback(os.close, fd) gw = execnet.makegateway(spec) # @UndefinedVariable stack.callback(gw.exit) if fix_stdout: with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: chan.send(cPickle.dumps((_run_in_subprocess_redirect_stdout, (fd,), {}), protocol=0)) chan.receive(None) with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: payload = (pkl_fn, tuple(i for t in (preargs, args) for i in t), kwargs) chan.send(cPickle.dumps(payload, protocol=0)) return cPickle.loads(chan.receive(timeout))
def test_run_in_subprocess_staticmethod(): class C(object): @staticmethod def fn(*args, **kwargs): return args, kwargs C.__name__ = 'C_' + str(uuid4()).replace('-', '_') C.__module__ = C.fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, C.__name__, C, create=True): cmdline.run_in_subprocess(C.fn, python='sentinel.python')(ARG, kw=KW) ((s,), _) = chan.send.call_args assert cPickle.loads(s) == (cmdline._invoke_method, (C, 'fn', ARG,), {'kw': KW}) ((remote_fn,), _) = gw.remote_exec.call_args ((chan.receive.return_value,), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with(cPickle.dumps(((ARG,), {'kw': KW}), protocol=0))
def test_run_in_subprocess_pickleable_function(): def fn(*args, **kwargs): return args, kwargs fn.__name__ = 'fn_' + str(uuid4()).replace('-', '_') fn.__module__ = 'pkglib_testing.cmdline' with patch('pkglib_testing.cmdline.execnet') as execnet: gw = execnet.makegateway.return_value chan = gw.remote_exec.return_value chan.receive.return_value = cPickle.dumps(sentinel.ret) with patch.object(cmdline, fn.__name__, fn, create=True): cmdline.run_in_subprocess(fn, python='sentinel.python')(ARG, kw=KW) ((s, ), _) = chan.send.call_args assert cPickle.loads(s) == (fn, (ARG, ), {'kw': KW}) ((remote_fn, ), _) = gw.remote_exec.call_args ((chan.receive.return_value, ), _) = chan.send.call_args remote_fn(chan) chan.send.assert_called_with( cPickle.dumps(((ARG, ), { 'kw': KW }), protocol=0))
def inner(*args, **kwargs): # execnet sends stdout to /dev/null :( fix_stdout = sys.version_info < ( 3, 0, 0) # Python 3 passes close_fds=True to subprocess.Popen with ExitStack() as stack: with ExitStack() as stack2: if fix_stdout: fd = os.dup(1) stack2.callback(os.close, fd) gw = execnet.makegateway(spec) # @UndefinedVariable stack.callback(gw.exit) if fix_stdout: with closing( gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: chan.send( cPickle.dumps( (_run_in_subprocess_redirect_stdout, (fd, ), {}), protocol=0)) chan.receive(None) with closing(gw.remote_exec(_run_in_subprocess_remote_fn)) as chan: payload = (pkl_fn, tuple(i for t in (preargs, args) for i in t), kwargs) chan.send(cPickle.dumps(payload, protocol=0)) return cPickle.loads(chan.receive(timeout))
def _run_in_subprocess_remote_fn(channel): from pkglib_util.six.moves import cPickle # @UnresolvedImport @Reimport # NOQA fn, args, kwargs = cPickle.loads(channel.receive(None)) channel.send(cPickle.dumps(fn(*args, **kwargs), protocol=0))