def test_pickling_special_file_handles(self):
# Warning: if you want to run your tests with nose, add -s option
for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415
print(out, type(out))
self.assertEquals(out, pickle.loads(cloudpickle.dumps(out)))
self.assertRaises(pickle.PicklingError,
lambda: cloudpickle.dumps(sys.stdin))
def test_closed_file(self):
# Write & close
with open(self.tmpfilepath, 'w') as f:
f.write(self.teststring)
with pytest.raises(pickle.PicklingError) as excinfo:
cloudpickle.dumps(f)
assert "Cannot pickle closed files" in str(excinfo.value)
os.remove(self.tmpfilepath)
def NOT_WORKING_test_tty(self):
# FIXME: Mocking 'file' is not trivial... and fails for now
from sys import version_info
if version_info.major == 2:
import __builtin__ as builtins # pylint:disable=import-error
else:
import builtins # pylint:disable=import-error
with patch.object(builtins, 'open', mock_open(), create=True):
with open('foo', 'w+') as handle:
cloudpickle.dumps(handle)
def test_w_mode(self):
with open(self.tmpfilepath, 'w') as f:
f.write(self.teststring)
f.seek(0)
self.assertRaises(pickle.PicklingError,
lambda: cloudpickle.dumps(f))
os.remove(self.tmpfilepath)
def test_extended_arg(self):
# Functions with more than 65535 global vars prefix some global
# variable references with the EXTENDED_ARG opcode.
nvars = 65537 + 258
names = ['g%d' % i for i in range(1, nvars)]
r = random.Random(42)
d = dict([(name, r.randrange(100)) for name in names])
# def f(x):
# x = g1, g2, ...
# return zlib.crc32(bytes(bytearray(x)))
code = """
import zlib
def f():
x = {tup}
return zlib.crc32(bytes(bytearray(x)))
""".format(tup=', '.join(names))
exec(textwrap.dedent(code), d, d)
f = d['f']
res = f()
data = cloudpickle.dumps([f, f])
d = f = None
f2, f3 = pickle.loads(data)
self.assertTrue(f2 is f3)
self.assertEqual(f2(), res)
def test_plus_mode(self):
# Write, then seek to 0
with open(self.tmpfilepath, 'w+') as f:
f.write(self.teststring)
f.seek(0)
new_f = pickle.loads(cloudpickle.dumps(f))
self.assertEquals(self.teststring, new_f.read())
os.remove(self.tmpfilepath)
def test_temp_file(self):
teststring = self.teststring.encode('UTF-8')
with tempfile.NamedTemporaryFile(mode='ab+') as fp:
fp.write(teststring)
fp.seek(0)
f = fp.file
# FIXME this doesn't work yet: cloudpickle.dumps(fp)
newfile = pickle.loads(cloudpickle.dumps(f))
self.assertEquals(teststring, newfile.read())
def test_r_mode(self):
# Write & close
with open(self.tmpfilepath, 'w') as f:
f.write(self.teststring)
# Open for reading
with open(self.tmpfilepath, 'r') as f:
new_f = pickle.loads(cloudpickle.dumps(f))
self.assertEquals(self.teststring, new_f.read())
os.remove(self.tmpfilepath)
def test_func_globals(self):
class Unpicklable(object):
def __reduce__(self):
raise Exception("not picklable")
global exit
exit = Unpicklable()
self.assertRaises(Exception, lambda: cloudpickle.dumps(exit))
def foo():
sys.exit(0)
func_code = getattr(foo, '__code__', None)
if func_code is None: # PY2 backwards compatibility
func_code = foo.func_code
self.assertTrue("exit" in func_code.co_names)
cloudpickle.dumps(foo)
def test_seek(self):
# Write, then seek to arbitrary position
with open(self.tmpfilepath, 'w+') as f:
f.write(self.teststring)
f.seek(4)
unpickled = pickle.loads(cloudpickle.dumps(f))
# unpickled StringIO is at position 4
self.assertEquals(4, unpickled.tell())
self.assertEquals(self.teststring[4:], unpickled.read())
# but unpickled StringIO also contained the start
unpickled.seek(0)
self.assertEquals(self.teststring, unpickled.read())
os.remove(self.tmpfilepath)
def test_tornado_coroutine(self):
# Pickling a locally defined coroutine function
from tornado import gen, ioloop
@gen.coroutine
def f(x, y):
yield gen.sleep(x)
raise gen.Return(y + 1)
@gen.coroutine
def g(y):
res = yield f(0.01, y)
raise gen.Return(res + 1)
data = cloudpickle.dumps([g, g])
f = g = None
g2, g3 = pickle.loads(data)
self.assertTrue(g2 is g3)
loop = ioloop.IOLoop.current()
res = loop.run_sync(functools.partial(g2, 5))
self.assertEqual(res, 7)
def test_empty_file(self):
# Empty file
open(self.tmpfilepath, 'w').close()
with open(self.tmpfilepath, 'r') as f:
self.assertEquals('', pickle.loads(cloudpickle.dumps(f)).read())
os.remove(self.tmpfilepath)
def test_pickling_file_handles(self):
out1 = sys.stderr
out2 = pickle.loads(cloudpickle.dumps(out1))
self.assertEquals(out1, out2)
def pickle_depickle(obj):
"""Helper function to test whether object pickled with cloudpickle can be
depickled with pickle
"""
return pickle.loads(cloudpickle.dumps(obj))
def test_loads_namespace(self):
obj = 1, 2, 3, 4
returned_obj = cloudpickle.loads(cloudpickle.dumps(obj))
self.assertEqual(obj, returned_obj)