def testLocalMethodCallChain(self):
exec_str = """
x = 1
class Bar(object):
@classmethod
def func1(self):
return x
def func2(self):
return self.func1() + 1
def func3(self):
try:
return x+2
except:
return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
with main_environ(exec_str):
_func1 = loads(dumps(MAIN.func1))
_func2 = loads(dumps(MAIN.func2))
_func3 = loads(dumps(MAIN.func3))
assert _func1() == MAIN.x
assert _func2() == MAIN.x + 1
assert _func3() == MAIN.x + 2
def testLocalMethodCallChain(self):
exec_str = """
x = 1
class Bar(object):
@classmethod
def func1(self):
return x
def func2(self):
return self.func1() + 1
def func3(self):
try:
return x+2
except:
return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
with main_environ(exec_str):
_func1 = loads(dumps(MAIN.func1))
_func2 = loads(dumps(MAIN.func2))
_func3 = loads(dumps(MAIN.func3))
assert _func1() == MAIN.x
assert _func2() == MAIN.x + 1
assert _func3() == MAIN.x + 2
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.__self__.__class__ is sample.__self__.__class__
assert _sample.__func__ is sample.__func__
assert isinstance(_sample.__self__, Random)
assert isinstance(sample.__self__, Random)
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.im_class is sample.im_class
assert _sample.im_func is sample.im_func
assert isinstance(_sample.im_self, Random)
assert isinstance(sample.im_self, Random)
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.im_class is sample.im_class
assert _sample.im_func is sample.im_func
assert isinstance(_sample.im_self, Random)
assert isinstance(sample.im_self, Random)
def test_long_recursion(self):
d = list(range(10))
rdd = self.sc.makeRDD(d)
for i in range(1000):
rdd = rdd.map(lambda x: x + 1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), [x + 1000 for x in d])
def test_long_recursion(self):
d = range(10)
rdd = self.sc.makeRDD(d)
for i in xrange(1000):
rdd = rdd.map(lambda x: x+1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
def test_long_recursion(self):
d = list(range(10))
rdd = self.sc.makeRDD(d)
for i in range(1000):
rdd = rdd.map(lambda x: x + 1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), [x + 1000 for x in d])
def test_long_recursion(self):
d = range(10)
rdd = self.sc.makeRDD(d)
for i in xrange(1000):
rdd = rdd.map(lambda x: x+1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
def testMemberDescriptor(self):
exec_str = """
class _ClsWithSlots(object):
__slots__=['x', 'y']
"""
with main_environ(exec_str):
f = MAIN._ClsWithSlots()
_f = loads(dumps(f))
assert _f.__slots__ == f.__slots__
def testMemberDescriptor(self):
exec_str = """
class _ClsWithSlots(object):
__slots__=['x', 'y']
"""
with main_environ(exec_str):
f = MAIN._ClsWithSlots()
_f = loads(dumps(f))
assert _f.__slots__ == f.__slots__
def testOldStyleClass(self):
exec_str = """
import csv
class DumDialect(csv.Dialect):
delimiter = '\\t'
quotechar = '"'
escapechar = '\\\\'
doublequote = False
skipinitialspace = False
lineterminator = '\\n'
quoting = csv.QUOTE_MINIMAL
"""
with main_environ(exec_str):
loads(dumps(MAIN.DumDialect))
def testOldStyleClass(self):
exec_str = """
import csv
class DumDialect(csv.Dialect):
delimiter = '\\t'
quotechar = '"'
escapechar = '\\\\'
doublequote = False
skipinitialspace = False
lineterminator = '\\n'
quoting = csv.QUOTE_MINIMAL
"""
with main_environ(exec_str):
loads(dumps(MAIN.DumDialect))
def createTask(self, o, job, t, available_cpus):
task = mesos_pb2.TaskInfo()
tid = "%s:%s:%s" % (job.id, t.id, t.tried)
task.name = "task %s" % tid
task.task_id.value = tid
task.slave_id.value = o.slave_id.value
task.data = compress(dumps((t, t.tried)))
task.executor.MergeFrom(self.executor)
if len(task.data) > 1000 * 1024:
logger.warning("task too large: %s %d", t, len(task.data))
cpu = task.resources.add()
cpu.name = 'cpus'
cpu.type = 0 #mesos_pb2.Value.SCALAR
cpu.scalar.value = min(t.cpus, available_cpus)
mem = task.resources.add()
mem.name = 'mem'
mem.type = 0 #mesos_pb2.Value.SCALAR
mem.scalar.value = t.mem
return task
def testRandomSample(self):
from random import sample
assert dumps(sample)
def __getstate__(self):
d = dict(self.__dict__)
del d['rdd']
del d['split']
return d, dumps(self.rdd), dumps(self.split)
def __getstate__(self):
d = RDD.__getstate__(self)
del d['filters']
return d, dumps(self.filters)
def __getstate__(self):
d = dict(self.__dict__)
del d['rdd']
del d['split']
return d, dumps(self.rdd), dumps(self.split)
def __getstate__(self):
d = dict(self.__dict__)
del d['func']
del d['rdd']
return d, dumps(self.rdd), dump_func(self.func)
def __getstate__(self):
d = RDD.__getstate__(self)
del d['filters']
return d, dumps(self.filters)