def testLocalMethodCallChain(self):
exec_str = """
x = 1
class Bar(object):
@classmethod
def func1(self):
return x
def func2(self):
return self.func1() + 1
def func3(self):
try:
return x+2
except:
return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
with main_environ(exec_str):
_func1 = loads(dumps(MAIN.func1))
_func2 = loads(dumps(MAIN.func2))
_func3 = loads(dumps(MAIN.func3))
assert _func1() == MAIN.x
assert _func2() == MAIN.x + 1
assert _func3() == MAIN.x + 2
def testLocalMethodCallChain(self):
exec_str = """
x = 1
class Bar(object):
@classmethod
def func1(self):
return x
def func2(self):
return self.func1() + 1
def func3(self):
try:
return x+2
except:
return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
with main_environ(exec_str):
_func1 = loads(dumps(MAIN.func1))
_func2 = loads(dumps(MAIN.func2))
_func3 = loads(dumps(MAIN.func3))
assert _func1() == MAIN.x
assert _func2() == MAIN.x + 1
assert _func3() == MAIN.x + 2
def test_long_recursion(self):
d = range(10)
rdd = self.sc.makeRDD(d)
for i in xrange(1000):
rdd = rdd.map(lambda x: x+1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
def test_long_recursion(self):
d = list(range(10))
rdd = self.sc.makeRDD(d)
for i in range(1000):
rdd = rdd.map(lambda x: x + 1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), [x + 1000 for x in d])
def test_long_recursion(self):
d = list(range(10))
rdd = self.sc.makeRDD(d)
for i in range(1000):
rdd = rdd.map(lambda x: x + 1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), [x + 1000 for x in d])
def test_long_recursion(self):
d = range(10)
rdd = self.sc.makeRDD(d)
for i in xrange(1000):
rdd = rdd.map(lambda x: x+1)
loads(dumps(rdd))
self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
def testOldStyleClass(self):
exec_str = """
import csv
class DumDialect(csv.Dialect):
delimiter = '\\t'
quotechar = '"'
escapechar = '\\\\'
doublequote = False
skipinitialspace = False
lineterminator = '\\n'
quoting = csv.QUOTE_MINIMAL
"""
with main_environ(exec_str):
loads(dumps(MAIN.DumDialect))
def testOldStyleClass(self):
exec_str = """
import csv
class DumDialect(csv.Dialect):
delimiter = '\\t'
quotechar = '"'
escapechar = '\\\\'
doublequote = False
skipinitialspace = False
lineterminator = '\\n'
quoting = csv.QUOTE_MINIMAL
"""
with main_environ(exec_str):
loads(dumps(MAIN.DumDialect))
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.__self__.__class__ is sample.__self__.__class__
assert _sample.__func__ is sample.__func__
assert isinstance(_sample.__self__, Random)
assert isinstance(sample.__self__, Random)
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.im_class is sample.im_class
assert _sample.im_func is sample.im_func
assert isinstance(_sample.im_self, Random)
assert isinstance(sample.im_self, Random)
def testRandomSample(self):
from random import sample, Random
_sample = loads(dumps(sample))
assert _sample.im_class is sample.im_class
assert _sample.im_func is sample.im_func
assert isinstance(_sample.im_self, Random)
assert isinstance(sample.im_self, Random)
def testMemberDescriptor(self):
exec_str = """
class _ClsWithSlots(object):
__slots__=['x', 'y']
"""
with main_environ(exec_str):
f = MAIN._ClsWithSlots()
_f = loads(dumps(f))
assert _f.__slots__ == f.__slots__
def testMemberDescriptor(self):
exec_str = """
class _ClsWithSlots(object):
__slots__=['x', 'y']
"""
with main_environ(exec_str):
f = MAIN._ClsWithSlots()
_f = loads(dumps(f))
assert _f.__slots__ == f.__slots__
def run_task(task_data):
try:
gc.disable()
task, task_try_id = loads(decompress(task_data))
ttid = TTID(task_try_id)
Accumulator.clear()
result = task.run(ttid.ttid)
env.task_stats.bytes_max_rss = resource.getrusage(
resource.RUSAGE_SELF).ru_maxrss * 1024
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception:
flag, data = 1, cPickle.dumps(result, -1)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
# shuffle_id start from 1
swd = ShuffleWorkDir(0, task.id, ttid.task_try)
tmppath = swd.alloc_tmp(len(data))
with open(tmppath, 'wb') as f:
f.write(data)
f.close()
path = swd.export(tmppath)
data = '/'.join([env.server_uri] + path.split('/')[-3:])
flag += 2
return TaskState.finished, cPickle.dumps(
((flag, data), accUpdate, env.task_stats), -1)
except FetchFailed as e:
return TaskState.failed, TaskEndReason.fetch_failed, str(
e), cPickle.dumps(e)
except Exception as e:
import traceback
msg = traceback.format_exc()
ename = e.__class__.__name__
fatal_exceptions = (DparkUserFatalError, ArithmeticError, ValueError,
LookupError, SyntaxError, TypeError,
AssertionError)
prefix = "FATAL" if isinstance(e, fatal_exceptions) else "FAILED"
return TaskState.failed, '{}_EXCEPTION_{}'.format(
prefix, ename), msg, cPickle.dumps(e)
finally:
gc.collect()
gc.enable()
def run_task(task_data):
try:
gc.disable()
task, task_try_id = loads(decompress(task_data))
ttid = TTID(task_try_id)
Accumulator.clear()
result = task.run(ttid.ttid)
env.task_stats.bytes_max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception:
flag, data = 1, cPickle.dumps(result, -1)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
# shuffle_id start from 1
swd = ShuffleWorkDir(0, task.id, ttid.task_try)
tmppath = swd.alloc_tmp(len(data))
with open(tmppath, 'wb') as f:
f.write(data)
f.close()
path = swd.export(tmppath)
data = '/'.join(
[env.server_uri] + path.split('/')[-3:]
)
flag += 2
return TaskState.finished, cPickle.dumps(((flag, data), accUpdate, env.task_stats), -1)
except FetchFailed as e:
return TaskState.failed, TaskEndReason.fetch_failed, str(e), cPickle.dumps(e)
except Exception as e:
import traceback
msg = traceback.format_exc()
ename = e.__class__.__name__
fatal_exceptions = (DparkUserFatalError, ArithmeticError,
ValueError, LookupError, SyntaxError,
TypeError, AssertionError)
prefix = "FATAL" if isinstance(e, fatal_exceptions) else "FAILED"
return TaskState.failed, '{}_EXCEPTION_{}'.format(prefix, ename), msg, cPickle.dumps(e)
finally:
gc.collect()
gc.enable()
def run_task(task_data):
try:
gc.disable()
task, ntry = loads(decompress(task_data))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception, e:
flag, data = 1, cPickle.dumps(result, -1)
else:
def run_task(task_data):
try:
gc.disable()
task, ntry = loads(decompress(task_data))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception, e:
flag, data = 1, cPickle.dumps(result, -1)
else:
def run_task(task_data):
try:
gc.disable()
task, task_try_id = loads(decompress(task_data))
ttid = TTID(task_try_id)
Accumulator.clear()
result = task.run(ttid.ttid)
env.task_stats.bytes_max_rss = resource.getrusage(
resource.RUSAGE_SELF).ru_maxrss * 1024
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception:
flag, data = 1, cPickle.dumps(result, -1)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
path = LocalFileShuffle.getOutputFile(0, task.id, ttid.task_try,
len(data))
f = open(path, 'wb')
f.write(data)
f.close()
data = '/'.join([LocalFileShuffle.getServerUri()] +
path.split('/')[-3:])
flag += 2
return TaskState.finished, cPickle.dumps(
((flag, data), accUpdate, env.task_stats), -1)
except FetchFailed as e:
return TaskState.failed, TaskEndReason.fetch_failed, str(
e), cPickle.dumps(e)
except Exception as e:
import traceback
msg = traceback.format_exc()
ename = e.__class__.__name__
return TaskState.failed, 'FAILED_EXCEPTION_{}'.format(
ename), msg, cPickle.dumps(e)
finally:
gc.collect()
gc.enable()
def run_task(task_data):
try:
gc.disable()
task, task_try_id = loads(decompress(task_data))
ttid = TTID(task_try_id)
Accumulator.clear()
result = task.run(ttid.ttid)
env.task_stats.bytes_max_rss = resource.getrusage(
resource.RUSAGE_SELF).ru_maxrss * 1024
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception:
flag, data = 1, cPickle.dumps(result, -1)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
path = LocalFileShuffle.getOutputFile(0, task.id, ttid.task_try,
len(data))
f = open(path, 'wb')
f.write(data)
f.close()
data = '/'.join([LocalFileShuffle.getServerUri()] +
path.split('/')[-3:])
flag += 2
return 'TASK_FINISHED', cPickle.dumps(
(Success(), (flag, data), accUpdate, env.task_stats), -1)
except FetchFailed as e:
return 'TASK_FAILED', cPickle.dumps((e, None, None, None), -1)
except:
import traceback
msg = traceback.format_exc()
return 'TASK_FAILED', cPickle.dumps(
(OtherFailure(msg), None, None, None), -1)
finally:
gc.collect()
gc.enable()
def run_task(task_data):
try:
gc.disable()
task, ntry = loads(decompress(task_data))
Accumulator.clear()
result = task.run(ntry)
accUpdate = Accumulator.values()
MutableDict.flush()
if marshalable(result):
try:
flag, data = 0, marshal.dumps(result)
except Exception as e:
flag, data = 1, cPickle.dumps(result, -1)
else:
flag, data = 1, cPickle.dumps(result, -1)
data = compress(data)
if len(data) > TASK_RESULT_LIMIT:
path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data))
f = open(path, 'w')
f.write(data)
f.close()
data = '/'.join(
[LocalFileShuffle.getServerUri()] + path.split('/')[-3:]
)
flag += 2
return 'TASK_FINISHED', cPickle.dumps(
(Success(), (flag, data), accUpdate), -1)
except FetchFailed as e:
return 'TASK_FAILED', cPickle.dumps((e, None, None), -1)
except:
import traceback
msg = traceback.format_exc()
return 'TASK_FAILED', cPickle.dumps(
(OtherFailure(msg), None, None), -1)
finally:
close_mfs()
gc.collect()
gc.enable()
def __setstate__(self, state):
self.__dict__, filters = state
self.filters = loads(filters)
def __setstate__(self, state):
d, rdd, func = state
self.__dict__.update(d)
self.rdd = loads(rdd)
self.func = load_func(func)
def __setstate__(self, state):
d, rdd, split = state
self.__dict__.update(d)
self.rdd = loads(rdd)
self.split = loads(split)
def __setstate__(self, state):
self.__dict__, filters = state
self.filters = loads(filters)
def __setstate__(self, state):
d, rdd, split = state
self.__dict__.update(d)
self.rdd = loads(rdd)
self.split = loads(split)