Exemple #1
0
    def testLocalMethodCallChain(self):
        exec_str = """
x = 1
class Bar(object):
    @classmethod
    def func1(self):
        return x
    def func2(self):
        return self.func1() + 1
    def func3(self):
        try:
            return x+2
        except:
            return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
        with main_environ(exec_str):
            _func1 = loads(dumps(MAIN.func1))
            _func2 = loads(dumps(MAIN.func2))
            _func3 = loads(dumps(MAIN.func3))

            assert _func1() == MAIN.x
            assert _func2() == MAIN.x + 1
            assert _func3() == MAIN.x + 2
Exemple #2
0
    def testLocalMethodCallChain(self):
        exec_str = """
x = 1
class Bar(object):
    @classmethod
    def func1(self):
        return x
    def func2(self):
        return self.func1() + 1
    def func3(self):
        try:
            return x+2
        except:
            return self.func2()
foo = Bar()
func1 = foo.func1
func2 = foo.func2
func3 = foo.func3
"""
        with main_environ(exec_str):
            _func1 = loads(dumps(MAIN.func1))
            _func2 = loads(dumps(MAIN.func2))
            _func3 = loads(dumps(MAIN.func3))

            assert _func1() == MAIN.x
            assert _func2() == MAIN.x + 1
            assert _func3() == MAIN.x + 2
Exemple #3
0
    def test_long_recursion(self):
        d = range(10)
        rdd = self.sc.makeRDD(d)
        for i in xrange(1000):
            rdd = rdd.map(lambda x: x+1)

        loads(dumps(rdd))
        self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
Exemple #4
0
    def test_long_recursion(self):
        d = list(range(10))
        rdd = self.sc.makeRDD(d)
        for i in range(1000):
            rdd = rdd.map(lambda x: x + 1)

        loads(dumps(rdd))
        self.assertEqual(rdd.collect(), [x + 1000 for x in d])
Exemple #5
0
    def test_long_recursion(self):
        d = list(range(10))
        rdd = self.sc.makeRDD(d)
        for i in range(1000):
            rdd = rdd.map(lambda x: x + 1)

        loads(dumps(rdd))
        self.assertEqual(rdd.collect(), [x + 1000 for x in d])
Exemple #6
0
    def test_long_recursion(self):
        d = range(10)
        rdd = self.sc.makeRDD(d)
        for i in xrange(1000):
            rdd = rdd.map(lambda x: x+1)

        loads(dumps(rdd))
        self.assertEqual(rdd.collect(), map(lambda x:x+1000, d))
Exemple #7
0
    def testOldStyleClass(self):
        exec_str = """
import csv

class DumDialect(csv.Dialect):
    delimiter = '\\t'
    quotechar = '"'
    escapechar = '\\\\'
    doublequote = False
    skipinitialspace = False
    lineterminator = '\\n'
    quoting = csv.QUOTE_MINIMAL
"""
        with main_environ(exec_str):
            loads(dumps(MAIN.DumDialect))
Exemple #8
0
    def testOldStyleClass(self):
        exec_str = """
import csv

class DumDialect(csv.Dialect):
    delimiter = '\\t'
    quotechar = '"'
    escapechar = '\\\\'
    doublequote = False
    skipinitialspace = False
    lineterminator = '\\n'
    quoting = csv.QUOTE_MINIMAL
"""
        with main_environ(exec_str):
            loads(dumps(MAIN.DumDialect))
Exemple #9
0
 def testRandomSample(self):
     from random import sample, Random
     _sample = loads(dumps(sample))
     assert _sample.__self__.__class__ is sample.__self__.__class__
     assert _sample.__func__ is sample.__func__
     assert isinstance(_sample.__self__, Random)
     assert isinstance(sample.__self__, Random)
Exemple #10
0
 def testRandomSample(self):
     from random import sample, Random
     _sample = loads(dumps(sample))
     assert _sample.im_class is sample.im_class
     assert _sample.im_func is sample.im_func
     assert isinstance(_sample.im_self, Random)
     assert isinstance(sample.im_self, Random)
Exemple #11
0
 def testRandomSample(self):
     from random import sample, Random
     _sample = loads(dumps(sample))
     assert _sample.im_class is sample.im_class
     assert _sample.im_func is sample.im_func
     assert isinstance(_sample.im_self, Random)
     assert isinstance(sample.im_self, Random)
Exemple #12
0
    def testMemberDescriptor(self):
        exec_str = """
class _ClsWithSlots(object):
    __slots__=['x', 'y']
"""
        with main_environ(exec_str):
            f = MAIN._ClsWithSlots()
            _f = loads(dumps(f))
            assert _f.__slots__ == f.__slots__
Exemple #13
0
    def testMemberDescriptor(self):
        exec_str = """
class _ClsWithSlots(object):
    __slots__=['x', 'y']
"""
        with main_environ(exec_str):
            f = MAIN._ClsWithSlots()
            _f = loads(dumps(f))
            assert _f.__slots__ == f.__slots__
Exemple #14
0
def run_task(task_data):
    try:
        gc.disable()
        task, task_try_id = loads(decompress(task_data))
        ttid = TTID(task_try_id)
        Accumulator.clear()
        result = task.run(ttid.ttid)
        env.task_stats.bytes_max_rss = resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss * 1024
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
            flag, data = 1, cPickle.dumps(result, -1)
        data = compress(data)

        if len(data) > TASK_RESULT_LIMIT:
            # shuffle_id start from 1
            swd = ShuffleWorkDir(0, task.id, ttid.task_try)
            tmppath = swd.alloc_tmp(len(data))
            with open(tmppath, 'wb') as f:
                f.write(data)
                f.close()
            path = swd.export(tmppath)
            data = '/'.join([env.server_uri] + path.split('/')[-3:])
            flag += 2

        return TaskState.finished, cPickle.dumps(
            ((flag, data), accUpdate, env.task_stats), -1)
    except FetchFailed as e:
        return TaskState.failed, TaskEndReason.fetch_failed, str(
            e), cPickle.dumps(e)
    except Exception as e:
        import traceback
        msg = traceback.format_exc()
        ename = e.__class__.__name__
        fatal_exceptions = (DparkUserFatalError, ArithmeticError, ValueError,
                            LookupError, SyntaxError, TypeError,
                            AssertionError)
        prefix = "FATAL" if isinstance(e, fatal_exceptions) else "FAILED"
        return TaskState.failed, '{}_EXCEPTION_{}'.format(
            prefix, ename), msg, cPickle.dumps(e)
    finally:
        gc.collect()
        gc.enable()
Exemple #15
0
def run_task(task_data):
    try:
        gc.disable()
        task, task_try_id = loads(decompress(task_data))
        ttid = TTID(task_try_id)
        Accumulator.clear()
        result = task.run(ttid.ttid)
        env.task_stats.bytes_max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
            flag, data = 1, cPickle.dumps(result, -1)
        data = compress(data)

        if len(data) > TASK_RESULT_LIMIT:
            # shuffle_id start from 1
            swd = ShuffleWorkDir(0, task.id, ttid.task_try)
            tmppath = swd.alloc_tmp(len(data))
            with open(tmppath, 'wb') as f:
                f.write(data)
                f.close()
            path = swd.export(tmppath)
            data = '/'.join(
                [env.server_uri] + path.split('/')[-3:]
            )
            flag += 2

        return TaskState.finished, cPickle.dumps(((flag, data), accUpdate, env.task_stats), -1)
    except FetchFailed as e:
        return TaskState.failed, TaskEndReason.fetch_failed, str(e), cPickle.dumps(e)
    except Exception as e:
        import traceback
        msg = traceback.format_exc()
        ename = e.__class__.__name__
        fatal_exceptions = (DparkUserFatalError, ArithmeticError,
                            ValueError, LookupError, SyntaxError,
                            TypeError, AssertionError)
        prefix = "FATAL" if isinstance(e, fatal_exceptions) else "FAILED"
        return TaskState.failed, '{}_EXCEPTION_{}'.format(prefix, ename), msg, cPickle.dumps(e)
    finally:
        gc.collect()
        gc.enable()
Exemple #16
0
def run_task(task_data):
    try:
        gc.disable()
        task, ntry = loads(decompress(task_data))
        Accumulator.clear()
        result = task.run(ntry)
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception, e:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
Exemple #17
0
def run_task(task_data):
    try:
        gc.disable()
        task, ntry = loads(decompress(task_data))
        Accumulator.clear()
        result = task.run(ntry)
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception, e:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
Exemple #18
0
def run_task(task_data):
    try:
        gc.disable()
        task, task_try_id = loads(decompress(task_data))
        ttid = TTID(task_try_id)
        Accumulator.clear()
        result = task.run(ttid.ttid)
        env.task_stats.bytes_max_rss = resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss * 1024
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
            flag, data = 1, cPickle.dumps(result, -1)
        data = compress(data)

        if len(data) > TASK_RESULT_LIMIT:
            path = LocalFileShuffle.getOutputFile(0, task.id, ttid.task_try,
                                                  len(data))
            f = open(path, 'wb')
            f.write(data)
            f.close()
            data = '/'.join([LocalFileShuffle.getServerUri()] +
                            path.split('/')[-3:])
            flag += 2

        return TaskState.finished, cPickle.dumps(
            ((flag, data), accUpdate, env.task_stats), -1)
    except FetchFailed as e:
        return TaskState.failed, TaskEndReason.fetch_failed, str(
            e), cPickle.dumps(e)
    except Exception as e:
        import traceback
        msg = traceback.format_exc()
        ename = e.__class__.__name__
        return TaskState.failed, 'FAILED_EXCEPTION_{}'.format(
            ename), msg, cPickle.dumps(e)
    finally:
        gc.collect()
        gc.enable()
Exemple #19
0
def run_task(task_data):
    try:
        gc.disable()
        task, task_try_id = loads(decompress(task_data))
        ttid = TTID(task_try_id)
        Accumulator.clear()
        result = task.run(ttid.ttid)
        env.task_stats.bytes_max_rss = resource.getrusage(
            resource.RUSAGE_SELF).ru_maxrss * 1024
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
            flag, data = 1, cPickle.dumps(result, -1)
        data = compress(data)

        if len(data) > TASK_RESULT_LIMIT:
            path = LocalFileShuffle.getOutputFile(0, task.id, ttid.task_try,
                                                  len(data))
            f = open(path, 'wb')
            f.write(data)
            f.close()
            data = '/'.join([LocalFileShuffle.getServerUri()] +
                            path.split('/')[-3:])
            flag += 2

        return 'TASK_FINISHED', cPickle.dumps(
            (Success(), (flag, data), accUpdate, env.task_stats), -1)
    except FetchFailed as e:
        return 'TASK_FAILED', cPickle.dumps((e, None, None, None), -1)
    except:
        import traceback
        msg = traceback.format_exc()
        return 'TASK_FAILED', cPickle.dumps(
            (OtherFailure(msg), None, None, None), -1)
    finally:
        gc.collect()
        gc.enable()
Exemple #20
0
def run_task(task_data):
    try:
        gc.disable()
        task, ntry = loads(decompress(task_data))
        Accumulator.clear()
        result = task.run(ntry)
        accUpdate = Accumulator.values()
        MutableDict.flush()

        if marshalable(result):
            try:
                flag, data = 0, marshal.dumps(result)
            except Exception as e:
                flag, data = 1, cPickle.dumps(result, -1)

        else:
            flag, data = 1, cPickle.dumps(result, -1)
        data = compress(data)

        if len(data) > TASK_RESULT_LIMIT:
            path = LocalFileShuffle.getOutputFile(0, ntry, task.id, len(data))
            f = open(path, 'w')
            f.write(data)
            f.close()
            data = '/'.join(
                [LocalFileShuffle.getServerUri()] + path.split('/')[-3:]
            )
            flag += 2

        return 'TASK_FINISHED', cPickle.dumps(
            (Success(), (flag, data), accUpdate), -1)
    except FetchFailed as e:
        return 'TASK_FAILED', cPickle.dumps((e, None, None), -1)
    except:
        import traceback
        msg = traceback.format_exc()
        return 'TASK_FAILED', cPickle.dumps(
            (OtherFailure(msg), None, None), -1)
    finally:
        close_mfs()
        gc.collect()
        gc.enable()
Exemple #21
0
 def __setstate__(self, state):
     self.__dict__, filters = state
     self.filters = loads(filters)
Exemple #22
0
 def __setstate__(self, state):
     d, rdd, func = state
     self.__dict__.update(d)
     self.rdd = loads(rdd)
     self.func = load_func(func)
Exemple #23
0
 def __setstate__(self, state):
     d, rdd, split = state
     self.__dict__.update(d)
     self.rdd = loads(rdd)
     self.split = loads(split)
Exemple #24
0
 def __setstate__(self, state):
     self.__dict__, filters = state
     self.filters = loads(filters)
Exemple #25
0
 def __setstate__(self, state):
     d, rdd, split = state
     self.__dict__.update(d)
     self.rdd = loads(rdd)
     self.split = loads(split)