def test_array_subclass(): try: import numpy as np class TestArray(np.ndarray): def __new__(cls, input_array, color): obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def __deepcopy__(self, memo): import copy import dill cls = self.__class__ result = cls.__new__(cls) memo[id(self)] = result for k, v in self.__dict__.items(): if v is self._cost: setattr(result, k, tuple(dill.copy(i) for i in v)) else: try: #XXX: work-around instancemethods in python2.6 setattr(result, k, copy.deepcopy(v, memo)) except TypeError: setattr(result, k, dill.copy(v)) return result
def errors(obj, depth=0, exact=False, safe=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception: import sys return sys.exc_info()[1] _dict = {} for attr in dir(obj): try: _attr = getattr(obj,attr) except Exception: import sys _dict[attr] = sys.exc_info()[1] continue if not pickles(_attr,exact,safe): _dict[attr] = errors(_attr,depth-1,exact,safe) return _dict
def test_data_unchanged(): FooS = dill.copy(Foo) try: res = FooS().data except Exception: e = sys.exc_info()[1] raise AssertionError(str(e)) else: assert res == 1
def test_super(): assert dill.copy(obj1(), byref=True) assert dill.copy(obj1(), byref=True, recurse=True) #assert dill.copy(obj1(), recurse=True) #FIXME: fails __main__.py assert dill.copy(obj1()) assert dill.copy(obj2(), byref=True) assert dill.copy(obj2(), byref=True, recurse=True) #assert dill.copy(obj2(), recurse=True) #FIXME: fails __main__.py assert dill.copy(obj2()) assert dill.copy(obj3(), byref=True) assert dill.copy(obj3(), byref=True, recurse=True) #assert dill.copy(obj3(), recurse=True) #FIXME: fails __main__.py assert dill.copy(obj3())
def test_mixins(): # test mixins assert double_add(1, 2, 3) == 2 * fx double_add.invert() assert double_add(1, 2, 3) == -2 * fx _d = dill.copy(double_add) assert _d(1, 2, 3) == -2 * fx _d.invert() assert _d(1, 2, 3) == 2 * fx assert _d.__wrapped__(1, 2, 3) == fx # XXX: issue or feature? in python3.4, inverted is linked through copy if not double_add.inverted[0]: double_add.invert() # test some stuff from source and pointers ds = dill.source dd = dill.detect assert ds.getsource(dd.freevars(quadish)["f"]) == "@quad_factory(a=0,b=4,c=0)\ndef quadish(x):\n return x+1\n" assert ds.getsource(dd.freevars(quadruple)["f"]) == "@doubler\ndef quadruple(x):\n return 2*x\n" assert ds.importable(quadish, source=False) == "from %s import quadish\n" % __name__ assert ds.importable(quadruple, source=False) == "from %s import quadruple\n" % __name__ assert ds.importable(quadratic, source=False) == "from %s import quadratic\n" % __name__ assert ds.importable(double_add, source=False) == "from %s import double_add\n" % __name__ assert ( ds.importable(quadruple, source=True) == "def doubler(f):\n def inner(*args, **kwds):\n fx = f(*args, **kwds)\n return 2*fx\n return inner\n\n@doubler\ndef quadruple(x):\n return 2*x\n" ) # ***** #FIXME: this needs work result = ds.importable(quadish, source=True) a, b, c, _, result = result.split("\n", 4) assert ( result == "def quad_factory(a=1,b=1,c=0):\n def dec(f):\n def func(*args,**kwds):\n fx = f(*args,**kwds)\n return a*fx**2 + b*fx + c\n return func\n return dec\n\n@quad_factory(a=0,b=4,c=0)\ndef quadish(x):\n return x+1\n" ) assert set([a, b, c]) == set(["a = 0", "c = 0", "b = 4"]) result = ds.importable(quadratic, source=True) a, b, c, result = result.split("\n", 3) assert ( result == "\ndef dec(f):\n def func(*args,**kwds):\n fx = f(*args,**kwds)\n return a*fx**2 + b*fx + c\n return func\n" ) assert set([a, b, c]) == set(["a = 1", "c = 0", "b = 1"]) result = ds.importable(double_add, source=True) a, b, c, d, _, result = result.split("\n", 5) assert ( result == "def quad(a=1, b=1, c=0):\n inverted = [False]\n def invert():\n inverted[0] = not inverted[0]\n def dec(f):\n def func(*args, **kwds):\n x = f(*args, **kwds)\n if inverted[0]: x = -x\n return a*x**2 + b*x + c\n func.__wrapped__ = f\n func.invert = invert\n func.inverted = inverted\n return func\n return dec\n\n@quad(a=0,b=2)\ndef double_add(*args):\n return sum(args)\n" ) assert set([a, b, c, d]) == set(["a = 0", "c = 0", "b = 2", "inverted = [True]"])
def errors(obj, depth=0, exact=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception, err: return err
def errors(obj, depth=0, exact=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception: import sys return sys.exc_info()[1] return dict(((attr, errors(getattr(obj,attr),depth-1,exact=exact)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact)))
def __init__(self, num_procs, controller_params, description): """ Initialization routine for PFASST controller Args: num_procs: number of parallel time steps (still serial, though), can be 1 controller_params: parameter set for the controller and the step class description: all the parameters to set up the rest (levels, problems, transfer, ...) """ # call parent's initialization routine super(allinclusive_classic_nonMPI, self).__init__(controller_params) self.logger.warning('classic controller is about to become deprecated, use multigrid controller instead') self.MS = [stepclass.step(description)] # try to initialize via dill.copy (much faster for many time-steps) try: for p in range(num_procs - 1): self.MS.append(dill.copy(self.MS[0])) # if this fails (e.g. due to un-picklable data in the steps), initialize seperately except dill.PicklingError and TypeError: self.logger.warning('Need to initialize steps separately due to pickling error') for p in range(num_procs - 1): self.MS.append(stepclass.step(description)) assert not (len(self.MS) > 1 and len(self.MS[0].levels) == 1), "ERROR: classic cannot do MSSDC" if self.params.dump_setup: self.dump_setup(step=self.MS[0], controller_params=controller_params, description=description) num_levels = len(self.MS[0].levels) if num_procs > 1 and num_levels > 1: for S in self.MS: for L in S.levels: if not L.sweep.coll.right_is_node or L.sweep.params.do_coll_update: raise ControllerError("For PFASST to work, we assume uend^k = u_M^k in this controller") for nl in range(len(self.MS[0].levels)): if self.MS[0].levels[nl].params.nsweeps > 1: raise ControllerError('classic controller cannot do multiple sweeps')
def __init__(self, num_procs, controller_params, description): """ Initialization routine for PFASST controller Args: num_procs: number of parallel time steps (still serial, though), can be 1 controller_params: parameter set for the controller and the steps description: all the parameters to set up the rest (levels, problems, transfer, ...) """ # call parent's initialization routine super(allinclusive_multigrid_nonMPI, self).__init__(controller_params) self.MS = [stepclass.step(description)] for p in range(num_procs - 1): self.MS.append(dill.copy(self.MS[0])) if self.params.dump_setup: self.dump_setup(step=self.MS[0], controller_params=controller_params, description=description) if num_procs > 1 and len(self.MS[0].levels) > 1: for S in self.MS: for L in S.levels: if not L.sweep.coll.right_is_node: raise ControllerError("For PFASST to work, we assume uend^k = u_M^k") if all(len(S.levels) == len(self.MS[0].levels) for S in self.MS): self.nlevels = len(self.MS[0].levels) else: raise ControllerError('all steps need to have the same number of levels') if self.nlevels == 0: raise ControllerError('need at least one level') self.nsweeps = [] for nl in range(self.nlevels): if all(S.levels[nl].params.nsweeps == self.MS[0].levels[nl].params.nsweeps for S in self.MS): self.nsweeps.append(self.MS[0].levels[nl].params.nsweeps) if self.nlevels > 1 and self.nsweeps[-1] > 1: raise ControllerError('this controller cannot do multiple sweeps on coarsest level')
def test_mixins(): # test mixins assert double_add(1,2,3) == 2*fx double_add.invert() assert double_add(1,2,3) == -2*fx _d = dill.copy(double_add) assert _d(1,2,3) == -2*fx #_d.invert() #FIXME: fails seemingly randomly #assert _d(1,2,3) == 2*fx assert _d.__wrapped__(1,2,3) == fx # XXX: issue or feature? in python3.4, inverted is linked through copy if not double_add.inverted[0]: double_add.invert() # test some stuff from source and pointers ds = dill.source dd = dill.detect assert ds.getsource(dd.freevars(quadish)['f']) == '@quad_factory(a=0,b=4,c=0)\ndef quadish(x):\n return x+1\n' assert ds.getsource(dd.freevars(quadruple)['f']) == '@doubler\ndef quadruple(x):\n return 2*x\n' assert ds.importable(quadish, source=False) == 'from %s import quadish\n' % __name__ assert ds.importable(quadruple, source=False) == 'from %s import quadruple\n' % __name__ assert ds.importable(quadratic, source=False) == 'from %s import quadratic\n' % __name__ assert ds.importable(double_add, source=False) == 'from %s import double_add\n' % __name__ assert ds.importable(quadruple, source=True) == 'def doubler(f):\n def inner(*args, **kwds):\n fx = f(*args, **kwds)\n return 2*fx\n return inner\n\n@doubler\ndef quadruple(x):\n return 2*x\n' #***** #FIXME: this needs work result = ds.importable(quadish, source=True) a,b,c,_,result = result.split('\n',4) assert result == 'def quad_factory(a=1,b=1,c=0):\n def dec(f):\n def func(*args,**kwds):\n fx = f(*args,**kwds)\n return a*fx**2 + b*fx + c\n return func\n return dec\n\n@quad_factory(a=0,b=4,c=0)\ndef quadish(x):\n return x+1\n' assert set([a,b,c]) == set(['a = 0', 'c = 0', 'b = 4']) result = ds.importable(quadratic, source=True) a,b,c,result = result.split('\n',3) assert result == '\ndef dec(f):\n def func(*args,**kwds):\n fx = f(*args,**kwds)\n return a*fx**2 + b*fx + c\n return func\n' assert set([a,b,c]) == set(['a = 1', 'c = 0', 'b = 1']) result = ds.importable(double_add, source=True) a,b,c,d,_,result = result.split('\n',5) assert result == 'def quad(a=1, b=1, c=0):\n inverted = [False]\n def invert():\n inverted[0] = not inverted[0]\n def dec(f):\n def func(*args, **kwds):\n x = f(*args, **kwds)\n if inverted[0]: x = -x\n return a*x**2 + b*x + c\n func.__wrapped__ = f\n func.invert = invert\n func.inverted = inverted\n return func\n return dec\n\n@quad(a=0,b=2)\ndef double_add(*args):\n return sum(args)\n' assert set([a,b,c,d]) == set(['a = 0', 'c = 0', 'b = 2', 'inverted = [True]'])
def solve(self, steps): self.problem.eval_count = 0 pool = pathos.pools.ProcessPool() n_cpus = pool.ncpus mapping_steps = int(steps**(1 / 5)) self.steps_per_pool = steps // (n_cpus * mapping_steps) print(self.steps_per_pool) print(mapping_steps) print(n_cpus) to_solve = dill.copy(self) for i in range(mapping_steps): solutions = pool.map(self._solve_pool, [to_solve] * n_cpus) self.best_solution = max( (s for s in solutions + [self.best_solution]), key=lambda x: x.score) self.problem.eval_count += self.steps_per_pool * n_cpus self.record() print(i * n_cpus * self.steps_per_pool)
return 2*fx return inner @doubler def quadruple(x): return 2*x if __name__ == '__main__': # test mixins assert double_add(1,2,3) == 2*fx double_add.invert() assert double_add(1,2,3) == -2*fx _d = dill.copy(double_add) assert _d(1,2,3) == -2*fx _d.invert() assert _d(1,2,3) == 2*fx assert _d.__wrapped__(1,2,3) == fx # XXX: issue or feature? in python3.4, inverted is linked through copy if not double_add.inverted[0]: double_add.invert() # test some stuff from source and pointers ds = dill.source dd = dill.detect assert ds.getsource(dd.freevars(quadish)['f']) == '@quad_factory(a=0,b=4,c=0)\ndef quadish(x):\n return x+1\n' assert ds.getsource(dd.freevars(quadruple)['f']) == '@doubler\ndef quadruple(x):\n return 2*x\n'
def test_data_not_none(): FooS = dill.copy(Foo) assert FooS.data.fget is not None assert FooS.data.fset is not None assert FooS.data.fdel is None
def test_super(): assert dill.copy(obj1(), byref=True) assert dill.copy(obj1(), byref=True, recurse=True) assert dill.copy(obj1(), recurse=True) assert dill.copy(obj1()) assert dill.copy(obj2(), byref=True) assert dill.copy(obj2(), byref=True, recurse=True) assert dill.copy(obj2(), recurse=True) assert dill.copy(obj2()) assert dill.copy(obj3(), byref=True) assert dill.copy(obj3(), byref=True, recurse=True) assert dill.copy(obj3(), recurse=True) assert dill.copy(obj3())
del sys.modules[module.__name__] del module module = dill.loads(pik_mod) assert hasattr(module, "a") and module.a == 1234 assert module.double_add(1, 2, 3) == 2 * module.fx except AttributeError: pass # clean up import os os.remove(cached) pycache = os.path.join(os.path.dirname(module.__file__), "__pycache__") if os.path.exists(pycache) and not os.listdir(pycache): os.removedirs(pycache) # test when module is None import math def get_lambda(str, **kwarg): return eval(str, kwarg, None) obj = get_lambda('lambda x: math.exp(x)', math=math) assert obj.__module__ is None assert dill.copy(obj)(3) == obj(3) # EOF
def dec(f): def func(*args, **kwds): x = f(*args, **kwds) if inverted[0]: x = -x return a*x**2 + b*x + c func.__wrapped__ = f func.invert = invert return func return dec @quad(a=0,b=2) def double_add(*args): return sum(args) fx = sum([1,2,3]) assert double_add(1,2,3) == 2*fx double_add.invert() assert double_add(1,2,3) == -2*fx _d = dill.copy(double_add) assert _d(1,2,3) == -2*fx _d.invert() assert _d(1,2,3) == 2*fx assert _d.__wrapped__(1,2,3) == fx # EOF
class TestArray(np.ndarray): def __new__(cls, input_array, color): obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def test_partial(): assert dill.copy(Machine(), byref=True) assert dill.copy(Machine(), byref=True, recurse=True) if not OLDER: assert dill.copy(Machine(), recurse=True) assert dill.copy(Machine())
def test_partials(): assert dill.copy(SubMachine(), byref=True) assert dill.copy(SubMachine(), byref=True, recurse=True) #if not OLDER: #FIXME: fails __main__.py # assert dill.copy(SubMachine(), recurse=True) assert dill.copy(SubMachine())
#Lets print the best fit params print("Loglik = %s" % fit.loglik) fit.print_info() #this is an obsolete function call, will be replaced! # To get the phased signals: for i in range(fit.npl): rv.phase_RV_planet_signal(fit, i + 1) ################# Dynamical fitting ############################# # now lets fit dynamics model starting from the best Keplerian derived above #first lets copy the Keplerian object, we will need it later for plotting import dill kep_fit = dill.copy(fit) ################# Plotting ############################# # lets make some basic plots with the "fit" object results: import matplotlib.pyplot as plt from matplotlib import gridspec import matplotlib as mpl import numpy as np ###### For nice plotting ############## mpl.rcParams['axes.linewidth'] = 2.0 #set the value globally mpl.rcParams['xtick.major.pad'] = '8'
def test_slots(): assert dill.pickles(Y) assert dill.pickles(y) assert dill.pickles(Y.y) assert dill.copy(y).y == value
obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1, 2, 3, 4, 5]) a3.color = 'green' assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def __init__(self, num_procs, controller_params, description): """ Initialization routine for PFASST controller Args: num_procs: number of parallel time steps (still serial, though), can be 1 controller_params: parameter set for the controller and the steps description: all the parameters to set up the rest (levels, problems, transfer, ...) """ if 'predict' in controller_params: raise ControllerError( 'predict flag is ignored, use predict_type instead') # call parent's initialization routine super(controller_nonMPI, self).__init__(controller_params) self.MS = [stepclass.step(description)] # try to initialize via dill.copy (much faster for many time-steps) try: for p in range(num_procs - 1): self.MS.append(dill.copy(self.MS[0])) # if this fails (e.g. due to un-picklable data in the steps), initialize seperately except dill.PicklingError and TypeError: self.logger.warning( 'Need to initialize steps separately due to pickling error') for p in range(num_procs - 1): self.MS.append(stepclass.step(description)) if self.params.dump_setup: self.dump_setup(step=self.MS[0], controller_params=controller_params, description=description) if num_procs > 1 and len(self.MS[0].levels) > 1: for S in self.MS: for L in S.levels: if not L.sweep.coll.right_is_node: raise ControllerError( "For PFASST to work, we assume uend^k = u_M^k") if all(len(S.levels) == len(self.MS[0].levels) for S in self.MS): self.nlevels = len(self.MS[0].levels) else: raise ControllerError( 'all steps need to have the same number of levels') if self.nlevels == 0: raise ControllerError('need at least one level') self.nsweeps = [] for nl in range(self.nlevels): if all(S.levels[nl].params.nsweeps == self.MS[0].levels[nl].params.nsweeps for S in self.MS): self.nsweeps.append(self.MS[0].levels[nl].params.nsweeps) if self.nlevels > 1 and self.nsweeps[-1] > 1: raise ControllerError( 'this controller cannot do multiple sweeps on coarsest level') if self.nlevels == 1 and self.params.predict_type is not None: self.logger.warning( 'you have specified a predictor type but only a single level.. ' 'predictor will be ignored')
#lets print the best fit params print("Loglik = %s"%fit.loglik) fit.print_info() #this is an obsolete function call, will be replaced! ################# Dynamical fitting ############################# # now lets fit dynamics model starting from the best Keplerian derived above #first lets copy the Keplerian object, we will need it later for plotting import dill kep_fit = dill.copy(fit) fit.mod_dynamical=True fit.fitting(outputfiles=[1,1,1], doGP=False, minimize_fortran=True, minimize_loglik=True, amoeba_starts=20, print_stat=False, eps=1000, dt=10, npoints=6000, model_max=0, model_min=0) #lets print the best fit params print("Loglik = %s"%fit.loglik) fit.print_info() #this is an obsolete function call, will be replaced! #lets copy the fit object as dyn_fit, we will need it later for plotting dyn_fit = dill.copy(fit)
def test_module_is_none(): assert obj.__module__ is None assert dill.copy(obj)(3) == obj(3)
class Foo(object): def __init__(self): self._data = 1 def _get_data(self): return self._data def _set_data(self, x): self._data = x data = property(_get_data, _set_data) FooS = dill.copy(Foo) assert FooS.data.fget is not None assert FooS.data.fset is not None assert FooS.data.fdel is None try: res = FooS().data except Exception: e = sys.exc_info()[1] raise AssertionError(str(e)) else: assert res == 1 try: f = FooS()