def test_array_subclass(): try: import numpy as np class TestArray(np.ndarray): def __new__(cls, input_array, color): obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def test_no_throw(self): if not has_ase_with_ce: self.skipTest("ASE version does not have CE") return no_throw = True msg = "" try: ceBulk, atoms = self.init_bulk_crystal() chem_pots = {"c1_0": 0.02, "c1_1": -0.03} T = 600.0 mc = SGCMonteCarlo(atoms, T, symbols=["Al", "Mg", "Si"]) mc.runMC(steps=100, chem_potential=chem_pots) E = mc.get_thermodynamic()["energy"] # Try with recycling mc = SGCMonteCarlo(atoms, T, symbols=["Al", "Mg", "Si"], recycle_waste=True) mc.runMC(steps=100, chem_potential=chem_pots) E2 = mc.get_thermodynamic()["energy"] rel_diff = abs(E2 - E) / abs(E) # Make sure that there is less than 10% difference to rule # out obvious bugs self.assertLess(rel_diff, 0.1) # Try to pickle import dill dill.pickles(mc) except Exception as exc: msg = str(exc) no_throw = False self.assertTrue(no_throw, msg=msg)
def badtypes(obj, depth=0, exact=False, safe=False): """get types for objects that fail to pickle""" from dill import pickles if not depth: if pickles(obj,exact,safe): return None return type(obj) return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
def badobjects(obj, depth=0, exact=False): """get objects that fail to pickle""" from dill import pickles if not depth: if pickles(obj,exact): return None return obj return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact=exact)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact)))
def badtypes(obj, depth=0, exact=False, safe=False): """get types for objects that fail to pickle""" from dill import pickles if not depth: if pickles(obj, exact, safe): return None return type(obj) return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe)))
def test_class(): o = _d() oo = _newclass() ok = dill.pickles(o) if verbose: print ("%s: %s, %s" % (ok, type(o), o)) assert ok ok = dill.pickles(oo) if verbose: print ("%s: %s, %s" % (ok, type(oo), oo)) assert ok if verbose: print ("")
def test_functools(): fp = functools.partial(f, 1, 2) gp = functools.partial(g, 1, c=2) hp = functools.partial(h, 1, c=2) bp = functools.partial(int, base=2) assert dill.pickles(fp, safe=True) assert dill.pickles(gp, safe=True) assert dill.pickles(hp, safe=True) assert dill.pickles(bp, safe=True)
def test_class(): o = _d() oo = _newclass() ok = dill.pickles(o) if verbose: print("%s: %s, %s" % (ok, type(o), o)) assert ok ok = dill.pickles(oo) if verbose: print("%s: %s, %s" % (ok, type(oo), oo)) assert ok if verbose: print("")
def test_class_descriptors(): d = _d.__dict__ for i in d.values(): ok = dill.pickles(i) if verbose: print ("%s: %s, %s" % (ok, type(i), i)) assert ok if verbose: print ("") od = _newclass.__dict__ for i in od.values(): ok = dill.pickles(i) if verbose: print ("%s: %s, %s" % (ok, type(i), i)) assert ok if verbose: print ("")
def test_class_descriptors(): d = _d.__dict__ for i in d.values(): ok = dill.pickles(i) if verbose: print("%s: %s, %s" % (ok, type(i), i)) assert ok if verbose: print("") od = _newclass.__dict__ for i in od.values(): ok = dill.pickles(i) if verbose: print("%s: %s, %s" % (ok, type(i), i)) assert ok if verbose: print("")
def errors(obj, depth=0, exact=False, safe=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception: import sys return sys.exc_info()[1] _dict = {} for attr in dir(obj): try: _attr = getattr(obj,attr) except Exception: import sys _dict[attr] = sys.exc_info()[1] continue if not pickles(_attr,exact,safe): _dict[attr] = errors(_attr,depth-1,exact,safe) return _dict
def errors(obj, depth=0, exact=False, safe=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception: import sys return sys.exc_info()[1] _dict = {} for attr in dir(obj): try: _attr = getattr(obj, attr) except Exception: import sys _dict[attr] = sys.exc_info()[1] continue if not pickles(_attr, exact, safe): _dict[attr] = errors(_attr, depth - 1, exact, safe) return _dict
def write_current_state(sampler, resume_file, sampling_time): """ Writes a checkpoint file Parameters ---------- sampler: dynesty.NestedSampler The sampler object itself resume_file: str The name of the resume/checkpoint file to use sampling_time: float The total sampling time in seconds """ print("") #print('prior transform',sampler.prior_transform) #delete the sample_rwalk_parallel_with_act function before pickle print(sampler.evolve_point) del sampler.evolve_point del sampler.prior_transform del sampler.loglikelihood sampler.kwargs["sampling_time"] = sampling_time # safe_file_dump(sampler, resume_file, dill) if dill.pickles(sampler): safe_file_dump(sampler, resume_file, dill) logger.info("Written checkpoint file {}".format(resume_file)) else: logger.warning("Cannot write pickle resume file! " "Job will not resume if interrupted.")
def _serialize_func(func): name = func.__name__ if name == '<lambda>': raise ValueError('Cannot serialize lambda expression') if not dill.pickles(func): raise ValueError('Cannot pickle func {}'.format(func)) module = func.__module__ return dict(class_name=name, module=module)
def test_monkey_patch_pickle(f): f_temp = _wrap_task_call(f) assert dill.pickles(f_temp), "{} is not pickling correctly!".format(f) # Pickle everything s1 = dill.dumps(f_temp) s2 = dill.loads(s1) dill.dumps(s2)
def test_frame_related(): g = _g(1) f = g.gi_frame e,t = _f() _is = lambda ok: not ok if dill.dill.IS_PYPY else ok ok = dill.pickles(f) if verbose: print ("%s: %s, %s" % (ok, type(f), f)) assert _is(not ok) #XXX: dill fails ok = dill.pickles(g) if verbose: print ("%s: %s, %s" % (ok, type(g), g)) assert _is(not ok) #XXX: dill fails ok = dill.pickles(t) if verbose: print ("%s: %s, %s" % (ok, type(t), t)) assert not ok #XXX: dill fails ok = dill.pickles(e) if verbose: print ("%s: %s, %s" % (ok, type(e), e)) assert ok if verbose: print ("")
def isValidFunc(function): res = dill.pickles(function) if res: return True else: logger = FlockLogger() logger.error("Your function is not pickable") print("Function not pickable") return False
def test_frame_related(): g = _g(1) f = g.gi_frame e, t = _f() _is = lambda ok: not ok if dill._dill.IS_PYPY else ok ok = dill.pickles(f) if verbose: print("%s: %s, %s" % (ok, type(f), f)) assert _is(not ok) #XXX: dill fails ok = dill.pickles(g) if verbose: print("%s: %s, %s" % (ok, type(g), g)) assert _is(not ok) #XXX: dill fails ok = dill.pickles(t) if verbose: print("%s: %s, %s" % (ok, type(t), t)) assert not ok #XXX: dill fails ok = dill.pickles(e) if verbose: print("%s: %s, %s" % (ok, type(e), e)) assert ok if verbose: print("")
def test_dict_contents(): c = type.__dict__ for i in c.values(): if dill.dill.IS_PYPY and type(i) is type(type.__dict__['__dict__']): print("SKIPPED: %s" % i) #FIXME: if verbose continue #NOTE: __objclass__ attribute is missing (pypy bug) ok = dill.pickles(i) if verbose: print ("%s: %s, %s" % (ok, type(i), i)) assert ok if verbose: print ("")
def test_dict_contents(): c = type.__dict__ for i,j in c.items(): #try: ok = dill.pickles(j) #except: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print ("%s: %s, %s" % (ok, type(j), j)) assert ok if verbose: print ("")
def test_dict_contents(): c = type.__dict__ for i, j in c.items(): #try: ok = dill.pickles(j) #except: # print ("FAIL: %s with %s" % (i, dill.detect.errors(j))) if verbose: print("%s: %s, %s" % (ok, type(j), j)) assert ok if verbose: print("")
def make_args_pickable(args): """ Makes a namespace pickable """ pickable_args = argparse.Namespace() # - go through fields in args, if they are not pickable make it a string else leave as it # The vars() function returns the __dict__ attribute of the given object. for field in vars(args): field_val = getattr(args, field) if not dill.pickles(field): field_val = str(field_val) setattr(args, field, field_val) return pickable_args
def isValidFunc(function): test1 = dill.pickles(function) test2 = isfunction(function) if all([test1, test2]): return True else: logger = FlockLogger() logger.error("Your function is not pickable") print("Function not pickable") return False
def clean_dict(config): """ Make sure that all values of a dictionary are dumpable to YAML. If they are not, replace them with their type. """ for key, value in config.items(): if isinstance(value, collections.abc.Mapping): config[key] = clean_dict(config[key]) else: if not dill.pickles(value): config[key] = type(value) return config
def test_array_subclass(): try: import numpy as np class TestArray(np.ndarray): def __new__(cls, input_array, color): obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') if dill._dill.PY3 and not dill._dill.IS_PYPY: assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] if dill._dill.PY3 and not dill._dill.IS_PYPY: assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1, 2, 3, 4, 5]) a3.color = 'green' if dill._dill.PY3 and not dill._dill.IS_PYPY: assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def save(self, filename): """ Save agent object. By default, the agent is pickled. If overridden, the load() method must also be overriden. Before saving, consider setting writer to None if it can't be pickled (tensorboard writers keep references to files and cannot be pickled). Note: dill[1]_ is used when pickle fails (see https://stackoverflow.com/a/25353243, for instance). Pickle is tried first, since it is faster. Parameters ---------- filename: Path or str File in which to save the Agent. Returns ------- pathlib.Path If save() is successful, a Path object corresponding to the filename is returned. Otherwise, None is returned. .. warning:: The returned filename might differ from the input filename: For instance, the method can append the correct suffix to the name before saving. References ---------- .. [1] https://github.com/uqfoundation/dill """ # remove writer if not pickleable if not dill.pickles(self.writer): self.set_writer(None) # save filename = Path(filename).with_suffix(".pickle") filename.parent.mkdir(parents=True, exist_ok=True) try: with filename.open("wb") as ff: pickle.dump(self.__dict__, ff) except Exception: try: with filename.open("wb") as ff: dill.dump(self.__dict__, ff) except Exception as ex: logger.warning("Agent instance cannot be pickled: " + str(ex)) return None return filename
def write_current_state(self): import dill logger.debug("Check point") check_directory_exists_and_if_not_mkdir(self.outdir) _pool = self.ptsampler.pool self.ptsampler.pool = None if dill.pickles(self.ptsampler): safe_file_dump(self.ptsampler, self.resume_file, dill) logger.info("Written checkpoint file {}".format(self.resume_file)) else: logger.warning("Cannot write pickle resume file! " "Job will not resume if interrupted.") self.ptsampler.pool = _pool
async def _serialize_namespace( self, input_serializers: MutableMapping[Text, Any], job: Job, namespace: MutableMapping[Text, Any]) -> Text: for name, value in namespace.items(): if name in input_serializers: intermediate_type = input_serializers[name].get('type', 'name') if intermediate_type == 'file': namespace[name] = dill.dumps(await self._transfer_file( job, dill.loads(value))) try: return await self._serialize_to_remote_file(job, namespace) except BaseException: for k, v in namespace.items(): if not dill.pickles(v, safe=True): raise WorkflowExecutionException( "Name {name} is not serializable".format(name=k)) raise
def errors(obj, depth=0, exact=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception: import sys return sys.exc_info()[1] return dict(((attr, errors(getattr(obj,attr),depth-1,exact=exact)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact)))
def save(self, filename): filename = Path(filename).with_suffix(".pickle") filename.parent.mkdir(parents=True, exist_ok=True) writer = None if dill.pickles(self.writer): writer = self.writer agent_data = dict( rng_key=self.rng_key, params=self._all_params, states=self._all_states, writer=writer, ) with filename.open("wb") as ff: dill.dump(agent_data, ff) return filename
def write_current_state(self): """ Write the current state of the sampler to disk. The sampler is pickle dumped using `dill`. The sampling time is also stored to get the full CPU time for the run. The check of whether the sampler is picklable is to catch an error when using pytest. Hopefully, this message won't be triggered during normal running. """ from ... import __version__ as bilby_version from dynesty import __version__ as dynesty_version import dill if getattr(self, "sampler", None) is None: # Sampler not initialized, not able to write current state return check_directory_exists_and_if_not_mkdir(self.outdir) end_time = datetime.datetime.now() if hasattr(self, 'start_time'): self.sampling_time += end_time - self.start_time self.start_time = end_time self.sampler.kwargs["sampling_time"] = self.sampling_time self.sampler.kwargs["start_time"] = self.start_time self.sampler.versions = dict( bilby=bilby_version, dynesty=dynesty_version ) self.sampler.pool = None self.sampler.M = map if dill.pickles(self.sampler): safe_file_dump(self.sampler, self.resume_file, dill) logger.info("Written checkpoint file {}".format(self.resume_file)) else: logger.warning( "Cannot write pickle resume file! " "Job will not resume if interrupted." ) self.sampler.pool = self.pool if self.sampler.pool is not None: self.sampler.M = self.sampler.pool.map
def write_current_state(sampler, resume_file, sampling_time): """Writes a checkpoint file Parameters ---------- sampler: dynesty.NestedSampler The sampler object itself resume_file: str The name of the resume/checkpoint file to use sampling_time: float The total sampling time in seconds """ print("") logger.info("Start checkpoint writing") sampler.kwargs["sampling_time"] = sampling_time if dill.pickles(sampler): safe_file_dump(sampler, resume_file, dill) logger.info("Written checkpoint file {}".format(resume_file)) else: logger.warning("Cannot write pickle resume file!")
def GPUCell(self, line, cell): var_names = list(self.shell.user_ns.keys()) var_dir = self.shell.user_ns print("Saving current environment... ") f = shelve.open('in.txt') for key in var_names: try: if (dill.pickles(var_dir[key]) and key != 'f'): f[key] = var_dir[key] except Exception as e: # # __builtins__, my_shelf, and imported modules can not be shelved. # # print('ERROR shelving: {0}'.format(key)) pass f.close() with open('tmp.py', 'w') as tmp: tmp.writelines(pre_code) tmp.writelines(cell) tmp.writelines(post_code) print("Executing GPU code.") p = subprocess.Popen(['python', 'tmp.py'], stdout=subprocess.PIPE) while True: output = p.stdout.readline() if output == '' and p.poll() is not None: break if output: print(output.strip()) p.poll() ___out_shelf = shelve.open('out.txt') for key in ___out_shelf: self.shell.user_ns[key] = ___out_shelf[key] ___out_shelf.close() return
def populateTopicNode(self, key): spot = key.find('.') depth = key[:spot] key = key[spot + 1:] node = self.nodes[key] if node.getDepthFound() == 0: node.setDepthFound(depth) keyxyz = node.getTopic().getName() if (node.isPopulated()): logging.warning(node.getTopic(), 'ERROR1') return None sourceCode = WebTool.getValidatedTopicSourceCode( node.getTopic().getName()) if sourceCode == None: logging.warning(node.getTopic(), 'ERROR2') return None sourceElement = SourceElement(sourceCode) sourceElement.validateName(node) if (node.isPopulated()): logging.warning(node.getTopic(), 'ERROR3') return None links = sourceElement.grabIntroAndSeeAlsoLinks(node) logging.debug('%s %s' % (node.getTopic(), '|')) self.addInfoToNewNodes(node, links) node.setCategory(sourceElement.getCategories()) node.setIsPopulated() self.createConnectionDetails(node) logging.debug() if dill.pickles(node): return node else: logging.debug(dill.detect.badtypes(node, depth=1).keys()) logging.critical(node.getTopic(), 'Failed pickle') return None
class TestArray(np.ndarray): def __new__(cls, input_array, color): obj = np.asarray(input_array).view(cls) obj.color = color return obj def __array_finalize__(self, obj): if obj is None: return if isinstance(obj, type(self)): self.color = obj.color def __getnewargs__(self): return np.asarray(self), self.color a1 = TestArray(np.zeros(100), color='green') assert dill.pickles(a1) assert a1.__dict__ == dill.copy(a1).__dict__ a2 = a1[0:9] assert dill.pickles(a2) assert a2.__dict__ == dill.copy(a2).__dict__ class TestArray2(np.ndarray): color = 'blue' a3 = TestArray2([1,2,3,4,5]) a3.color = 'green' assert dill.pickles(a3) assert a3.__dict__ == dill.copy(a3).__dict__ except ImportError: pass
def f(func): def w(*args): return f(*args) return w @f def f2(): pass # check when __main__ and on import assert dill.pickles(f2) import doctest import logging logging.basicConfig(level=logging.DEBUG) class SomeUnreferencedUnpicklableClass(object): def __reduce__(self): raise Exception unpicklable = SomeUnreferencedUnpicklableClass() # This works fine outside of Doctest:
def test_none(): assert dill.pickles(type(None))
def test_class_instances(): assert dill.pickles(o) assert dill.pickles(oc) assert dill.pickles(n) assert dill.pickles(nc) assert dill.pickles(m)
def test_slots(): assert dill.pickles(Y) assert dill.pickles(y) assert dill.pickles(Y.y) assert dill.copy(y).y == value
return er, tb class _d(object): def _method(self): pass from dill import objects from dill import load_types load_types(pickleable=True,unpickleable=False) _newclass = objects['ClassObjectType'] del objects # getset_descriptor for new-style classes (fails on '_method', if not __main__) d = _d.__dict__ for i in d.values(): print ("%s: %s, %s" % (dill.pickles(i), type(i), i)) print ("") od = _newclass.__dict__ for i in od.values(): print ("%s: %s, %s" % (dill.pickles(i), type(i), i)) print ("") """ # (__main__) class instance for new-style classes o = _d() oo = _newclass() print ("%s: %s, %s" % (dill.pickles(o), type(o), o)) print ("%s: %s, %s" % (dill.pickles(oo), type(oo), oo)) print ("") """
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2008-2014 California Institute of Technology. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE import functools import dill def f(a, b, c): # without keywords pass def g(a, b, c=2): # with keywords pass def h(a=1, b=2, c=3): # without args pass fp = functools.partial(f, 1, 2) gp = functools.partial(g, 1, c=2) hp = functools.partial(h, 1, c=2) bp = functools.partial(int, base=2) assert dill.pickles(fp, safe=True) assert dill.pickles(gp, safe=True) assert dill.pickles(hp, safe=True) assert dill.pickles(bp, safe=True)
:rtype: """ with open('package_data.pickle', 'wb') as file: dill.dump(self, file) @staticmethod def load_from_pickle(): """ Returns a new Packages() initialized from pickled data. If no pickled data exists, we return a newly initialized Packages(). :return: A new initialized Packages() :rtype: Packages """ try: with open('package_data.pickle', 'rb') as file: ret = dill.load(file) return ret except FileNotFoundError: return Packages() if __name__ == '__main__': packages = Packages() packages.add_package("usps", "12836933") print(dill.pickles(packages)) # Test to ensure object is still picklable
def ok(self): return True _mclass = _meta("_mclass", (object,), {"__call__": __call__, "ok": ok}) del __call__ del ok o = _class() oc = _class2() n = _newclass() nc = _newclass2() m = _mclass() # test pickles for class instances assert dill.pickles(o) assert dill.pickles(oc) assert dill.pickles(n) assert dill.pickles(nc) assert dill.pickles(m) clslist = [_class,_class2,_newclass,_newclass2,_mclass] objlist = [o,oc,n,nc,m] _clslist = [dill.dumps(obj) for obj in clslist] _objlist = [dill.dumps(obj) for obj in objlist] for obj in clslist: globals().pop(obj.__name__) del clslist for obj in ['o','oc','n','nc']: globals().pop(obj)
def test_decorated(): assert dill.pickles(f2)
for attr in dir(obj) if not pickles(getattr(obj,attr),exact))) def badtypes(obj, depth=0, exact=False): """get types for objects that fail to pickle""" from dill import pickles if not depth: if pickles(obj,exact): return None return type(obj) return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact=exact)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact))) def errors(obj, depth=0, exact=False): """get errors for objects that fail to pickle""" from dill import pickles, copy if not depth: try: pik = copy(obj) if exact: assert pik == obj, \ "Unpickling produces %s instead of %s" % (pik,obj) assert type(pik) == type(obj), \ "Unpickling produces %s instead of %s" % (type(pik),type(obj)) return None except Exception, err: return err return dict(((attr, errors(getattr(obj,attr),depth-1,exact=exact)) \ for attr in dir(obj) if not pickles(getattr(obj,attr),exact))) # EOF
# License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/dill/LICENSE import dill dill.settings['recurse'] = True def f(func): def w(*args): return f(*args) return w @f def f2(): pass # check when __main__ and on import assert dill.pickles(f2) import doctest import logging logging.basicConfig(level=logging.DEBUG) class SomeUnreferencedUnpicklableClass(object): def __reduce__(self): raise Exception unpicklable = SomeUnreferencedUnpicklableClass() # This works fine outside of Doctest: serialized = dill.dumps(lambda x: x)