def run(self, cell_model, param_values, sim=None, isolate=None): """Instantiate protocol""" if isolate is None: isolate = True if isolate: def _reduce_method(meth): """Overwrite reduce""" return (getattr, (meth.__self__, meth.__func__.__name__)) import copy_reg import types copy_reg.pickle(types.MethodType, _reduce_method) import multiprocessing pool = multiprocessing.Pool(1, maxtasksperchild=1) responses = pool.apply( self._run_func, kwds={ 'cell_model': cell_model, 'param_values': param_values, 'sim': sim}) pool.terminate() pool.join() del pool else: responses = self._run_func( cell_model=cell_model, param_values=param_values, sim=sim) return responses
def __init__(self,N,ntree,maxfea,leafsize,N_proc=None): self.N = N self.ntree = ntree; self.maxfea = maxfea; self.leafsize = leafsize self.N_proc = N_proc if N_proc is not None else max(1,multiprocessing.cpu_count()-1) # fix pickling when using bound methods in classes pickle(MethodType, _pickle_method, _pickle_method)
def test_factory_registered_with_copy_reg(self): # Factories registered with copy_reg.pickle loose their __name__. # We simply ignore those. class AnonymousFactory(object): def __new__(cls, name): return object.__new__(cls) def __init__(self, name): self._name = name def getName(self): return self._name sys.modules['module1'].AnonymousFactory = AnonymousFactory sys.modules['module1'].AnonymousFactory.__module__ = 'module1' sys.modules['module1'].Anonymous = AnonymousFactory('Anonymous') import copy_reg copy_reg.pickle(AnonymousFactory, AnonymousFactory.getName, AnonymousFactory) self.root['test'] = sys.modules['module1'].Anonymous transaction.commit() updater = self.update() self.assertEquals('module1', self.root['test'].__class__.__module__) self.assertEquals('AnonymousFactory', self.root['test'].__class__.__name__) renames = updater.processor.get_found_implicit_rules() self.assertEquals({}, renames)
def install(): try: import copy_reg except ImportError: import copyreg as copy_reg copy_reg.pickle(TracebackType, pickle_traceback)
def initmessage(message_descriptor, cls): """constructs a new message instance (called before instance's __init__).""" cls._extensions_by_name = {} _addinitmethod(message_descriptor, cls) _addmessagemethods(message_descriptor, cls) _addpropertiesforextensions(message_descriptor, cls) copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
def _register_patched_dtype_reduce(): """ Numpy < 1.7 has a bug when copying/pickling dtype objects with a zero-width void type--i.e. ``np.dtype('V0')``. Specifically, although creating a void type is perfectly valid, it crashes when instantiating a dtype using a format string of 'V0', which is what is normally returned by dtype.__reduce__() for these dtypes. See https://github.com/astropy/astropy/pull/3283#issuecomment-81667461 """ if NUMPY_LT_1_7: import numpy as np import copy_reg # Originally this created an alternate constructor that fixed this # issue, and returned that constructor from the new reduce_dtype; # however that broke pickling since functions can't be pickled, so now # we fix the issue directly within the custom __reduce__ def reduce_dtype(obj): info = obj.__reduce__() args = info[1] if args[0] == 'V0': args = ('V',) + args[1:] info = (info[0], args) + info[2:] return info copy_reg.pickle(np.dtype, reduce_dtype)
def test_dynamic_class(self): a = create_dynamic_class("my_dynamic_class", (object,)) copy_reg.pickle(pickling_metaclass, pickling_metaclass.__reduce__) for proto in protocols: s = self.dumps(a, proto) b = self.loads(s) self.assertEqual(a, b)
def rebuildCtype(type_, wrapper, length): if length is not None: type_ = type_ * length if sys.platform == 'win32' and type_ not in copy_reg.dispatch_table: copy_reg.pickle(type_, reduceCtype) obj = type_.from_address(wrapper.getAddress()) obj._wrapper = wrapper return obj
def enable(self): if not self._enabled: self._enabled = True self._remap_modules({a: b for a, b in self._remap}) copy_reg.pickle(datetime_orig.date, lambda d: (_WarpedDatetime.date,) + d.__reduce__()[1:]) copy_reg.pickle(datetime_orig.datetime, lambda d: (_WarpedDatetime.datetime,) + d.__reduce__()[1:])
def reg_code_pickler(): """ Called automatically when the module is loaded, this function will ensure that the CodeType has pickle/unpickle functions registered with copy_reg """ copy_reg.pickle(CodeType, _pickle_code, _unpickle_code)
def _registerEnumPicklers(): from copy_reg import constructor, pickle def reduce_enum(e): enum = type(e).__name__.split('.')[-1] return ( _tuple2enum, ( enum, int(e) ) ) constructor( _tuple2enum) for e in [ e for e in vars(ctimb).itervalues() if isEnumType(e) ]: pickle(e, reduce_enum)
def mk_slots (*objs): """Make the classes that have __slots__ but not __dict__ serialisable. Takes a number of types (new-style classes) to make serialisable. """ for cls in objs: copy_reg.pickle(cls, _reduce_slots)
def __init__(self, package): # Ensure that instance methods can be pickled. def reduce_method(m): return getattr, (m.__self__, m.__func__.__name__) copy_reg.pickle(types.MethodType, reduce_method) super(EventStorage, self).__init__('data/%s/' % package)
def _mpq_pickle_support(): """Allow instances of gmpy.mpq to pickle.""" from gmpy import mpq mpq_type = type(mpq(1, 10)) # gmpy doesn't appear to expose the type another way import copy_reg copy_reg.pickle(mpq_type, lambda q: (mpq, (q.digits(),)))
def Flags(cls): if cls.__flags_class__ is not None: return cls.__flags_class__ name = cls.__name__ + 'Flags' flags_class = type(name, (cls.IntEnumFlags,), {}) flags_class.__enum_class__ = cls cls.__flags_class__ = flags_class copy_reg.pickle(flags_class, _int_enum_flags_pickler) return flags_class
def main(): #Add this so we can pickle QTextDocuments copy_reg.pickle(QTextDocument,pickle_QtD,unpickle_QtD) app = QApplication(sys.argv) mainWindow = MainWindow() mainWindow.setWindowTitle("WireFrame Model") mainWindow.show() mainWindow.centralWidget().viewingWindow.animate() app.exec_()
def __do_wrap( value ): if isinstance( value, SafeStringWrapper ): # Only ever wrap one-layer return value if callable( value ): safe_class = CallableSafeStringWrapper else: safe_class = SafeStringWrapper if isinstance( value, no_wrap_classes ): return value if isinstance( value, __DONT_WRAP_TYPES__ ): return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) if isinstance( value, __WRAP_NO_SUBCLASS__ ): return safe_class( value, safe_string_wrapper_function=__do_wrap ) for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: if isinstance( value, this_type ): return this_type( map( __do_wrap, value ) ) for this_type in __WRAP_MAPPINGS__: if isinstance( value, this_type ): # Wrap both key and value return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. # This allows e.g. isinstance to continue to work. try: wrapped_class_name = value.__name__ wrapped_class = value except: wrapped_class_name = value.__class__.__name__ wrapped_class = value.__class__ value_mod = inspect.getmodule( value ) if value_mod: wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) do_wrap_func = __do_wrap global_dict = globals() if wrapped_class_name in global_dict: # Check to see if we have created a wrapper for this class yet, if so, reuse wrapped_class = global_dict.get( wrapped_class_name ) do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) else: try: wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) except TypeError, e: # Fail-safe for when a class cannot be dynamically subclassed. log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): # Save this wrapper for reuse and pickling/copying global_dict[ wrapped_class_name ] = wrapped_class do_wrap_func.__name__ = do_wrap_func_name global_dict[ do_wrap_func_name ] = do_wrap_func def pickle_safe_object( safe_object ): return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) # Set pickle and copy properties copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func )
def _get_proxy_type(name): clsname = "IPCProxy_" + name mod = sys.modules[__name__] if hasattr(mod, clsname): cls = getattr(mod, clsname) else: cls = classobj(clsname, (IPCProxy,), {}) copy_reg.pickle(cls, _pickle_proxy, _unpickle_proxy) setattr(mod, clsname, cls) return cls
def int_enum_flags(name, enum_class, module=None): assert issubclass(enum_class, IntEnum) if enum_class.__flags_class__ is not None: return enum_class.__flags_class__ dct = {'__module__': module or enum_class.__module__} flags_class = type(name, (_IntEnumFlags,), dct) flags_class.__enum_class__ = enum_class enum_class.__flags_class__ = flags_class copy_reg.pickle(flags_class, _int_enum_flags_pickler) return flags_class
def _register_pickling(): crh = connected_region_handler # Register picking functions for connected region import copy_reg def reduce_connected_region(obj): return ConnectedRegion, (crh.get_shape(obj), crh.get_value(obj), crh.get_start_row(obj), crh.get_rowptr(obj), crh.get_colptr(obj)) copy_reg.pickle(ConnectedRegion, reduce_connected_region)
def install_cacheops(): """ Installs cacheops by numerous monkey patches """ monkey_mix(Manager, ManagerMixin) monkey_mix(QuerySet, QuerySetMixin) QuerySet._cacheprofile = QuerySetMixin._cacheprofile QuerySet._cloning = QuerySetMixin._cloning # DateQuerySet existed in Django 1.7 and earlier # Values*QuerySet existed in Django 1.8 and earlier from django.db.models import query for cls_name in ('ValuesQuerySet', 'ValuesListQuerySet', 'DateQuerySet'): if hasattr(query, cls_name): cls = getattr(query, cls_name) monkey_mix(cls, QuerySetMixin, ['iterator']) try: # Use app registry in Django 1.7 from django.apps import apps admin_used = apps.is_installed('django.contrib.admin') get_models = apps.get_models except ImportError: # Introspect INSTALLED_APPS in older djangos from django.conf import settings admin_used = 'django.contrib.admin' in settings.INSTALLED_APPS from django.db.models import get_models # Install profile and signal handlers for any earlier created models for model in get_models(include_auto_created=True): model._default_manager._install_cacheops(model) # Turn off caching in admin if admin_used: from django.contrib.admin.options import ModelAdmin # Renamed queryset to get_queryset in Django 1.6 method_name = 'get_queryset' if hasattr(ModelAdmin, 'get_queryset') else 'queryset' @monkey(ModelAdmin, name=method_name) def get_queryset(self, request): return get_queryset.original(self, request).nocache() # Bind m2m changed handler m2m_changed.connect(invalidate_m2m) # Make buffers/memoryviews pickleable to serialize binary field data if six.PY2: import copy_reg copy_reg.pickle(buffer, lambda b: (buffer, (bytes(b),))) if six.PY3: import copyreg copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),)))
def test_copy_registry(self): class C(object): def __new__(cls, foo): obj = object.__new__(cls) obj.foo = foo return obj def pickle_C(obj): return (C, (obj.foo,)) x = C(42) self.assertRaises(TypeError, copy.copy, x) copy_reg.pickle(C, pickle_C, C) y = copy.copy(x)
def __init__(self, block_graph=None): if block_graph is None: block_graph = BlockGraph(#parents=dict(), children=defaultdict(functools.partial(defaultdict, set)) ) # This last one is equal to children=defaultdict( lambda:defaultdict(set) )) # Note the extra brackets at the end. TODO: Learn more FP. # Make explicit the fact that the files, defs, and classes members # together form the universal set of elements. There is redundancy # here, but it adds semantic value to the data structure. self.block_graph = block_graph copy_reg.pickle(functools.partial, Children_ToReduce)
def fit(self, X, y=None): self.X = X self.y = y creator.create("FitnessMax", base.Fitness, weights=(1.0,)) creator.create("Individual", list, fitness=creator.FitnessMax) toolbox = base.Toolbox() toolbox.register("attr_bool", random.randint, 0, 1) toolbox.register("individual", tools.initRepeat, creator.Individual, toolbox.attr_bool, n=self.individual_size) toolbox.register("population", tools.initRepeat, list, toolbox.individual) toolbox.register("evaluate", self._evalFunction) toolbox.register("mate", tools.cxTwoPoint) toolbox.register("mutate", tools.mutFlipBit, indpb=self.mutation_prob) toolbox.register("select", tools.selTournament, tournsize=self.tournament_size) if self.n_jobs > 1: copyreg.pickle(types.MethodType, _reduce_method) pool = Pool(processes=self.n_jobs) # self.toolbox.register("map", parmap) toolbox.register("map", pool.map) pop = toolbox.population(n=self.population_size) hof = tools.HallOfFame(1) stats = tools.Statistics(lambda ind: ind.fitness.values) stats.register("min", np.min) stats.register("avg", np.mean) stats.register("max", np.max) if self.verbose: print('--- Evolve in {0} possible combinations ---'.format(len(self.possible_params))) pop, logbook = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2, ngen=self.generations_number, stats=stats, halloffame=hof, verbose=self.verbose) self.best_score_ = hof[0].fitness self.best_params_ = self._individual_to_params(BitArray(hof[0]).uint) if self.verbose: import json print("Best individual is: %s\nwith fitness: %s" % ( json.dumps(self.best_params_), hof[0].fitness) ) if self.refit: self.best_estimator_ = clone(self.estimator) if self.best_params_: self.best_estimator_.set_params(**self.best_params_) self.best_estimator_.fit(self.X, self.y)
def __init__(self, mesh, data, singularity, symmetry, indices_pack, name, derived): self._mesh = mesh self._data = data self._singularity = singularity self._symmetry = symmetry self._indices_pack = indices_pack self.name = name self._derived = derived self.indicesR, self.indicesL = indices_pack copy_reg.pickle(derived, reductor, builder_cls_with_dict_arg )
def pickle_it(self, obj, path): global pickle try: pickle except NameError: import cPickle as pickle import copy_reg copy_reg.pickle(slice, lambda s: (slice, (s.start, s.stop, s.step))) sys.stderr.write("writing [%s]/%s\n" % (self.package_name, os.path.basename(path))) with open(path, 'wb') as f: pickle.dump((pyke.version, pyke.compiler_version), f) pickle.dump(obj, f)
def install_cacheops(): """ Installs cacheops by numerous monkey patches """ monkey_mix(Manager, ManagerMixin) monkey_mix(QuerySet, QuerySetMixin) # Use app registry to introspect used apps from django.apps import apps # Install profile and signal handlers for any earlier created models for model in apps.get_models(include_auto_created=True): if family_has_profile(model): if not isinstance(model._default_manager, Manager): raise ImproperlyConfigured("Can't install cacheops for %s.%s model:" " non-django model class or manager is used." % (model._meta.app_label, model._meta.model_name)) model._default_manager._install_cacheops(model) # Bind m2m changed handlers rel_attr = 'remote_field' if django.VERSION >= (1, 9) else 'rel' m2ms = (f for f in model._meta.get_fields(include_hidden=True) if f.many_to_many) for m2m in m2ms: rel = m2m if hasattr(m2m, 'through') else getattr(m2m, rel_attr, m2m) opts = rel.through._meta m2m_changed.connect(invalidate_m2m, sender=rel.through, dispatch_uid=(opts.app_label, opts.model_name)) # Turn off caching in admin if apps.is_installed('django.contrib.admin'): from django.contrib.admin.options import ModelAdmin @monkey(ModelAdmin) def get_queryset(self, request): return get_queryset.original(self, request).nocache() # Make buffers/memoryviews pickleable to serialize binary field data if six.PY2: import copy_reg copy_reg.pickle(buffer, lambda b: (buffer, (bytes(b),))) # noqa if six.PY3: import copyreg copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),))) # Fix random ordered dict keys producing different SQL for same QuerySet if (3, 3) <= sys.version_info < (3, 6): from django.db.models.query_utils import Q def Q__init__(self, *args, **kwargs): # noqa super(Q, self).__init__(children=list(args) + list(sorted(kwargs.items()))) Q.__init__ = Q__init__
def prepare(): """ Initialise the state module. """ global state_file, loaded state_file = config.get('state') if os.path.exists(state_name): state_file = state_name else: state_file = os.path.join(plat.state_path, state_name) # do not load any state file from a package if config.package: state_file = '' # register the picklers for file and cStringIO copy_reg.pickle(file, pickle_file) copy_reg.pickle(cStringIO.OutputType, pickle_StringIO)
def _instance_method_pickle_support(): """Allow instance methods to pickle.""" # CB: well, it seems to work - maybe there are cases where this # wouldn't work? # Alternative technique (totally different approach), but would # only work with pickle (not cPickle): # http://code.activestate.com/recipes/572213/ def _pickle_instance_method(mthd): mthd_name = mthd.im_func.__name__ obj = mthd.im_self return getattr, (obj,mthd_name) import copy_reg, types copy_reg.pickle(types.MethodType, _pickle_instance_method)
def _numpy_ufunc_pickle_support(): """ Allow instances of numpy.ufunc to pickle. """ # Remove this when numpy.ufuncs themselves support pickling. # Code from Robert Kern; see: # http://news.gmane.org/find-root.php?group=gmane.comp.python.numeric.general&article=13400 from numpy import ufunc import copy_reg def ufunc_pickler(ufunc): """Return the ufunc's name""" return ufunc.__name__ copy_reg.pickle(ufunc, ufunc_pickler)
# Attempt a last-ditch fix before giving up. If classes have changed # around since we pickled this method, we may still be able to get it # by looking on the instance's current class. if im_self.__class__ is im_class: raise return unpickleMethod(im_name, im_self, im_self.__class__) else: if _PY3: maybeClass = () else: maybeClass = tuple([im_class]) bound = types.MethodType(methodFunction, im_self, *maybeClass) return bound copy_reg.pickle(types.MethodType, pickleMethod, unpickleMethod) def _pickleFunction(f): """ Reduce, in the sense of L{pickle}'s C{object.__reduce__} special method, a function object into its constituent parts. @param f: The function to reduce. @type f: L{types.FunctionType} @return: a 2-tuple of a reference to L{_unpickleFunction} and a tuple of its arguments, a 1-tuple of the function's fully qualified name. @rtype: 2-tuple of C{callable, native string} """ if f.__name__ == '<lambda>':
##----------------------------------------------------------------------------- def _pickle_method(method): fct_name = method.im_func.__name__ obj = method.im_self cl = method.im_class ## handle mangled function name if(fct_name.startswith("__"))and(not fct_name.endswith("__")): cl_name = cl.__name__.lstrip("_") fct_name = "_" + cl_name + fct_name return _unpickle_method, (fct_name, obj, cl) ##----------------------------------------------------------------------------- def _unpickle_method(fct_name, obj, cl): for oo in cl.__mro__: try: fct = oo.__dict__[fct_name] except(KeyError): pass else: break return fct.__get__(obj, cl) ##----------------------------------------------------------------------------- ## enable instance methods pickling cr.pickle(mt, _pickle_method, _unpickle_method)
Match = type(_pat.match('')) del _pat # We'll define an alias for the 'compile' function so that the repr of a # pattern object is eval-able. Regex = compile # Register myself for pickling. import copy_reg as _copy_reg def _pickle(pattern): return _regex.compile, pattern._pickled_data _copy_reg.pickle(Pattern, _pickle) if not hasattr(str, "format"): # Strings don't have the .format method (below Python 2.6). while True: _start = __doc__.find(" subf") if _start < 0: break _end = __doc__.find("\n", _start) + 1 while __doc__.startswith(" ", _end): _end = __doc__.find("\n", _end) + 1 __doc__ = __doc__[:_start] + __doc__[_end:] __all__ = [_name for _name in __all__ if not _name.startswith("subf")]
import marshal def unpickle_code(ms): co = marshal.loads(ms) raise isinstance(co, types.CodeType) or AssertionError return co def pickle_code(co): raise isinstance(co, types.CodeType) or AssertionError ms = marshal.dumps(co) return (unpickle_code, (ms, )) copy_reg.pickle(types.CodeType, pickle_code, unpickle_code) BUFSIZE = 8 * 1024 LOCALHOST = '127.0.0.1' class RPCServer(SocketServer.TCPServer): def __init__(self, addr, handlerclass=None): if handlerclass is None: handlerclass = RPCHandler SocketServer.TCPServer.__init__(self, addr, handlerclass) return def server_bind(self): """Override TCPServer method, no bind() phase for connecting entity""" pass
func_name = method.__name__ obj = method.__self__ cls = obj.__class__ return unpickle_method, (func_name, obj, cls) def unpickle_method(func_name, obj, cls): for cls in cls.mro(): try: func = cls.__dict__[func_name] except KeyError: pass else: break return func.__get__(obj, cls) copy_reg.pickle(types.MethodType, pickle_method, unpickle_method) # concatenates con. and disc. specs along axis 0, handling empty disc. specs def concat_specs(c_specs, d_specs): if len(d_specs): return np.concatenate((c_specs, d_specs), axis=0) else: return c_specs # handle Pool not being a context manager in Python < 3.4 @contextlib.contextmanager def create_pool(*args, **kwargs): if sys.version_info < (3, 4): pool = multiprocessing.Pool(*args, **kwargs) yield pool pool.terminate()
def register_reduce(mcls): """Register __reduce__ as reduction function for mcls instances.""" copyreg.pickle(mcls, mcls.__reduce__) return mcls
import types import cumulative import matplotlib import time matplotlib.use("Agg") def _pickle_method(m): if m.im_self is None: return getattr, (m.im_class, m.im_func.func_name) else: return getattr, (m.im_self, m.im_func.func_name) copy_reg.pickle(types.MethodType, _pickle_method) # --------------------- # DPGMM posterior class # --------------------- class DPGMMSkyPosterior(object): """ Dirichlet Process Gaussian Mixture model class input parameters: posterior_samples: posterior samples for which the density estimate needs to be calculated dimension: the dimensionality of the problem. default = 3
from types import MethodType from TarThread import TarThread from mongodb_consistent_backup.Errors import Error, OperationError from mongodb_consistent_backup.Pipeline import Task # Allows pooled .apply_async()s to work on Class-methods: def _reduce_method(m): if m.im_self is None: return getattr, (m.im_class, m.im_func.func_name) else: return getattr, (m.im_self, m.im_func.func_name) pickle(MethodType, _reduce_method) class Tar(Task): def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs): super(Tar, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir, **kwargs) self.compression_method = self.config.archive.tar.compression self.binary = "tar" self._pool = None self._pooled = [] def done(self, done_dir): if done_dir in self._pooled: logging.debug("Archiving completed for: %s" % done_dir) self._pooled.remove(done_dir)
import copy_reg, pickle from SpheralCompiledPackages import * #------------------------------------------------------------------------------- # Vector1d #------------------------------------------------------------------------------- def construct_Vector1d(x): return Vector1d(x) def reduce_Vector1d(obj): return construct_Vector1d, (obj.x, ) copy_reg.pickle(type(Vector1d()), reduce_Vector1d, construct_Vector1d) #------------------------------------------------------------------------------- # Vector2d #------------------------------------------------------------------------------- def construct_Vector2d(x, y): return Vector2d(x, y) def reduce_Vector2d(obj): return construct_Vector2d, (obj.x, obj.y) copy_reg.pickle(type(Vector2d()), reduce_Vector2d, construct_Vector2d)
# We define _pattern_type here after all the support objects have been defined. _pattern_type = type(_compile("", 0, {})) # We'll define an alias for the 'compile' function so that the repr of a # pattern object is eval-able. Regex = compile # Register myself for pickling. import copy_reg as _copy_reg def _pickle(pattern): return _regex.compile, pattern._pickled_data _copy_reg.pickle(_pattern_type, _pickle) if not hasattr(str, "format"): # Strings don't have the .format method (below Python 2.6). while True: _start = __doc__.find(" subf") if _start < 0: break _end = __doc__.find("\n", _start) + 1 while __doc__.startswith(" ", _end): _end = __doc__.find("\n", _end) + 1 __doc__ = __doc__[:_start] + __doc__[_end:] __all__ = [_name for _name in __all__ if not _name.startswith("subf")]
} """, version=3) # THIS WORKS But CudaNdarray instances don't compare equal to one # another, and what about __hash__ ? So the unpickled version doesn't # equal the pickled version, and the cmodule cache is not happy with # the situation. def CudaNdarray_unpickler(npa): if config.experimental.unpickle_gpu_on_cpu: # directly return numpy array warnings.warn("config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray") return npa elif cuda: return cuda.CudaNdarray(npa) else: raise ImportError("Cuda not found. Cannot unpickle CudaNdarray") copy_reg.constructor(CudaNdarray_unpickler) def CudaNdarray_pickler(cnda): return (CudaNdarray_unpickler, (numpy.asarray(cnda),)) # In case cuda is not imported. if cuda is not None: copy_reg.pickle(cuda.CudaNdarray, CudaNdarray_pickler, CudaNdarray_unpickler)
from PYME.localization.cModels.gauss_app import * #from scipy import weave from PYME.Analysis._fithelpers import * def pickleSlice(slice): return unpickleSlice, (slice.start, slice.stop, slice.step) def unpickleSlice(start, stop, step): return slice(start, stop, step) copy_reg.pickle(slice, pickleSlice, unpickleSlice) def f_gauss2d(p, X, Y): """2D Gaussian model function with linear background - parameter vector [A, x0, y0, sigma, background, lin_x, lin_y]""" A, x0, y0, s, b, b_x, b_y = p #return A*scipy.exp(-((X-x0)**2 + (Y - y0)**2)/(2*s**2)) + b + b_x*X + b_y*Y r = genGauss(X, Y, A, x0, y0, s, b, b_x, b_y) r.strides = r.strides #Really dodgy hack to get around something which numpy is not doing right .... return r def f_gauss2dF(p, X, Y): """2D Gaussian model function with linear background - parameter vector [A, x0, y0, sigma, background, lin_x, lin_y] - uses fast exponential approx""" A, x0, y0, s, b, b_x, b_y = p #return A*scipy.exp(-((X-x0)**2 + (Y - y0)**2)/(2*s**2)) + b + b_x*X + b_y*Y
def filter(match, template=template): return sre_parse.expand_template(template, match) return filter # register myself for pickling import copy_reg def _pickle(p): return _compile, (p.pattern, p.flags) copy_reg.pickle(_pattern_type, _pickle, _compile) # -------------------------------------------------------------------- # experimental stuff (see python-dev discussions for details) class Scanner: def __init__(self, lexicon, flags=0): from sre_constants import BRANCH, SUBPATTERN self.lexicon = lexicon # combine phrases into a compound pattern p = [] s = sre_parse.Pattern() s.flags = flags for phrase, action in lexicon: p.append(
def __getslice__(self, i, j): return str(self)[i:j] def __iter__(self): for c in str(self): yield c def __len__(self): return len(self.text) def XML_unpickle(data): return XML(marshal.loads(data)) def XML_pickle(data): return XML_unpickle, (marshal.dumps(str(data)),) copy_reg.pickle(XML, XML_pickle, XML_unpickle) # ################################################################ # BEAUTIFY everything # ################################################################ def BEAUTIFY(obj): # FIX ME, dealing with very large objects if isinstance(obj, TAGGER): return obj elif isinstance(obj, list): return UL(*[LI(BEAUTIFY(item)) for item in obj]) elif isinstance(obj, dict): return TABLE(TBODY(*[TR(TH(XML(key)),TD(BEAUTIFY(value))) for key, value in obj.items()])) else: return XML(obj)
response.session_file.truncate() finally: self._close(response) self.save_session_id_cookie() return True def _unlock(self, response): if response and response.session_file and response.session_locked: try: portalocker.unlock(response.session_file) response.session_locked = False except: # this should never happen but happens in Windows pass def _close(self, response): if response and response.session_file: self._unlock(response) try: response.session_file.close() del response.session_file except: pass def pickle_session(s): return Session, (dict(s), ) copy_reg.pickle(Session, pickle_session)
# use numpy's default reduction return numpy_reduce(obj) else: # some other unknown buffer # use numpy's default reduction return numpy_reduce(obj) def rebuild_array(wrapper, shape, dtype, order, strides, offset): """ Rebuild an array with the given information. """ arr = np.asarray(BufferWrapperArray(wrapper, shape, dtype, order, strides, offset)) return arr numpy_reduce = np.ndarray.__reduce__ copy_reg.pickle(np.ndarray, reduce_shared_array) def shared_empty(shape, dtype=float, order='C'): """ Create a shared-memory ndarray without initializing its contents. """ dtype = np.dtype(dtype) if isinstance(shape, (int, long, np.integer)): shape = (shape,) shape = tuple(shape) size = int(np.prod(shape)) nbytes = size * dtype.itemsize wrapper = heap.BufferWrapper(nbytes) strides = None offset = 0 arr = rebuild_array(wrapper, shape, dtype, order, strides, offset)
finite_genome=finite_genome, u=u, v=v, alternate_fg=alternate_fg) #return self # comment out (returned for testing earlier) # Allow TLSpectrum objects to be pickled. # See http://effbot.org/librarybook/copy-reg.htm try: import copy_reg except: import copyreg def TLSpectrum_pickler(fs): # Collect all the info necessary to save the state of a TLSpectrum return TLSpectrum_unpickler, (fs.data, fs.mask, fs.folded) def TLSpectrum_unpickler(data, mask, folded): # Use that info to recreate the TLSpectrum return TLSpectrum(data, mask, mask_infeasible=False, data_folded=folded) try: copy_reg.pickle(TLSpectrum, TLSpectrum_pickler, TLSpectrum_unpickler) except: copyreg.pickle(TLSpectrum, TLSpectrum_pickler, TLSpectrum_unpickler)
if _coconut_sys.version_info < (2, 7): import functools as _coconut_functools, copy_reg as _coconut_copy_reg def _coconut_new_partial(func, args, keywords): return _coconut_functools.partial( func, *(args if args is not None else ()), **(keywords if keywords is not None else {})) _coconut_copy_reg.constructor(_coconut_new_partial) def _coconut_reduce_partial(self): return (_coconut_new_partial, (self.func, self.args, self.keywords)) _coconut_copy_reg.pickle(_coconut_functools.partial, _coconut_reduce_partial) else: py_chr, py_filter, py_hex, py_input, py_int, py_map, py_oct, py_open, py_print, py_range, py_str, py_zip = chr, filter, hex, input, int, map, oct, open, print, range, str, zip class _coconut(object): import collections, functools, imp, itertools, operator, types, copy, pickle if _coconut_sys.version_info < (3, 3): abc = collections else: import collections.abc as abc IndexError, NameError, ValueError, map, zip, dict, filter, frozenset, getattr, hasattr, hash, isinstance, iter, len, list, min, max, next, object, range, reversed, set, slice, str, sum, super, tuple, bytearray, repr = IndexError, NameError, ValueError, map, zip, dict, filter, frozenset, getattr, hasattr, hash, isinstance, iter, len, list, min, max, next, object, range, reversed, set, slice, str, sum, super, tuple, bytearray, staticmethod( repr) class MatchError(Exception):
cls = method.im_class return _unpickle_method, (func_name, obj, cls) def _unpickle_method(func_name, obj, cls): for cls in cls.mro(): try: func = cls.__dict__[func_name] except KeyError: pass else: break return func.__get__(obj, cls) pickle(MethodType, _pickle_method, _unpickle_method) gr.SetBatch(True) # NEEDS TO BE SET FOR MULTIPROCESSING OF plot.Draw() Cut = namedtuple('Cut', ['name', 'cut']) int_lumi = 41000.0 # pb #### FIXME #int_lumi = 80000.0 # pb #### FIXME ## RICCARDO # cuts.append(Cut('ttjetsloose', 'nbj>1')) # cuts.append(Cut('zmmloose' , 'l1_pt>5 & l2_pt>5 & l1_q!=l2_q & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto')) # cuts.append(Cut('zmmhighpt', 'l1_pt>15 & l2_pt>15 & l1_q!=l2_q & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto')) # cuts.append(Cut('zmm' , 'l1_pt>10 & l2_pt>10 & l1_q!=l2_q & !l0_eid_mva_iso_loose & l0_reliso05>0.15 & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto')) # cuts.append(Cut('inclusive' , 'l0_pt>30 & l1_pt>4 & l2_pt>4 & l1_q != l2_q & l0_eid_mva_iso_loose & l0_reliso05<0.15')) # cuts.append(Cut('inclusive' , 'l0_pt>30 & l1_pt>4 & l2_pt>4 & l1_q != l2_q & l0_eid_mva_iso_loose & l0_reliso05<0.15 & l1_id_m & l2_id_m & l1_reliso05<0.2 & l2_reliso05<0.2'))
s = st s += 'Layer {0}\n'.format(self.layer_index) s += st for n, model in enumerate(self): s += '\n' s += model.__repr__() if n == len(self) - 1: s += '\n' return s def add(self, model): model.model_index = len(self) self.append(model) self.input_index_set.add(model.u1_index) self.input_index_set.add(model.u2_index) def delete(self, index): self.pop(index) for n in range(index, len(self)): self[n].model_index = n self.input_index_set.clear() for model in self: self.input_index_set.add(model.u1_index) self.input_index_set.add(model.u2_index) import sys if sys.version_info.major == 2: import copy_reg as cpr cpr.pickle(types.MethodType, _pickle_method, _unpickle_method)
#proxy file starts first -- maybe change this import socket import signal import time from multiprocessing import Process, Manager import copy_reg from multiprocessing.reduction import rebuild_socket, reduce_socket copy_reg.pickle(socket.socket, reduce_socket, rebuild_socket) ##function to handle incoming connections def handleConnection(connections, c, addr): print "handler started for %s" % (c) name = c.recv(100) print "Connection is from %s" % (name) #add alice connection to the connection pool print "type c = " + str(type(c)) ############################ THIS PART DOES NOT WORK ############# THE SHARED MEMORY item = connections[name] = list() item.append(c) connections[name] = item print "connections : " + str(connections) c.send('Thank you for connecting %s\n' % (name)) ###### listen forever from client till client says to stop ##### while True: message = "" message = c.recv(1024) if message.split()[0] == 'System': print name + ' closed Connection'
def get_ilwdchar_class(tbl_name, col_name, namespace=globals()): """ Searches this module's namespace for a subclass of _ilwd.ilwdchar whose table_name and column_name attributes match those provided. If a matching subclass is found it is returned; otherwise a new class is defined, added to this module's namespace, and returned. Example: >>> process_id = get_ilwdchar_class("process", "process_id") >>> x = process_id(10) >>> str(type(x)) "<class 'pycbc.ligolw.ilwd.process_process_id_class'>" >>> str(x) 'process:process_id:10' Retrieving and storing the class provides a convenient mechanism for quickly constructing new ID objects. Example: >>> for i in range(10): ... print str(process_id(i)) ... process:process_id:0 process:process_id:1 process:process_id:2 process:process_id:3 process:process_id:4 process:process_id:5 process:process_id:6 process:process_id:7 process:process_id:8 process:process_id:9 """ # # if the class already exists, retrieve and return it # key = (str(tbl_name), str(col_name)) cls_name = "%s_%s_class" % key assert cls_name != "get_ilwdchar_class" try: return namespace[cls_name] except KeyError: pass # # otherwise define a new class, and add it to the cache # class new_class(_ilwd.ilwdchar): __slots__ = () table_name, column_name = key index_offset = len("%s:%s:" % key) new_class.__name__ = cls_name namespace[cls_name] = new_class # # pickle support # copy_reg.pickle(new_class, lambda x: (ilwdchar, (unicode(x), ))) # # return the new class # return new_class
import popen2 stdout, stdin = popen2.popen4(cmd, bufsize) return stdin, stdout __all__.append("popen4") import copy_reg as _copy_reg def _make_stat_result(tup, dict): return stat_result(tup, dict) def _pickle_stat_result(sr): (type, args) = sr.__reduce__() return (_make_stat_result, args) try: _copy_reg.pickle(stat_result, _pickle_stat_result, _make_stat_result) except NameError: # stat_result may not exist pass def _make_statvfs_result(tup, dict): return statvfs_result(tup, dict) def _pickle_statvfs_result(sr): (type, args) = sr.__reduce__() return (_make_statvfs_result, args) try: _copy_reg.pickle(statvfs_result, _pickle_statvfs_result, _make_statvfs_result) except NameError: # statvfs_result may not exist pass
""" import sys import array import copy_reg import numpy as np __all__ = ['Vector', 'DenseVector', 'SparseVector', 'Vectors'] if sys.version_info[:2] == (2, 7): # speed up pickling array in Python 2.7 def fast_pickle_array(ar): return array.array, (ar.typecode, ar.tostring()) copy_reg.pickle(array.array, fast_pickle_array) # Check whether we have SciPy. MLlib works without it too, but if we have it, some methods, # such as _dot and _serialize_double_vector, start to support scipy.sparse matrices. try: import scipy.sparse _have_scipy = True except: # No SciPy in environment, but that's okay _have_scipy = False def _convert_to_vector(l): if isinstance(l, Vector): return l
return module.__spec__ is None else: # Backward compat for Python 2 import imp try: path = None for part in module.__name__.split('.'): if path is not None: path = [path] f, path, description = imp.find_module(part, path) if f is not None: f.close() except ImportError: return True return False """ Use copy_reg to extend global pickle definitions """ if sys.version_info < (3, 4): # pragma: no branch method_descriptor = type(str.upper) def _reduce_method_descriptor(obj): return (getattr, (obj.__objclass__, obj.__name__)) try: import copy_reg as copyreg except ImportError: import copyreg copyreg.pickle(method_descriptor, _reduce_method_descriptor)
def copy_reg_pickle(type, function): return copy_reg.pickle(type, function)
import copy_reg __author__ = 'Daniel' class State(object): def __init__(self, level=0, lives=4, points=0): self.level = level self.lives = lives self.points = points def pickle_state(instance): kwargs = instance.__dict__ return unpickle_state, (kwargs, ) def unpickle_state(kwargs): return State(**kwargs) copy_reg.pickle(State, pickle_state)
if not '!langcode!' in sentences: sentences['!langcode!'] = ('en' if language in ('default', 'en') else language) if not '!langname!' in sentences: sentences['!langname!'] = ('English' if language in ('default', 'en') else sentences['!langcode!']) write_dict(lang_file, sentences) ### important to allow safe session.flash=T(....) def lazyT_unpickle(data): return marshal.loads(data) def lazyT_pickle(data): return lazyT_unpickle, (marshal.dumps(str(data)), ) copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle) def update_all_languages(application_path): path = ospath.join(application_path, 'languages/') for language in listdir(path, regex_langfile): findT(application_path, language[:-3]) if __name__ == '__main__': import doctest doctest.testmod()
from perfkitbenchmarker import vm_util def PickleLock(lock): return UnPickleLock, (lock.locked(),) def UnPickleLock(locked, *args): lock = threading.Lock() if locked: if not lock.acquire(False): raise pickle.UnpicklingError('Cannot acquire lock') return lock copy_reg.pickle(thread.LockType, PickleLock) SUPPORTED = 'strict' NOT_EXCLUDED = 'permissive' SKIP_CHECK = 'none' FLAGS = flags.FLAGS flags.DEFINE_enum('cloud', providers.GCP, providers.VALID_CLOUDS, 'Name of the cloud to use.') flags.DEFINE_string('scratch_dir', None, 'Base name for all scratch disk directories in the VM. ' 'Upon creation, these directories will have numbers ' 'appended to them (for example /scratch0, /scratch1, etc).') flags.DEFINE_string('startup_script', None, 'Script to run right after vm boot.')
self.level = 0 self.lives = 4 def pickle_obj(obj): kwargs = obj.__dict__ return unpickle_obj, (kwargs, ) def unpickle_obj(kwargs): return GameState(**kwargs) import copy_reg copy_reg.pickle(GameState, pickle_obj) state = GameState() state.level += 2 state.lives -= 3 print state.__dict__ import pickle state_path = './game_state_v2.pickle' with open(state_path, 'wb') as f: pickle.dump(state, f) class GameState(object):