def save_function(self, obj): lasterr = None try: return self.save_global(obj) except PicklingError as e: lasterr = e try: # Check copy_reg.dispatch_table reduce = dispatch_table.get(type(obj)) if reduce: rv = reduce(obj) else: # Check for a __reduce_ex__ method, fall back to # __reduce__ reduce = getattr(obj, "__reduce_ex__", None) if reduce: rv = reduce(self.proto) else: reduce = getattr(obj, "__reduce__", None) if reduce: rv = reduce() else: raise last_exc return self.save_reduce(obj=obj, *rv) finally: if lasterr is not None: del lasterr
def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) copier = getattr(cls, "__copy__", None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) return _reconstruct(x, rv, 0)
def copy(x): cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) try: issc = issubclass(cls, type) except TypeError: issc = False if issc: return _copy_immutable(x) copier = getattr(cls, '__copy__', None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, '__reduce_ex__', None) if reductor: rv = reductor(2) else: reductor = getattr(x, '__reduce__', None) if reductor: rv = reductor() else: raise Error('un(shallow)copyable object of type %s' % cls) return _reconstruct(x, rv, 0)
def deepcopy(x): cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x) else: if issubclass(cls, type): y = _deepcopy_atomic(x) else: copier = getattr(x, "__deepcopy__", None) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() if isinstance(rv, str): y = x else: y = _reconstruct(x, *rv) return y
def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) try: issc = issubclass(cls, type) except TypeError: issc = False if issc: return _copy_immutable(x) copier = getattr(cls, '__copy__', None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, '__reduce_ex__', None) if reductor: rv = reductor(4) else: reductor = getattr(x, '__reduce__', None) if reductor: rv = reductor() else: raise Error('un(shallow)copyable object of type %s' % cls) if isinstance(rv, str): return x return _reconstruct(x, None, *rv)
def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ # print(f"DEEPCOPY\t{x}") try: if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: # cls is not a class (old Boost; see SF #502085) issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error( "un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y except Exception as e: return e
def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) # y is the unique identifier (address) of x IF X in memo; else it is NILL if y is not _nil: # x has already been copied and is stored in the memo return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): y = _deepcopy_atomic(x, memo) elif str(cls ) == "<class 'torch.Tensor'>" and 'cpu' not in x.device.type: # Using a string so we don't have to import torch # We have to ensure tensor is not already in CPU -- else .cpu() is no-op y = x.cpu() # This achieves a fast copy else: copier = getattr(x, "__deepcopy__", None) if copier is not None: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y
def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) if hasattr(x, 'cpu') and (callable(getattr(x, 'cpu')) or 'method' in str(type(getattr(x, 'cpu')))): x = x.cpu() y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier is not None: y = copier(x, memo) else: if issubclass(cls, type): y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier is not None: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y
def deepfreeze(x, memo=None, _nil=[]): """Deep freeze operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepfreeze_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: # cls is not a class (old Boost; see SF #502085) issc = 0 if issc: y = _deepfreeze_atomic(x, memo) else: copier = getattr(x, "__deepfreeze__", None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error( "un(deep)copyable object of type %s" % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) # If is its own copy, don't memoize. if y is not x: memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y
def deepcopy(x, memo=None, _nil=None): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if _nil is None: _nil = [] if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: # cls is not a class (old Boost; see SF #502085) issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, "__deepcopy__", None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(deep)copyable object of type %s" % cls) y = _reconstruct(x, rv, 1, memo) memo[d] = y _keep_alive(x, memo) # Make sure x lives at least as long as d return y
def deepcopy(x, memo=None, _nil=[]): """Deep copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y else: cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, '__deepcopy__', None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, '__reduce_ex__', None) if reductor: rv = reductor(4) else: reductor = getattr(x, '__reduce__', None) if reductor: rv = reductor() else: raise Error( 'un(deep)copyable object of type %s' % cls) if isinstance(rv, str): y = x else: y = _reconstruct(x, memo, *rv) if y is not x: memo[d] = y _keep_alive(x, memo) return y
def get_reduction(self, obj, elt): try: reduction = obj.__reduce__() except AttributeError: reduce_method = dispatch_table.get(type(obj)) if reduce_method: reduction = reduce_method(obj) else: raise PicklingError('%s item cannot be pickled' % type(obj)) if isinstance(reduction, basestring): reduction = globals().get(reduction) outtag = self.as_element(reduction, element_name=PKL_PREFIX + 'Reduction', parent=elt) obj = None
def visit_object(self, o, offset, start=None): oId = id(o) reduce = dispatch_table.get(type(o)) if reduce: rv = reduce(obj) reduce = getattr(o, '__reduce_ex__', None) if reduce: state = reduce(3) return with_reduce(self, o, state, offset, start) else: reduce = getattr(o, '__reduce__', None) if reduce: state = reduce() return with_reduce(self, o, state, offset, start) else: return without_reduce(self, o, offset, start)
def deepcopy(x, memo=None, _nil=[]): if memo is None: memo = {} d = id(x) y = memo.get(d, _nil) if y is not _nil: return y cls = type(x) copier = _deepcopy_dispatch.get(cls) if copier: y = copier(x, memo) else: try: issc = issubclass(cls, type) except TypeError: issc = 0 if issc: y = _deepcopy_atomic(x, memo) else: copier = getattr(x, '__deepcopy__', None) if copier: y = copier(memo) else: reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, '__reduce_ex__', None) if reductor: rv = reductor(2) else: reductor = getattr(x, '__reduce__', None) if reductor: rv = reductor() else: raise Error('un(deep)copyable object of type %s' % cls) y = _reconstruct(x, rv, 1, memo) if y is not x: memo[d] = y _keep_alive(x, memo) return y
def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) try: issc = issubclass(cls, type) except TypeError: # cls is not a class issc = False if issc: # treat it as a regular class: return _copy_immutable(x) copier = getattr(cls, "__copy__", None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) if isinstance(rv, str): return x return _reconstruct(x, None, *rv)
def save_function(self, obj): try: return self.save_global(obj) except PicklingError: pass # Check copy_reg.dispatch_table reduce = dispatch_table.get(type(obj)) if reduce: rv = reduce(obj) else: # Check for a __reduce_ex__ method, fall back to __reduce__ reduce = getattr(obj, "__reduce_ex__", None) if reduce: rv = reduce(self.proto) else: reduce = getattr(obj, "__reduce__", None) if reduce: rv = reduce() else: raise e return self.save_reduce(obj=obj, *rv)
def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) try: issc = issubclass(cls, type) except TypeError: # cls is not a class issc = False if issc: # treat it as a regular class: return _copy_immutable(x) copier = getattr(cls, "__copy__", None) if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: rv = reductor(2) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) return _reconstruct(x, rv, 0)
def copy(x): """Shallow copy operation on arbitrary Python objects. See the module's __doc__ string for more info. """ cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) if issubclass(cls, type): # treat it as a regular class: return _copy_immutable(x) copier = getattr(cls, "__copy__", None) if copier is not None: return copier(x) reductor = dispatch_table.get(cls) if reductor is not None: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() else: raise Error("un(shallow)copyable object of type %s" % cls) if isinstance(rv, str): return x return _reconstruct(x, None, *rv)
def copy(x): cls = type(x) copier = _copy_dispatch.get(cls) if copier: return copier(x) if issubclass(cls, type): return _copy_immutable(x) copier = getattr(cls, "__copy__", None) if copier is not None: return copier(x) reductor = dispatch_table.get(cls) if reductor is not None: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: rv = reductor(4) else: reductor = getattr(x, "__reduce__", None) if reductor: rv = reductor() if isinstance(rv, str): return x return _reconstruct(x, None, *rv)
def save(self, obj): # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid: self.save_pers(pid) return # Check the memo x = self.memo.get(id(obj)) if x: self.write(self.get(x[0])) return # Check the type dispatch table t = type(obj) f = self.dispatch.get(t) if f: f(self, obj) # Call unbound method with explicit self return # Check for a class with a custom metaclass; treat as regular class try: issc = issubclass(t, type) except TypeError: # t is not a class (old Boost; see SF #502085) issc = 0 if issc: self.save_global(obj) return # Check copy_reg.dispatch_table reduce = dispatch_table.get(t) if reduce: rv = reduce(obj) else: # New code: check for a __reduce_persist__ method, then # fall back to standard methods. reduce = getattr(obj, "__reduce_persist__", None) if reduce: rv = reduce(self) else: # Check for a __reduce_ex__ method, fall back to __reduce__ reduce = getattr(obj, "__reduce_ex__", None) if reduce: rv = reduce(self.proto) else: reduce = getattr(obj, "__reduce__", None) if reduce: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % (t.__name__, obj)) # Check for string returned by reduce(), meaning "save as global" if type(rv) is str: self.save_global(obj, rv) return # Assert that reduce() returned a tuple if type(rv) is not tuple: raise PicklingError("%s must return string or tuple" % reduce) # Assert that it returned an appropriately sized tuple l = len(rv) if not (2 <= l <= 5): raise PicklingError("Tuple returned by %s must have " "two to five elements" % reduce) # Save the reduce() output and finally memoize the object self.save_reduce(obj=obj, *rv)
def deterministic_state(self, obj, first_obj=False): v = self.memo.get(id(obj)) if v: return(v) t = type(obj) if t in (NoneType, bool, int, float, FunctionType, BuiltinFunctionType, type): return obj self.memo[id(obj)] = _MemoKey(len(self.memo), obj) if t in STRING_TYPES: return obj if t is np.ndarray and t.dtype != object: return obj if t is tuple: return (tuple,) + tuple(self.deterministic_state(x) for x in obj) if t is list: return [self.deterministic_state(x) for x in obj] if t in (set, frozenset): return (t,) + tuple(self.deterministic_state(x) for x in sorted(obj)) if t is dict: return (dict,) + tuple((k if type(k) is str else self.deterministic_state(k), self.deterministic_state(v)) for k, v in sorted(obj.items())) if issubclass(t, ImmutableInterface): if hasattr(obj, 'sid') and not obj._sid_contains_cycles: return (t, obj.sid) if not first_obj: if id(obj) in self.seen_immutables: raise _SIDGenerationRecursionError try: obj._generate_sid(self.debug, self.seen_immutables) return (t, obj.sid) except _SIDGenerationRecursionError: self.has_cycles = True self.logger.debug('{}: contains cycles of immutable objects, consider refactoring'.format(obj.name)) if obj._implements_reduce: self.logger.debug('{}: __reduce__ is implemented, not using sid_ignore'.format(obj.name)) return self.handle_reduce_value(obj, t, obj.__reduce_ex__(HIGHEST_PROTOCOL), first_obj) else: try: state = obj.__getstate__() except AttributeError: state = obj.__dict__ state = {k: v for k, v in state.items() if k not in obj.sid_ignore} return self.deterministic_state(state) if first_obj else (t, self.deterministic_state(state)) sid = getattr(obj, 'sid', None) if sid: return sid if first_obj else (t, sid) reduce = dispatch_table.get(t) if reduce: rv = reduce(obj) else: if issubclass(t, type): return obj reduce = getattr(obj, '__reduce_ex__', None) if reduce: rv = reduce(HIGHEST_PROTOCOL) else: reduce = getattr(obj, '__reduce__', None) if reduce: rv = reduce() else: raise SIDGenerationError('Cannot handle {} of type {}'.format(obj, t.__name__)) return self.handle_reduce_value(obj, t, rv, first_obj)
def deterministic_state(self, obj, first_obj=False): v = self.memo.get(id(obj)) if v: return (v) t = type(obj) if t in (NoneType, bool, int, float, FunctionType, BuiltinFunctionType, type): return obj self.memo[id(obj)] = _MemoKey(len(self.memo), obj) if t in STRING_TYPES: return obj if t is np.ndarray and t.dtype != object: return obj if t is tuple: return (tuple, ) + tuple(self.deterministic_state(x) for x in obj) if t is list: return [self.deterministic_state(x) for x in obj] if t in (set, frozenset): return (t, ) + tuple( self.deterministic_state(x) for x in sorted(obj)) if t is dict: return (dict, ) + tuple( (k if type(k) is str else self.deterministic_state(k), self.deterministic_state(v)) for k, v in sorted(obj.items())) if issubclass(t, ImmutableInterface): if hasattr(obj, 'sid') and not obj._sid_contains_cycles: return (t, obj.sid) if not first_obj: if id(obj) in self.seen_immutables: raise _SIDGenerationRecursionError try: obj._generate_sid(self.debug, self.seen_immutables) return (t, obj.sid) except _SIDGenerationRecursionError: self.has_cycles = True self.logger.debug( f'{obj.name}: contains cycles of immutable objects, consider refactoring' ) if obj._implements_reduce: self.logger.debug( f'{obj.name}: __reduce__ is implemented, not using sid_ignore' ) return self.handle_reduce_value( obj, t, obj.__reduce_ex__(HIGHEST_PROTOCOL), first_obj) else: try: state = obj.__getstate__() except AttributeError: state = obj.__dict__ state = { k: v for k, v in state.items() if k not in obj.sid_ignore } return self.deterministic_state(state) if first_obj else ( t, self.deterministic_state(state)) sid = getattr(obj, 'sid', None) if sid: return sid if first_obj else (t, sid) reduce = dispatch_table.get(t) if reduce: rv = reduce(obj) else: if issubclass(t, type): return obj reduce = getattr(obj, '__reduce_ex__', None) if reduce: rv = reduce(HIGHEST_PROTOCOL) else: reduce = getattr(obj, '__reduce__', None) if reduce: rv = reduce() else: raise SIDGenerationError( f'Cannot handle {obj} of type {t.__name__}') return self.handle_reduce_value(obj, t, rv, first_obj)
def save(self, obj, save_persistent_id=True): if self.proto >= 4: self.framer.commit_frame() # Check for persistent id (defined by a subclass) pid = self.persistent_id(obj) if pid is not None and save_persistent_id: self.save_pers(pid) return # Check if this is a Panda type that we've already saved; if so, store # a mapping to the canonical copy, so that Python's memoization system # works properly. This is needed because Python uses id(obj) for # memoization, but there may be multiple Python wrappers for the same # C++ pointer, and we don't want that to result in duplication. t = type(obj) if issubclass(t, TypedObject.__base__): canonical = self._canonical.get(obj.this) if canonical is not None: obj = canonical else: # First time we're seeing this C++ pointer; save it as the # "canonical" version. self._canonical[obj.this] = obj # Check the memo x = self.memo.get(id(obj)) if x: self.write(self.get(x[0])) return # Check the type dispatch table f = self.dispatch.get(t) if f: f(self, obj) # Call unbound method with explicit self return # Check for a class with a custom metaclass; treat as regular class try: issc = issubclass(t, type) except TypeError: # t is not a class (old Boost; see SF #502085) issc = 0 if issc: self.save_global(obj) return # Check copy_reg.dispatch_table reduce = dispatch_table.get(t) if reduce: rv = reduce(obj) else: # New code: check for a __reduce_persist__ method, then # fall back to standard methods. reduce = getattr(obj, "__reduce_persist__", None) if reduce: rv = reduce(self) else: # Check for a __reduce_ex__ method, fall back to __reduce__ reduce = getattr(obj, "__reduce_ex__", None) if reduce: rv = reduce(self.proto) else: reduce = getattr(obj, "__reduce__", None) if reduce: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % (t.__name__, obj)) # Check for string returned by reduce(), meaning "save as global" if type(rv) is str: self.save_global(obj, rv) return # Assert that reduce() returned a tuple if type(rv) is not tuple: raise PicklingError("%s must return string or tuple" % reduce) # Assert that it returned an appropriately sized tuple l = len(rv) if not (2 <= l <= 5): raise PicklingError("Tuple returned by %s must have " "two to five elements" % reduce) # Save the reduce() output and finally memoize the object self.save_reduce(obj=obj, *rv)
issc = issubclass(cls, type) except TypeError: # cls is not a class issc = False if issc: ======= if issubclass(cls, type): >>>>>>> 716b15a33aed978ded8a6bde17855cb6c6aa7f78 # treat it as a regular class: return _copy_immutable(x) copier = getattr(cls, "__copy__", None) <<<<<<< HEAD if copier: return copier(x) reductor = dispatch_table.get(cls) if reductor: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor: ======= if copier is not None: return copier(x) reductor = dispatch_table.get(cls) if reductor is not None: rv = reductor(x) else: reductor = getattr(x, "__reduce_ex__", None) if reductor is not None: