def test_factory_registered_with_copy_reg(self): # Factories registered with copy_reg.pickle loose their __name__. # We simply ignore those. from six.moves import copyreg class AnonymousFactory(object): def __new__(cls, name): return object.__new__(cls) def __init__(self, name): self._name = name def getName(self): return self._name sys.modules['module1'].AnonymousFactory = AnonymousFactory sys.modules['module1'].AnonymousFactory.__module__ = 'module1' sys.modules['module1'].Anonymous = AnonymousFactory('Anonymous') copyreg.pickle(AnonymousFactory, AnonymousFactory.getName, AnonymousFactory) self.root['test'] = sys.modules['module1'].Anonymous transaction.commit() updater = self.update() self.assertEqual('module1', self.root['test'].__class__.__module__) self.assertEqual( 'AnonymousFactory', self.root['test'].__class__.__name__) renames = updater.processor.get_rules(implicit=True) self.assertEqual({}, renames)
def __do_wrap( value ): if isinstance( value, SafeStringWrapper ): # Only ever wrap one-layer return value if callable( value ): safe_class = CallableSafeStringWrapper else: safe_class = SafeStringWrapper if isinstance( value, no_wrap_classes ): return value if isinstance( value, __DONT_WRAP_TYPES__ ): return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP ) if isinstance( value, __WRAP_NO_SUBCLASS__ ): return safe_class( value, safe_string_wrapper_function=__do_wrap ) for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: if isinstance( value, this_type ): return this_type( map( __do_wrap, value ) ) for this_type in __WRAP_MAPPINGS__: if isinstance( value, this_type ): # Wrap both key and value return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) ) # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. # This allows e.g. isinstance to continue to work. try: wrapped_class_name = value.__name__ wrapped_class = value except: wrapped_class_name = value.__class__.__name__ wrapped_class = value.__class__ value_mod = inspect.getmodule( value ) if value_mod: wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name ) wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) ) do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name ) do_wrap_func = __do_wrap global_dict = globals() if wrapped_class_name in global_dict: # Check to see if we have created a wrapper for this class yet, if so, reuse wrapped_class = global_dict.get( wrapped_class_name ) do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap ) else: try: wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} ) except TypeError as e: # Fail-safe for when a class cannot be dynamically subclassed. log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e ) wrapped_class = type( wrapped_class_name, ( safe_class, ), {} ) if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ): # Save this wrapper for reuse and pickling/copying global_dict[ wrapped_class_name ] = wrapped_class do_wrap_func.__name__ = do_wrap_func_name global_dict[ do_wrap_func_name ] = do_wrap_func def pickle_safe_object( safe_object ): return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) ) # Set pickle and copy properties copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func ) return wrapped_class( value, safe_string_wrapper_function=do_wrap_func )
def __do_wrap(value): if isinstance(value, SafeStringWrapper): # Only ever wrap one-layer return value if isinstance(value, collections.Callable): safe_class = CallableSafeStringWrapper else: safe_class = SafeStringWrapper if isinstance(value, no_wrap_classes): return value if isinstance(value, __DONT_WRAP_TYPES__): return sanitize_lists_to_string(value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP) if isinstance(value, __WRAP_NO_SUBCLASS__): return safe_class(value, safe_string_wrapper_function=__do_wrap) for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__: if isinstance(value, this_type): return this_type(list(map(__do_wrap, value))) for this_type in __WRAP_MAPPINGS__: if isinstance(value, this_type): # Wrap both key and value return this_type((__do_wrap(x[0]), __do_wrap(x[1])) for x in value.items()) # Create a dynamic class that joins SafeStringWrapper with the object being wrapped. # This allows e.g. isinstance to continue to work. try: wrapped_class_name = value.__name__ wrapped_class = value except Exception: wrapped_class_name = value.__class__.__name__ wrapped_class = value.__class__ value_mod = inspect.getmodule(value) if value_mod: wrapped_class_name = "{}.{}".format(value_mod.__name__, wrapped_class_name) wrapped_class_name = "SafeStringWrapper({}:{})".format(wrapped_class_name, ",".join(sorted(map(str, no_wrap_classes)))) do_wrap_func_name = "__do_wrap_%s" % (wrapped_class_name) do_wrap_func = __do_wrap global_dict = globals() if wrapped_class_name in global_dict: # Check to see if we have created a wrapper for this class yet, if so, reuse wrapped_class = global_dict.get(wrapped_class_name) do_wrap_func = global_dict.get(do_wrap_func_name, __do_wrap) else: try: wrapped_class = type(wrapped_class_name, (safe_class, wrapped_class, ), {}) except TypeError as e: # Fail-safe for when a class cannot be dynamically subclassed. log.warning("Unable to create dynamic subclass for %s, %s: %s", type(value), value, e) wrapped_class = type(wrapped_class_name, (safe_class, ), {}) if wrapped_class not in (SafeStringWrapper, CallableSafeStringWrapper): # Save this wrapper for reuse and pickling/copying global_dict[wrapped_class_name] = wrapped_class do_wrap_func.__name__ = do_wrap_func_name global_dict[do_wrap_func_name] = do_wrap_func def pickle_safe_object(safe_object): return (wrapped_class, (safe_object.unsanitized, do_wrap_func, )) # Set pickle and copy properties copy_reg.pickle(wrapped_class, pickle_safe_object, do_wrap_func) return wrapped_class(value, safe_string_wrapper_function=do_wrap_func)
def InitMessage(descriptor, cls): cls._decoders_by_tag = {} cls._extensions_by_name = {} cls._extensions_by_number = {} if (descriptor.has_options and descriptor.GetOptions().message_set_wire_format): cls._decoders_by_tag[decoder.MESSAGE_SET_ITEM_TAG] = ( decoder.MessageSetItemDecoder(cls._extensions_by_number), None) for field in descriptor.fields: _AttachFieldHelpers(cls, field) _AddEnumValues(descriptor, cls) _AddInitMethod(descriptor, cls) _AddPropertiesForFields(descriptor, cls) _AddPropertiesForExtensions(descriptor, cls) _AddStaticMethods(cls) _AddMessageMethods(descriptor, cls) _AddPrivateHelperMethods(descriptor, cls) copyreg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
from py_stringsimjoin.utils.generic_helper import build_dict_from_table, \ find_output_attribute_indices, get_attrs_to_project, \ get_num_processes_to_launch, get_output_header_from_tables, \ get_output_row_from_tables, remove_redundant_attrs, split_table, COMP_OP_MAP from py_stringsimjoin.utils.pickle import pickle_instance_method, \ unpickle_instance_method from py_stringsimjoin.utils.validation import validate_attr, \ validate_comp_op, validate_key_attr, validate_input_table, \ validate_tokenizer, validate_output_attrs # Register pickle and unpickle methods for handling instance methods. # This is because joblib doesn't pickle instance methods, by default. # Hence, if the sim_function supplied to apply_matcher is an instance # method, it will result in an error. To avoid this, we register custom # functions to pickle and unpickle instance methods. copyreg.pickle(types.MethodType, pickle_instance_method, unpickle_instance_method) def apply_matcher(candset, candset_l_key_attr, candset_r_key_attr, ltable, rtable, l_key_attr, r_key_attr, l_match_attr, r_match_attr, tokenizer, sim_function, threshold, comp_op='>=',
while isinstance(other, SafeStringWrapper): other = other.unsanitized return coerce(self.unsanitized, other) def __enter__(self): return self.unsanitized.__enter__() def __exit__(self, *args): return self.unsanitized.__exit__(*args) class CallableSafeStringWrapper(SafeStringWrapper): def __call__(self, *args, **kwds): return self.__safe_string_wrapper_function__( self.unsanitized(*args, **kwds)) # Enable pickling/deepcopy def pickle_SafeStringWrapper(safe_object): args = (safe_object.unsanitized, ) cls = SafeStringWrapper if isinstance(safe_object, CallableSafeStringWrapper): cls = CallableSafeStringWrapper return (cls, args) copy_reg.pickle(SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string) copy_reg.pickle(CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string)
# THIS WORKS But CudaNdarray instances don't compare equal to one # another, and what about __hash__ ? So the unpickled version doesn't # equal the pickled version, and the cmodule cache is not happy with # the situation. def CudaNdarray_unpickler(npa): if (config.experimental.unpickle_gpu_on_cpu and config.device == 'cpu'): # directly return numpy array warnings.warn( "config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray" ) return npa elif cuda: return cuda.CudaNdarray(npa) else: raise ImportError("Cuda not found. Cannot unpickle CudaNdarray") copyreg.constructor(CudaNdarray_unpickler) def CudaNdarray_pickler(cnda): return (CudaNdarray_unpickler, (numpy.asarray(cnda), )) # In case cuda is not imported. if cuda is not None: copyreg.pickle(cuda.CudaNdarray, CudaNdarray_pickler, CudaNdarray_unpickler)
def pickle_func(func, target=None, protocol=None, b64encode=None, *args): """pickle_func(func, *args, target=None, protocol=None, b64encode=None) Encode a function in such a way that when it's unpickled, the function is reconstructed and called with the given arguments. Note: Compatibility between python versions is not guaranteed. Depending on the `target` python version, the opcodes of the provided function are transcribed to try to maintain compatibility. If an opcode is emitted which is not supported by the target python version, a KeyError will be raised. Constructs that are known to be problematic: - Python 2.6 and 2.7/3.0 use very different, incompatible opcodes for conditional jumps (if, while, etc). Serializing those is not always possible between python 2.6 to 2.7/3.0. - Exception handling uses different, incompatible opcodes between python 2 and 3. - Python 2 and python 3 handle nested functions very differently: the same opcode is used in a different way and leads to a crash. Avoid nesting functions if you want to pickle across python functions. Arguments: func(callable): The function to serialize and call when unpickled. args(tuple): The arguments to call the callable with. target(int): The target python version (``26`` for python 2.6, ``27`` for python 2.7, or ``30`` for python 3.0+). Can be ``None`` in which case the current python version is assumed. protocol(int): The pickle protocol version to use. b64encode(bool): Whether to base64 certain code object fields. Required when you prepare a pickle for python 3 on python 2. If it's ``None`` it defaults to ``False`` unless pickling from python 2 to python 3. Returns: bytes: The data that when unpickled calls ``func(*args)``. Example: >>> from pwny import * >>> import pickle >>> def hello(arg): ... print('Hello, %s!' % arg) ... >>> p = pickle_func(hello, 'world') >>> del hello >>> pickle.loads(p) Hello, world! """ def code_reduce_v2(code): # Translate the opcodes to the target python's opcode map. co_code, co_stacksize = translate_opcodes( code.co_code, pwnypack.bytecode.OP_SPECS[target]) if b64encode: # b64encode co_code and co_lnotab as they contain 8bit data. co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code)) co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab)) else: co_lnotab = code.co_lnotab if six.PY3: # Encode unicode to bytes as python 2 doesn't support unicode identifiers. co_names = tuple(n.encode('ascii') for n in code.co_names) co_varnames = tuple(n.encode('ascii') for n in code.co_varnames) co_filename = code.co_filename.encode('ascii') co_name = code.co_name.encode('ascii') else: co_names = code.co_names co_varnames = code.co_varnames co_filename = code.co_filename co_name = code.co_name return types.CodeType, (code.co_argcount, code.co_nlocals, co_stacksize, code.co_flags, co_code, code.co_consts, co_names, co_varnames, co_filename, co_name, code.co_firstlineno, co_lnotab) def code_reduce_v3(code): # Translate the opcodes to the target python's opcode map. co_code, co_stacksize = translate_opcodes( code.co_code, pwnypack.bytecode.OP_SPECS[target]) if b64encode: # b64encode co_code and co_lnotab as they contain 8bit data. co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code)) co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab)) else: co_lnotab = code.co_lnotab if six.PY2: co_kwonlyargcount = 0 else: co_kwonlyargcount = code.co_kwonlyargcount return types.CodeType, (code.co_argcount, co_kwonlyargcount, code.co_nlocals, co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, co_lnotab) # Stubs to trick cPickle into pickling calls to CodeType/FunctionType. class CodeType(object): # pragma: no cover pass CodeType.__module__ = 'types' CodeType.__qualname__ = 'CodeType' class FunctionType(object): # pragma: no cover pass FunctionType.__module__ = 'types' FunctionType.__qualname__ = 'FunctionType' protocol = get_protocol_version(target, protocol) code = six.get_function_code(func) old_code_reduce = copyreg.dispatch_table.pop(types.CodeType, None) if target in (26, 27) or (target is None and six.PY2): copyreg.pickle(types.CodeType, code_reduce_v2) else: if six.PY2: if b64encode is False: warnings.warn( 'Enabling b64encode, pickling from python 2 to 3.') b64encode = True copyreg.pickle(types.CodeType, code_reduce_v3) # This has an astonishing level of evil just to convince pickle to pickle CodeType and FunctionType: old_code_type, types.CodeType = types.CodeType, CodeType old_function_type, types.FunctionType = types.FunctionType, FunctionType try: build_func = PickleInvoke(types.FunctionType, code, PickleInvoke(globals)) return cPickle.dumps(PickleInvoke(build_func, *args), protocol) finally: types.CodeType = old_code_type types.FunctionType = old_function_type if old_code_reduce is not None: copyreg.pickle(types.CodeType, old_code_reduce) else: del copyreg.dispatch_table[types.CodeType]
For backwards compatibility; please use helpers._check_for_bedtools() """ return helpers._check_for_bedtools(program_to_check, force_check) # Allow Interval objects to be pickled -- required if you want to pass them # across process boundaries def interval_constructor(fields): return create_interval_from_list(list(fields)) def interval_reducer(interval): return interval_constructor, (tuple(interval.fields), ) copyreg.pickle(Interval, interval_reducer, interval_constructor) def load_path_config(fn): """ You can use a config file to specify installation paths of various programs used by pybedtools. This can be useful for testing, or using different versions of programs. `fn` is a config file with the following format. If an entry is blank, then assume it's already on the path. All items must be lowercase:: [paths] bedtools=/tools/BEDTools/bin r= tabix=
if real_func is None: method = lambda: None else: method = getattr(classifier_cls, real_func) if hasattr(method, 'im_self') and getattr(method, 'im_self'): types.MethodType(method, proxy_class) elif not hasattr(method, 'im_self'): method = staticmethod(method) setattr(proxy_class, proxy_func, method) return proxy_class copyreg.constructor(_create_proxy) copyreg.pickle(_dynamic_proxy_class, _dynamic_proxy_class.__reduce__, _create_proxy) class ProxyClassifierFactory(object): def __init__(self, classifier_cls, learn_func=None, predict_func=None, weights_func=None): learn_func, predict_func, weights_func = \ ProxyClassifierFactory.__find_funcs(classifier_cls, learn_func, predict_func, weights_func) self.__proxyclass = _create_proxy( classifier_cls, learn_func, predict_func, weights_func )
constructing class sage: c == c2 calling __eq__ defined in Metaclass True """ def __eq__(self, other): print("calling __eq__ defined in Metaclass") return (type(self) is type(other)) and (self.reduce_args == other.reduce_args) def __reduce__(self): """ Implements the pickle protocol for classes in this metaclass (not for the instances of this class!!!) EXAMPLES:: sage: from sage.misc.test_class_pickling import metaclass, bar sage: c = metaclass("foo3", (object, bar,)) constructing class sage: c.__class__.__reduce__(c) reducing a class (<function metaclass at ...>, ('foo3', (<type 'object'>, <class sage.misc.test_class_pickling.bar at ...>))) """ print("reducing a class") return (metaclass, self.reduce_args) from six.moves import copyreg copyreg.pickle(Metaclass, Metaclass.__reduce__)
sage: c2 = cPickle.loads(s) constructing class sage: c == c2 calling __eq__ defined in Metaclass True """ def __eq__(self, other): print("calling __eq__ defined in Metaclass") return (type(self) is type(other)) and (self.reduce_args == other.reduce_args) def __reduce__(self): """ Implement the pickle protocol for classes in this metaclass (not for the instances of this class!!!) EXAMPLES:: sage: from sage.misc.test_class_pickling import metaclass, bar sage: c = metaclass("foo3", (object, bar,)) constructing class sage: c.__class__.__reduce__(c) reducing a class (<function metaclass at ...>, ('foo3', (<... 'object'>, <class sage.misc.test_class_pickling.bar at ...>))) """ print("reducing a class") return (metaclass, self.reduce_args) from six.moves import copyreg copyreg.pickle(Metaclass, Metaclass.__reduce__)
# We have to put in some hooks to make instances of Clustered2DSkyKDE picklable # because we dynamically create subclasses with different values of the 'frame' # class variable. This gets even trickier because we need both the class and # instance objects to be picklable. class _Clustered2DSkyKDEMeta(type): """Metaclass to make dynamically created subclasses of Clustered2DSkyKDE picklable.""" def _Clustered2DSkyKDEMeta_pickle(cls): """Pickle dynamically created subclasses of Clustered2DSkyKDE.""" return type, (cls.__name__, cls.__bases__, {'frame': cls.frame}) # Register function to pickle subclasses of Clustered2DSkyKDE. copyreg.pickle(_Clustered2DSkyKDEMeta, _Clustered2DSkyKDEMeta_pickle) def _Clustered2DSkyKDE_factory(name, frame): """Unpickle instances of dynamically created subclasses of Clustered2DSkyKDE. FIXME: In Python 3, we could make this a class method of Clustered2DSkyKDE. Unfortunately, Python 2 is picky about pickling bound class methods.""" new_cls = type(name, (Clustered2DSkyKDE,), {'frame': frame}) return super(Clustered2DSkyKDE, Clustered2DSkyKDE).__new__(new_cls) class Clustered2DSkyKDE(with_metaclass(_Clustered2DSkyKDEMeta, SkyKDE)): r"""Represents a kernel-density estimate of a sky-position PDF that has been decomposed into clusters, using a different kernel for each cluster.
""" if isinstance(array, GPUArray): return array with using_device(device): return gpuarray.to_gpu(array, allocator=mem_alloc) # Pickle redefinition of GPUArray. Note that pickling and unpickling of GPUArray # do not preserve device information, i.e. the unpickled GPUArray may reside on # a GPU different from the GPU that the original has resided on. def _reconstruct(array, is_chainer_array): if is_chainer_array: return to_gpu(array) return gpuarray.to_gpu(array) copyreg.pickle( GPUArray, lambda data: (_reconstruct, (data.get(), hasattr(data.gpudata, 'device'))), _reconstruct) def to_gpu_async(array, stream=None): """Copies the given CPU array asynchronously to the current device. Args: array: Array to be sent to GPU. If it is :class:`~numpy.ndarray`, then its memory must be pagelocked. stream (~pycuda.driver.Stream): CUDA stream. Returns: ~pycuda.gpuarray.GPUArray: Array on GPU. If given ``array`` is already on GPU, then this function just returns
get_num_processes_to_launch, get_output_header_from_tables, \ get_output_row_from_tables, remove_redundant_attrs, split_table, COMP_OP_MAP from py_stringsimjoin.utils.pickle import pickle_instance_method, \ unpickle_instance_method from py_stringsimjoin.utils.validation import validate_attr, \ validate_comp_op, validate_key_attr, validate_input_table, \ validate_tokenizer, validate_output_attrs # Register pickle and unpickle methods for handling instance methods. # This is because joblib doesn't pickle instance methods, by default. # Hence, if the sim_function supplied to apply_matcher is an instance # method, it will result in an error. To avoid this, we register custom # functions to pickle and unpickle instance methods. copyreg.pickle(types.MethodType, pickle_instance_method, unpickle_instance_method) def apply_matcher(candset, candset_l_key_attr, candset_r_key_attr, ltable, rtable, l_key_attr, r_key_attr, l_match_attr, r_match_attr, tokenizer, sim_function, threshold, comp_op='>=', allow_missing=False, l_out_attrs=None, r_out_attrs=None, l_out_prefix='l_', r_out_prefix='r_', out_sim_score=True, n_jobs=1, show_progress=True): """Find matching string pairs from the candidate set (typically produced by
def fix_proto_pickling(): """Fix pickling issues (see b/121323638).""" for proto_cls in _PROTO_CLASSES: copyreg.pickle(proto_cls, _pickle_proto)
def pickle_func(func, target=None, protocol=None, b64encode=None, *args): """pickle_func(func, *args, target=None, protocol=None, b64encode=None) Encode a function in such a way that when it's unpickled, the function is reconstructed and called with the given arguments. Note: Compatibility between python versions is not guaranteed. Depending on the `target` python version, the opcodes of the provided function are transcribed to try to maintain compatibility. If an opcode is emitted which is not supported by the target python version, a KeyError will be raised. Constructs that are known to be problematic: - Python 2.6 and 2.7/3.0 use very different, incompatible opcodes for conditional jumps (if, while, etc). Serializing those is not always possible between python 2.6 to 2.7/3.0. - Exception handling uses different, incompatible opcodes between python 2 and 3. - Python 2 and python 3 handle nested functions very differently: the same opcode is used in a different way and leads to a crash. Avoid nesting functions if you want to pickle across python functions. Arguments: func(callable): The function to serialize and call when unpickled. args(tuple): The arguments to call the callable with. target(int): The target python version (``26`` for python 2.6, ``27`` for python 2.7, or ``30`` for python 3.0+). Can be ``None`` in which case the current python version is assumed. protocol(int): The pickle protocol version to use. b64encode(bool): Whether to base64 certain code object fields. Required when you prepare a pickle for python 3 on python 2. If it's ``None`` it defaults to ``False`` unless pickling from python 2 to python 3. Returns: bytes: The data that when unpickled calls ``func(*args)``. Example: >>> from pwny import * >>> import pickle >>> def hello(arg): ... print('Hello, %s!' % arg) ... >>> p = pickle_func(hello, 'world') >>> del hello >>> pickle.loads(p) Hello, world! """ def code_reduce_v2(code): # Translate the opcodes to the target python's opcode map. co_code, co_stacksize = translate_opcodes(code.co_code, pwnypack.bytecode.OP_SPECS[target]) if b64encode: # b64encode co_code and co_lnotab as they contain 8bit data. co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code)) co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab)) else: co_lnotab = code.co_lnotab if six.PY3: # Encode unicode to bytes as python 2 doesn't support unicode identifiers. co_names = tuple(n.encode('ascii') for n in code.co_names) co_varnames = tuple(n.encode('ascii') for n in code.co_varnames) co_filename = code.co_filename.encode('ascii') co_name = code.co_name.encode('ascii') else: co_names = code.co_names co_varnames = code.co_varnames co_filename = code.co_filename co_name = code.co_name return types.CodeType, (code.co_argcount, code.co_nlocals, co_stacksize, code.co_flags, co_code, code.co_consts, co_names, co_varnames, co_filename, co_name, code.co_firstlineno, co_lnotab) def code_reduce_v3(code): # Translate the opcodes to the target python's opcode map. co_code, co_stacksize = translate_opcodes(code.co_code, pwnypack.bytecode.OP_SPECS[target]) if b64encode: # b64encode co_code and co_lnotab as they contain 8bit data. co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code)) co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab)) else: co_lnotab = code.co_lnotab if six.PY2: co_kwonlyargcount = 0 else: co_kwonlyargcount = code.co_kwonlyargcount return types.CodeType, (code.co_argcount, co_kwonlyargcount, code.co_nlocals, co_stacksize, code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames, code.co_filename, code.co_name, code.co_firstlineno, co_lnotab) # Stubs to trick cPickle into pickling calls to CodeType/FunctionType. class CodeType(object): # pragma: no cover pass CodeType.__module__ = 'types' CodeType.__qualname__ = 'CodeType' class FunctionType(object): # pragma: no cover pass FunctionType.__module__ = 'types' FunctionType.__qualname__ = 'FunctionType' protocol = get_protocol_version(target, protocol) code = six.get_function_code(func) old_code_reduce = copyreg.dispatch_table.pop(types.CodeType, None) if target in (26, 27) or (target is None and six.PY2): copyreg.pickle(types.CodeType, code_reduce_v2) else: if six.PY2: if b64encode is False: warnings.warn('Enabling b64encode, pickling from python 2 to 3.') b64encode = True copyreg.pickle(types.CodeType, code_reduce_v3) # This has an astonishing level of evil just to convince pickle to pickle CodeType and FunctionType: old_code_type, types.CodeType = types.CodeType, CodeType old_function_type, types.FunctionType = types.FunctionType, FunctionType try: build_func = PickleInvoke(types.FunctionType, code, PickleInvoke(globals)) return cPickle.dumps(PickleInvoke(build_func, *args), protocol) finally: types.CodeType = old_code_type types.FunctionType = old_function_type if old_code_reduce is not None: copyreg.pickle(types.CodeType, old_code_reduce) else: del copyreg.dispatch_table[types.CodeType]
im_self) return bound except AttributeError: # assert im_self is not None,"No recourse: no instance to guess from." # Attempt a common fix before bailing -- if classes have # changed around since we pickled this method, we may still be # able to get it by looking on the instance's current class. unbound = getattr(im_self.__class__, im_name) if im_self is None: return unbound bound = types.MethodType(get_method_function(unbound), im_self) return bound copyreg.pickle(types.MethodType, pickleMethod, unpickleMethod) oldModules = {} def pickleModule(module): 'support function for copyreg to pickle module refs' return unpickleModule, (module.__name__,) def unpickleModule(name): 'support function for copyreg to unpickle module refs' if name in oldModules: name = oldModules[name] return __import__(name, {}, {}, 'x')
def clone_layer(layer): """Clones a layer.""" # TODO: clean this up when this change is released: # https://github.com/tensorflow/tensorflow/commit/4fd10c487c7e287f99b9a1831316add453dcba04 copyreg.pickle(threading.local, lambda _: (threading.local, [])) return copy.deepcopy(layer)
from tensorflow_metadata.proto.v0 import schema_pb2 # pylint: disable=invalid-name def serialize_schema(schema): return (deserialize_schema, (schema.SerializeToString(), )) def deserialize_schema(serialized): result = schema_pb2.Schema() result.MergeFromString(serialized) return result # Override default pickling. copyreg.pickle(schema_pb2.Schema, serialize_schema) @deprecation.deprecated( None, 'Schema is a deprecated, use schema_utils.schema_from_feature_spec ' 'to create a `Schema`') def Schema(column_schemas): """Legacy constructor for a tensorflow_metadata.proto.v0.schema_pb2.Schema. Use schema_utils.schema_from_feature_spec instead. Args: column_schemas: (optional) A dict from logical column names to `ColumnSchema`s. Returns:
# THIS WORKS But GpuArray instances don't compare equal to one # another, and what about __hash__ ? So the unpickled version doesn't # equal the pickled version, and the cmodule cache is not happy with # the situation. The old back-end have this same comment and use the # same mechanism. def GpuArray_unpickler(npa, ctx_name): if config.experimental.unpickle_gpu_on_cpu: # directly return numpy array warnings.warn( "config.experimental.unpickle_gpu_on_cpu is set to True. " "Unpickling GpuArray as numpy.ndarray") return npa elif pygpu: ctx = get_context(ctx_name) return pygpu.gpuarray.array(npa, copy=True, context=ctx) else: raise ImportError("pygpu not found. Cannot unpickle GpuArray") copyreg.constructor(GpuArray_unpickler) def GpuArray_pickler(cnda): ctx_name = _name_for_ctx(cnda.context) return (GpuArray_unpickler, (numpy.asarray(cnda), ctx_name)) # In case pygpu is not imported. if pygpu is not None: copyreg.pickle(pygpu.gpuarray.GpuArray, GpuArray_pickler, GpuArray_unpickler)
def kerasmodel_unpickler(s): print("unpickling keras model") modelstr, weightss = pickle.loads(s) from .core import ConcatFixedStd model = model_from_json(modelstr, custom_objects={"ConcatFixedStd": ConcatFixedStd}) assert len(model.layers) == len(weightss) for (layer, weights) in zip(model.layers, weightss): layer.set_weights(weights) return model def kerasmodel_pickler(model): print("pickling keras model") modelstr = model.to_json() weightss = [] for layer in model.layers: weightss.append(layer.get_weights()) s = pickle.dumps((modelstr, weightss), -1) return kerasmodel_unpickler, (s, ) def function_pickler(_): raise RuntimeError("Trying to pickle theano function") copyreg.pickle(Sequential, kerasmodel_pickler, kerasmodel_unpickler) copyreg.pickle(theano.compile.function_module.Function, function_pickler)
""" return self._reduction class DynamicClasscallMetaclass(DynamicMetaclass, ClasscallMetaclass): pass class DynamicInheritComparisonMetaclass(DynamicMetaclass, InheritComparisonMetaclass): pass class DynamicInheritComparisonClasscallMetaclass(DynamicMetaclass, InheritComparisonClasscallMetaclass): pass # This registers the appropriate reduction methods (see Trac #5985) from six.moves import copyreg for M in [DynamicMetaclass, DynamicClasscallMetaclass, DynamicInheritComparisonMetaclass, DynamicInheritComparisonClasscallMetaclass]: copyreg.pickle(M, M.__reduce__) class TestClass: """ A class used for checking that introspection works """ def bla(): """ bla ... """ pass
# -*- coding: utf-8 -*- __author__ = 'Rami Al-Rfou' __email__ = '*****@*****.**' __version__ = '15.04.22' import types from six.moves import copyreg from .base import Sequence, TokenSequence from .utils import _pickle_method, _unpickle_method __all__ = ['Sequence', 'TokenSequence'] data_path = '~/' copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
# another, and what about __hash__ ? So the unpickled version doesn't # equal the pickled version, and the cmodule cache is not happy with # the situation. The old back-end have this same comment and use the # same mechanism. def GpuArray_unpickler(npa, ctx_name): if config.experimental.unpickle_gpu_on_cpu: # directly return numpy array warnings.warn( "config.experimental.unpickle_gpu_on_cpu is set to True. " "Unpickling GpuArray as numpy.ndarray") return npa elif pygpu: ctx = get_context(ctx_name) return pygpu.gpuarray.array(npa, copy=True, context=ctx) else: raise ImportError("pygpu not found. Cannot unpickle GpuArray") copyreg.constructor(GpuArray_unpickler) def GpuArray_pickler(cnda): ctx_name = _name_for_ctx(cnda.context) return (GpuArray_unpickler, (np.asarray(cnda), ctx_name)) # In case pygpu is not imported. if pygpu is not None: copyreg.pickle(pygpu.gpuarray.GpuArray, GpuArray_pickler, GpuArray_unpickler)
version=3, ) # THIS WORKS But CudaNdarray instances don't compare equal to one # another, and what about __hash__ ? So the unpickled version doesn't # equal the pickled version, and the cmodule cache is not happy with # the situation. def CudaNdarray_unpickler(npa): if config.experimental.unpickle_gpu_on_cpu: # directly return numpy array warnings.warn("config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray") return npa elif cuda: return cuda.CudaNdarray(npa) else: raise ImportError("Cuda not found. Cannot unpickle CudaNdarray") copyreg.constructor(CudaNdarray_unpickler) def CudaNdarray_pickler(cnda): return (CudaNdarray_unpickler, (numpy.asarray(cnda),)) # In case cuda is not imported. if cuda is not None: copyreg.pickle(cuda.CudaNdarray, CudaNdarray_pickler, CudaNdarray_unpickler)
def __index__( self ): return self.unsanitized.index() def __coerce__( self, other ): while isinstance( other, SafeStringWrapper ): other = other.unsanitized return coerce( self.unsanitized, other ) def __enter__( self ): return self.unsanitized.__enter__() def __exit__( self, *args ): return self.unsanitized.__exit__( *args ) class CallableSafeStringWrapper( SafeStringWrapper ): def __call__( self, *args, **kwds ): return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) ) # Enable pickling/deepcopy def pickle_SafeStringWrapper( safe_object ): args = ( safe_object.unsanitized, ) cls = SafeStringWrapper if isinstance( safe_object, CallableSafeStringWrapper ): cls = CallableSafeStringWrapper return ( cls, args ) copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string ) copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string )
# -*- coding: utf-8 -*- __author__ = 'Rami Al-Rfou' __email__ = '*****@*****.**' __version__ = '16.07.04' import os import sys import types from six.moves import copyreg from .utils import _pickle_method, _unpickle_method # On Windows, use %APPDATA% if sys.platform == 'win32' and 'APPDATA' in os.environ: data_path = os.environ['APPDATA'] else: data_path = '~/' data_path = os.environ.get('POLYGLOT_DATA_PATH', data_path) data_path = os.path.abspath(os.path.expanduser(data_path)) polyglot_path = os.path.join(data_path, "polyglot_data") copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
""" For backwards compatibility; please use helpers._check_for_bedtools() """ return helpers._check_for_bedtools(program_to_check, force_check) # Allow Interval objects to be pickled -- required if you want to pass them # across process boundaries def interval_constructor(fields): return create_interval_from_list(list(fields)) def interval_reducer(interval): return interval_constructor, (tuple(interval.fields), ) copyreg.pickle(Interval, interval_reducer, interval_constructor) def load_path_config(fn): """ You can use a config file to specify installation paths of various programs used by pybedtools. This can be useful for testing, or using different versions of programs. `fn` is a config file with the following format. If an entry is blank, then assume it's already on the path. All items must be lowercase:: [paths] bedtools=/tools/BEDTools/bin r= tabix=