def test_factory_registered_with_copy_reg(self):
# Factories registered with copy_reg.pickle loose their __name__.
# We simply ignore those.
from six.moves import copyreg
class AnonymousFactory(object):
def __new__(cls, name):
return object.__new__(cls)
def __init__(self, name):
self._name = name
def getName(self):
return self._name
sys.modules['module1'].AnonymousFactory = AnonymousFactory
sys.modules['module1'].AnonymousFactory.__module__ = 'module1'
sys.modules['module1'].Anonymous = AnonymousFactory('Anonymous')
copyreg.pickle(AnonymousFactory,
AnonymousFactory.getName,
AnonymousFactory)
self.root['test'] = sys.modules['module1'].Anonymous
transaction.commit()
updater = self.update()
self.assertEqual('module1', self.root['test'].__class__.__module__)
self.assertEqual(
'AnonymousFactory',
self.root['test'].__class__.__name__)
renames = updater.processor.get_rules(implicit=True)
self.assertEqual({}, renames)
def __do_wrap( value ):
if isinstance( value, SafeStringWrapper ):
# Only ever wrap one-layer
return value
if callable( value ):
safe_class = CallableSafeStringWrapper
else:
safe_class = SafeStringWrapper
if isinstance( value, no_wrap_classes ):
return value
if isinstance( value, __DONT_WRAP_TYPES__ ):
return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP )
if isinstance( value, __WRAP_NO_SUBCLASS__ ):
return safe_class( value, safe_string_wrapper_function=__do_wrap )
for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__:
if isinstance( value, this_type ):
return this_type( map( __do_wrap, value ) )
for this_type in __WRAP_MAPPINGS__:
if isinstance( value, this_type ):
# Wrap both key and value
return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) )
# Create a dynamic class that joins SafeStringWrapper with the object being wrapped.
# This allows e.g. isinstance to continue to work.
try:
wrapped_class_name = value.__name__
wrapped_class = value
except:
wrapped_class_name = value.__class__.__name__
wrapped_class = value.__class__
value_mod = inspect.getmodule( value )
if value_mod:
wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name )
wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) )
do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name )
do_wrap_func = __do_wrap
global_dict = globals()
if wrapped_class_name in global_dict:
# Check to see if we have created a wrapper for this class yet, if so, reuse
wrapped_class = global_dict.get( wrapped_class_name )
do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap )
else:
try:
wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} )
except TypeError as e:
# Fail-safe for when a class cannot be dynamically subclassed.
log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e )
wrapped_class = type( wrapped_class_name, ( safe_class, ), {} )
if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ):
# Save this wrapper for reuse and pickling/copying
global_dict[ wrapped_class_name ] = wrapped_class
do_wrap_func.__name__ = do_wrap_func_name
global_dict[ do_wrap_func_name ] = do_wrap_func
def pickle_safe_object( safe_object ):
return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) )
# Set pickle and copy properties
copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func )
return wrapped_class( value, safe_string_wrapper_function=do_wrap_func )
def __do_wrap(value):
if isinstance(value, SafeStringWrapper):
# Only ever wrap one-layer
return value
if isinstance(value, collections.Callable):
safe_class = CallableSafeStringWrapper
else:
safe_class = SafeStringWrapper
if isinstance(value, no_wrap_classes):
return value
if isinstance(value, __DONT_WRAP_TYPES__):
return sanitize_lists_to_string(value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP)
if isinstance(value, __WRAP_NO_SUBCLASS__):
return safe_class(value, safe_string_wrapper_function=__do_wrap)
for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__:
if isinstance(value, this_type):
return this_type(list(map(__do_wrap, value)))
for this_type in __WRAP_MAPPINGS__:
if isinstance(value, this_type):
# Wrap both key and value
return this_type((__do_wrap(x[0]), __do_wrap(x[1])) for x in value.items())
# Create a dynamic class that joins SafeStringWrapper with the object being wrapped.
# This allows e.g. isinstance to continue to work.
try:
wrapped_class_name = value.__name__
wrapped_class = value
except Exception:
wrapped_class_name = value.__class__.__name__
wrapped_class = value.__class__
value_mod = inspect.getmodule(value)
if value_mod:
wrapped_class_name = "{}.{}".format(value_mod.__name__, wrapped_class_name)
wrapped_class_name = "SafeStringWrapper({}:{})".format(wrapped_class_name, ",".join(sorted(map(str, no_wrap_classes))))
do_wrap_func_name = "__do_wrap_%s" % (wrapped_class_name)
do_wrap_func = __do_wrap
global_dict = globals()
if wrapped_class_name in global_dict:
# Check to see if we have created a wrapper for this class yet, if so, reuse
wrapped_class = global_dict.get(wrapped_class_name)
do_wrap_func = global_dict.get(do_wrap_func_name, __do_wrap)
else:
try:
wrapped_class = type(wrapped_class_name, (safe_class, wrapped_class, ), {})
except TypeError as e:
# Fail-safe for when a class cannot be dynamically subclassed.
log.warning("Unable to create dynamic subclass for %s, %s: %s", type(value), value, e)
wrapped_class = type(wrapped_class_name, (safe_class, ), {})
if wrapped_class not in (SafeStringWrapper, CallableSafeStringWrapper):
# Save this wrapper for reuse and pickling/copying
global_dict[wrapped_class_name] = wrapped_class
do_wrap_func.__name__ = do_wrap_func_name
global_dict[do_wrap_func_name] = do_wrap_func
def pickle_safe_object(safe_object):
return (wrapped_class, (safe_object.unsanitized, do_wrap_func, ))
# Set pickle and copy properties
copy_reg.pickle(wrapped_class, pickle_safe_object, do_wrap_func)
return wrapped_class(value, safe_string_wrapper_function=do_wrap_func)
def InitMessage(descriptor, cls):
cls._decoders_by_tag = {}
cls._extensions_by_name = {}
cls._extensions_by_number = {}
if (descriptor.has_options and
descriptor.GetOptions().message_set_wire_format):
cls._decoders_by_tag[decoder.MESSAGE_SET_ITEM_TAG] = (
decoder.MessageSetItemDecoder(cls._extensions_by_number), None)
for field in descriptor.fields:
_AttachFieldHelpers(cls, field)
_AddEnumValues(descriptor, cls)
_AddInitMethod(descriptor, cls)
_AddPropertiesForFields(descriptor, cls)
_AddPropertiesForExtensions(descriptor, cls)
_AddStaticMethods(cls)
_AddMessageMethods(descriptor, cls)
_AddPrivateHelperMethods(descriptor, cls)
copyreg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
from py_stringsimjoin.utils.generic_helper import build_dict_from_table, \
find_output_attribute_indices, get_attrs_to_project, \
get_num_processes_to_launch, get_output_header_from_tables, \
get_output_row_from_tables, remove_redundant_attrs, split_table, COMP_OP_MAP
from py_stringsimjoin.utils.pickle import pickle_instance_method, \
unpickle_instance_method
from py_stringsimjoin.utils.validation import validate_attr, \
validate_comp_op, validate_key_attr, validate_input_table, \
validate_tokenizer, validate_output_attrs
# Register pickle and unpickle methods for handling instance methods.
# This is because joblib doesn't pickle instance methods, by default.
# Hence, if the sim_function supplied to apply_matcher is an instance
# method, it will result in an error. To avoid this, we register custom
# functions to pickle and unpickle instance methods.
copyreg.pickle(types.MethodType, pickle_instance_method,
unpickle_instance_method)
def apply_matcher(candset,
candset_l_key_attr,
candset_r_key_attr,
ltable,
rtable,
l_key_attr,
r_key_attr,
l_match_attr,
r_match_attr,
tokenizer,
sim_function,
threshold,
comp_op='>=',
while isinstance(other, SafeStringWrapper):
other = other.unsanitized
return coerce(self.unsanitized, other)
def __enter__(self):
return self.unsanitized.__enter__()
def __exit__(self, *args):
return self.unsanitized.__exit__(*args)
class CallableSafeStringWrapper(SafeStringWrapper):
def __call__(self, *args, **kwds):
return self.__safe_string_wrapper_function__(
self.unsanitized(*args, **kwds))
# Enable pickling/deepcopy
def pickle_SafeStringWrapper(safe_object):
args = (safe_object.unsanitized, )
cls = SafeStringWrapper
if isinstance(safe_object, CallableSafeStringWrapper):
cls = CallableSafeStringWrapper
return (cls, args)
copy_reg.pickle(SafeStringWrapper, pickle_SafeStringWrapper,
wrap_with_safe_string)
copy_reg.pickle(CallableSafeStringWrapper, pickle_SafeStringWrapper,
wrap_with_safe_string)
# THIS WORKS But CudaNdarray instances don't compare equal to one
# another, and what about __hash__ ? So the unpickled version doesn't
# equal the pickled version, and the cmodule cache is not happy with
# the situation.
def CudaNdarray_unpickler(npa):
if (config.experimental.unpickle_gpu_on_cpu and config.device == 'cpu'):
# directly return numpy array
warnings.warn(
"config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray"
)
return npa
elif cuda:
return cuda.CudaNdarray(npa)
else:
raise ImportError("Cuda not found. Cannot unpickle CudaNdarray")
copyreg.constructor(CudaNdarray_unpickler)
def CudaNdarray_pickler(cnda):
return (CudaNdarray_unpickler, (numpy.asarray(cnda), ))
# In case cuda is not imported.
if cuda is not None:
copyreg.pickle(cuda.CudaNdarray, CudaNdarray_pickler,
CudaNdarray_unpickler)
def pickle_func(func, target=None, protocol=None, b64encode=None, *args):
"""pickle_func(func, *args, target=None, protocol=None, b64encode=None)
Encode a function in such a way that when it's unpickled, the function is
reconstructed and called with the given arguments.
Note:
Compatibility between python versions is not guaranteed. Depending on
the `target` python version, the opcodes of the provided function are
transcribed to try to maintain compatibility. If an opcode is emitted
which is not supported by the target python version, a KeyError will
be raised.
Constructs that are known to be problematic:
- Python 2.6 and 2.7/3.0 use very different, incompatible opcodes for
conditional jumps (if, while, etc). Serializing those is not
always possible between python 2.6 to 2.7/3.0.
- Exception handling uses different, incompatible opcodes between
python 2 and 3.
- Python 2 and python 3 handle nested functions very differently: the
same opcode is used in a different way and leads to a crash. Avoid
nesting functions if you want to pickle across python functions.
Arguments:
func(callable): The function to serialize and call when unpickled.
args(tuple): The arguments to call the callable with.
target(int): The target python version (``26`` for python 2.6, ``27``
for python 2.7, or ``30`` for python 3.0+). Can be ``None`` in
which case the current python version is assumed.
protocol(int): The pickle protocol version to use.
b64encode(bool): Whether to base64 certain code object fields. Required
when you prepare a pickle for python 3 on python 2. If it's
``None`` it defaults to ``False`` unless pickling from python 2 to
python 3.
Returns:
bytes: The data that when unpickled calls ``func(*args)``.
Example:
>>> from pwny import *
>>> import pickle
>>> def hello(arg):
... print('Hello, %s!' % arg)
...
>>> p = pickle_func(hello, 'world')
>>> del hello
>>> pickle.loads(p)
Hello, world!
"""
def code_reduce_v2(code):
# Translate the opcodes to the target python's opcode map.
co_code, co_stacksize = translate_opcodes(
code.co_code, pwnypack.bytecode.OP_SPECS[target])
if b64encode:
# b64encode co_code and co_lnotab as they contain 8bit data.
co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code))
co_lnotab = PickleInvoke(base64.b64decode,
base64.b64encode(code.co_lnotab))
else:
co_lnotab = code.co_lnotab
if six.PY3:
# Encode unicode to bytes as python 2 doesn't support unicode identifiers.
co_names = tuple(n.encode('ascii') for n in code.co_names)
co_varnames = tuple(n.encode('ascii') for n in code.co_varnames)
co_filename = code.co_filename.encode('ascii')
co_name = code.co_name.encode('ascii')
else:
co_names = code.co_names
co_varnames = code.co_varnames
co_filename = code.co_filename
co_name = code.co_name
return types.CodeType, (code.co_argcount, code.co_nlocals,
co_stacksize, code.co_flags, co_code,
code.co_consts, co_names, co_varnames,
co_filename, co_name, code.co_firstlineno,
co_lnotab)
def code_reduce_v3(code):
# Translate the opcodes to the target python's opcode map.
co_code, co_stacksize = translate_opcodes(
code.co_code, pwnypack.bytecode.OP_SPECS[target])
if b64encode:
# b64encode co_code and co_lnotab as they contain 8bit data.
co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code))
co_lnotab = PickleInvoke(base64.b64decode,
base64.b64encode(code.co_lnotab))
else:
co_lnotab = code.co_lnotab
if six.PY2:
co_kwonlyargcount = 0
else:
co_kwonlyargcount = code.co_kwonlyargcount
return types.CodeType, (code.co_argcount, co_kwonlyargcount,
code.co_nlocals, co_stacksize, code.co_flags,
co_code, code.co_consts, code.co_names,
code.co_varnames, code.co_filename,
code.co_name, code.co_firstlineno, co_lnotab)
# Stubs to trick cPickle into pickling calls to CodeType/FunctionType.
class CodeType(object): # pragma: no cover
pass
CodeType.__module__ = 'types'
CodeType.__qualname__ = 'CodeType'
class FunctionType(object): # pragma: no cover
pass
FunctionType.__module__ = 'types'
FunctionType.__qualname__ = 'FunctionType'
protocol = get_protocol_version(target, protocol)
code = six.get_function_code(func)
old_code_reduce = copyreg.dispatch_table.pop(types.CodeType, None)
if target in (26, 27) or (target is None and six.PY2):
copyreg.pickle(types.CodeType, code_reduce_v2)
else:
if six.PY2:
if b64encode is False:
warnings.warn(
'Enabling b64encode, pickling from python 2 to 3.')
b64encode = True
copyreg.pickle(types.CodeType, code_reduce_v3)
# This has an astonishing level of evil just to convince pickle to pickle CodeType and FunctionType:
old_code_type, types.CodeType = types.CodeType, CodeType
old_function_type, types.FunctionType = types.FunctionType, FunctionType
try:
build_func = PickleInvoke(types.FunctionType, code,
PickleInvoke(globals))
return cPickle.dumps(PickleInvoke(build_func, *args), protocol)
finally:
types.CodeType = old_code_type
types.FunctionType = old_function_type
if old_code_reduce is not None:
copyreg.pickle(types.CodeType, old_code_reduce)
else:
del copyreg.dispatch_table[types.CodeType]
For backwards compatibility; please use helpers._check_for_bedtools()
"""
return helpers._check_for_bedtools(program_to_check, force_check)
# Allow Interval objects to be pickled -- required if you want to pass them
# across process boundaries
def interval_constructor(fields):
return create_interval_from_list(list(fields))
def interval_reducer(interval):
return interval_constructor, (tuple(interval.fields), )
copyreg.pickle(Interval, interval_reducer, interval_constructor)
def load_path_config(fn):
"""
You can use a config file to specify installation paths of various programs
used by pybedtools. This can be useful for testing, or using different
versions of programs.
`fn` is a config file with the following format. If an entry is blank,
then assume it's already on the path. All items must be lowercase::
[paths]
bedtools=/tools/BEDTools/bin
r=
tabix=
if real_func is None:
method = lambda: None
else:
method = getattr(classifier_cls, real_func)
if hasattr(method, 'im_self') and getattr(method, 'im_self'):
types.MethodType(method, proxy_class)
elif not hasattr(method, 'im_self'):
method = staticmethod(method)
setattr(proxy_class, proxy_func, method)
return proxy_class
copyreg.constructor(_create_proxy)
copyreg.pickle(_dynamic_proxy_class, _dynamic_proxy_class.__reduce__, _create_proxy)
class ProxyClassifierFactory(object):
def __init__(self, classifier_cls, learn_func=None, predict_func=None, weights_func=None):
learn_func, predict_func, weights_func = \
ProxyClassifierFactory.__find_funcs(classifier_cls, learn_func, predict_func, weights_func)
self.__proxyclass = _create_proxy(
classifier_cls,
learn_func,
predict_func,
weights_func
)
constructing class
sage: c == c2
calling __eq__ defined in Metaclass
True
"""
def __eq__(self, other):
print("calling __eq__ defined in Metaclass")
return (type(self) is type(other)) and (self.reduce_args
== other.reduce_args)
def __reduce__(self):
"""
Implements the pickle protocol for classes in this metaclass
(not for the instances of this class!!!)
EXAMPLES::
sage: from sage.misc.test_class_pickling import metaclass, bar
sage: c = metaclass("foo3", (object, bar,))
constructing class
sage: c.__class__.__reduce__(c)
reducing a class
(<function metaclass at ...>, ('foo3', (<type 'object'>, <class sage.misc.test_class_pickling.bar at ...>)))
"""
print("reducing a class")
return (metaclass, self.reduce_args)
from six.moves import copyreg
copyreg.pickle(Metaclass, Metaclass.__reduce__)
sage: c2 = cPickle.loads(s)
constructing class
sage: c == c2
calling __eq__ defined in Metaclass
True
"""
def __eq__(self, other):
print("calling __eq__ defined in Metaclass")
return (type(self) is type(other)) and (self.reduce_args == other.reduce_args)
def __reduce__(self):
"""
Implement the pickle protocol for classes in this metaclass
(not for the instances of this class!!!)
EXAMPLES::
sage: from sage.misc.test_class_pickling import metaclass, bar
sage: c = metaclass("foo3", (object, bar,))
constructing class
sage: c.__class__.__reduce__(c)
reducing a class
(<function metaclass at ...>, ('foo3', (<... 'object'>, <class sage.misc.test_class_pickling.bar at ...>)))
"""
print("reducing a class")
return (metaclass, self.reduce_args)
from six.moves import copyreg
copyreg.pickle(Metaclass, Metaclass.__reduce__)
# We have to put in some hooks to make instances of Clustered2DSkyKDE picklable
# because we dynamically create subclasses with different values of the 'frame'
# class variable. This gets even trickier because we need both the class and
# instance objects to be picklable.
class _Clustered2DSkyKDEMeta(type):
"""Metaclass to make dynamically created subclasses of Clustered2DSkyKDE
picklable."""
def _Clustered2DSkyKDEMeta_pickle(cls):
"""Pickle dynamically created subclasses of Clustered2DSkyKDE."""
return type, (cls.__name__, cls.__bases__, {'frame': cls.frame})
# Register function to pickle subclasses of Clustered2DSkyKDE.
copyreg.pickle(_Clustered2DSkyKDEMeta, _Clustered2DSkyKDEMeta_pickle)
def _Clustered2DSkyKDE_factory(name, frame):
"""Unpickle instances of dynamically created subclasses of
Clustered2DSkyKDE.
FIXME: In Python 3, we could make this a class method of Clustered2DSkyKDE.
Unfortunately, Python 2 is picky about pickling bound class methods."""
new_cls = type(name, (Clustered2DSkyKDE,), {'frame': frame})
return super(Clustered2DSkyKDE, Clustered2DSkyKDE).__new__(new_cls)
class Clustered2DSkyKDE(with_metaclass(_Clustered2DSkyKDEMeta, SkyKDE)):
r"""Represents a kernel-density estimate of a sky-position PDF that has
been decomposed into clusters, using a different kernel for each
cluster.
"""
if isinstance(array, GPUArray):
return array
with using_device(device):
return gpuarray.to_gpu(array, allocator=mem_alloc)
# Pickle redefinition of GPUArray. Note that pickling and unpickling of GPUArray
# do not preserve device information, i.e. the unpickled GPUArray may reside on
# a GPU different from the GPU that the original has resided on.
def _reconstruct(array, is_chainer_array):
if is_chainer_array:
return to_gpu(array)
return gpuarray.to_gpu(array)
copyreg.pickle(
GPUArray,
lambda data: (_reconstruct, (data.get(), hasattr(data.gpudata, 'device'))),
_reconstruct)
def to_gpu_async(array, stream=None):
"""Copies the given CPU array asynchronously to the current device.
Args:
array: Array to be sent to GPU. If it is :class:`~numpy.ndarray`, then
its memory must be pagelocked.
stream (~pycuda.driver.Stream): CUDA stream.
Returns:
~pycuda.gpuarray.GPUArray: Array on GPU.
If given ``array`` is already on GPU, then this function just returns
get_num_processes_to_launch, get_output_header_from_tables, \
get_output_row_from_tables, remove_redundant_attrs, split_table, COMP_OP_MAP
from py_stringsimjoin.utils.pickle import pickle_instance_method, \
unpickle_instance_method
from py_stringsimjoin.utils.validation import validate_attr, \
validate_comp_op, validate_key_attr, validate_input_table, \
validate_tokenizer, validate_output_attrs
# Register pickle and unpickle methods for handling instance methods.
# This is because joblib doesn't pickle instance methods, by default.
# Hence, if the sim_function supplied to apply_matcher is an instance
# method, it will result in an error. To avoid this, we register custom
# functions to pickle and unpickle instance methods.
copyreg.pickle(types.MethodType,
pickle_instance_method,
unpickle_instance_method)
def apply_matcher(candset,
candset_l_key_attr, candset_r_key_attr,
ltable, rtable,
l_key_attr, r_key_attr,
l_match_attr, r_match_attr,
tokenizer, sim_function,
threshold, comp_op='>=',
allow_missing=False,
l_out_attrs=None, r_out_attrs=None,
l_out_prefix='l_', r_out_prefix='r_',
out_sim_score=True, n_jobs=1, show_progress=True):
"""Find matching string pairs from the candidate set (typically produced by
def fix_proto_pickling():
"""Fix pickling issues (see b/121323638)."""
for proto_cls in _PROTO_CLASSES:
copyreg.pickle(proto_cls, _pickle_proto)
def pickle_func(func, target=None, protocol=None, b64encode=None, *args):
"""pickle_func(func, *args, target=None, protocol=None, b64encode=None)
Encode a function in such a way that when it's unpickled, the function is
reconstructed and called with the given arguments.
Note:
Compatibility between python versions is not guaranteed. Depending on
the `target` python version, the opcodes of the provided function are
transcribed to try to maintain compatibility. If an opcode is emitted
which is not supported by the target python version, a KeyError will
be raised.
Constructs that are known to be problematic:
- Python 2.6 and 2.7/3.0 use very different, incompatible opcodes for
conditional jumps (if, while, etc). Serializing those is not
always possible between python 2.6 to 2.7/3.0.
- Exception handling uses different, incompatible opcodes between
python 2 and 3.
- Python 2 and python 3 handle nested functions very differently: the
same opcode is used in a different way and leads to a crash. Avoid
nesting functions if you want to pickle across python functions.
Arguments:
func(callable): The function to serialize and call when unpickled.
args(tuple): The arguments to call the callable with.
target(int): The target python version (``26`` for python 2.6, ``27``
for python 2.7, or ``30`` for python 3.0+). Can be ``None`` in
which case the current python version is assumed.
protocol(int): The pickle protocol version to use.
b64encode(bool): Whether to base64 certain code object fields. Required
when you prepare a pickle for python 3 on python 2. If it's
``None`` it defaults to ``False`` unless pickling from python 2 to
python 3.
Returns:
bytes: The data that when unpickled calls ``func(*args)``.
Example:
>>> from pwny import *
>>> import pickle
>>> def hello(arg):
... print('Hello, %s!' % arg)
...
>>> p = pickle_func(hello, 'world')
>>> del hello
>>> pickle.loads(p)
Hello, world!
"""
def code_reduce_v2(code):
# Translate the opcodes to the target python's opcode map.
co_code, co_stacksize = translate_opcodes(code.co_code, pwnypack.bytecode.OP_SPECS[target])
if b64encode:
# b64encode co_code and co_lnotab as they contain 8bit data.
co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code))
co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab))
else:
co_lnotab = code.co_lnotab
if six.PY3:
# Encode unicode to bytes as python 2 doesn't support unicode identifiers.
co_names = tuple(n.encode('ascii') for n in code.co_names)
co_varnames = tuple(n.encode('ascii') for n in code.co_varnames)
co_filename = code.co_filename.encode('ascii')
co_name = code.co_name.encode('ascii')
else:
co_names = code.co_names
co_varnames = code.co_varnames
co_filename = code.co_filename
co_name = code.co_name
return types.CodeType, (code.co_argcount, code.co_nlocals, co_stacksize, code.co_flags,
co_code, code.co_consts, co_names, co_varnames, co_filename,
co_name, code.co_firstlineno, co_lnotab)
def code_reduce_v3(code):
# Translate the opcodes to the target python's opcode map.
co_code, co_stacksize = translate_opcodes(code.co_code, pwnypack.bytecode.OP_SPECS[target])
if b64encode:
# b64encode co_code and co_lnotab as they contain 8bit data.
co_code = PickleInvoke(base64.b64decode, base64.b64encode(co_code))
co_lnotab = PickleInvoke(base64.b64decode, base64.b64encode(code.co_lnotab))
else:
co_lnotab = code.co_lnotab
if six.PY2:
co_kwonlyargcount = 0
else:
co_kwonlyargcount = code.co_kwonlyargcount
return types.CodeType, (code.co_argcount, co_kwonlyargcount, code.co_nlocals, co_stacksize,
code.co_flags, co_code, code.co_consts, code.co_names, code.co_varnames,
code.co_filename, code.co_name, code.co_firstlineno, co_lnotab)
# Stubs to trick cPickle into pickling calls to CodeType/FunctionType.
class CodeType(object): # pragma: no cover
pass
CodeType.__module__ = 'types'
CodeType.__qualname__ = 'CodeType'
class FunctionType(object): # pragma: no cover
pass
FunctionType.__module__ = 'types'
FunctionType.__qualname__ = 'FunctionType'
protocol = get_protocol_version(target, protocol)
code = six.get_function_code(func)
old_code_reduce = copyreg.dispatch_table.pop(types.CodeType, None)
if target in (26, 27) or (target is None and six.PY2):
copyreg.pickle(types.CodeType, code_reduce_v2)
else:
if six.PY2:
if b64encode is False:
warnings.warn('Enabling b64encode, pickling from python 2 to 3.')
b64encode = True
copyreg.pickle(types.CodeType, code_reduce_v3)
# This has an astonishing level of evil just to convince pickle to pickle CodeType and FunctionType:
old_code_type, types.CodeType = types.CodeType, CodeType
old_function_type, types.FunctionType = types.FunctionType, FunctionType
try:
build_func = PickleInvoke(types.FunctionType, code, PickleInvoke(globals))
return cPickle.dumps(PickleInvoke(build_func, *args), protocol)
finally:
types.CodeType = old_code_type
types.FunctionType = old_function_type
if old_code_reduce is not None:
copyreg.pickle(types.CodeType, old_code_reduce)
else:
del copyreg.dispatch_table[types.CodeType]
im_self)
return bound
except AttributeError:
# assert im_self is not None,"No recourse: no instance to guess from."
# Attempt a common fix before bailing -- if classes have
# changed around since we pickled this method, we may still be
# able to get it by looking on the instance's current class.
unbound = getattr(im_self.__class__, im_name)
if im_self is None:
return unbound
bound = types.MethodType(get_method_function(unbound),
im_self)
return bound
copyreg.pickle(types.MethodType,
pickleMethod,
unpickleMethod)
oldModules = {}
def pickleModule(module):
'support function for copyreg to pickle module refs'
return unpickleModule, (module.__name__,)
def unpickleModule(name):
'support function for copyreg to unpickle module refs'
if name in oldModules:
name = oldModules[name]
return __import__(name, {}, {}, 'x')
def clone_layer(layer): """Clones a layer.""" # TODO: clean this up when this change is released: # https://github.com/tensorflow/tensorflow/commit/4fd10c487c7e287f99b9a1831316add453dcba04 copyreg.pickle(threading.local, lambda _: (threading.local, [])) return copy.deepcopy(layer)
from tensorflow_metadata.proto.v0 import schema_pb2
# pylint: disable=invalid-name
def serialize_schema(schema):
return (deserialize_schema, (schema.SerializeToString(), ))
def deserialize_schema(serialized):
result = schema_pb2.Schema()
result.MergeFromString(serialized)
return result
# Override default pickling.
copyreg.pickle(schema_pb2.Schema, serialize_schema)
@deprecation.deprecated(
None, 'Schema is a deprecated, use schema_utils.schema_from_feature_spec '
'to create a `Schema`')
def Schema(column_schemas):
"""Legacy constructor for a tensorflow_metadata.proto.v0.schema_pb2.Schema.
Use schema_utils.schema_from_feature_spec instead.
Args:
column_schemas: (optional) A dict from logical column names to
`ColumnSchema`s.
Returns:
# THIS WORKS But GpuArray instances don't compare equal to one
# another, and what about __hash__ ? So the unpickled version doesn't
# equal the pickled version, and the cmodule cache is not happy with
# the situation. The old back-end have this same comment and use the
# same mechanism.
def GpuArray_unpickler(npa, ctx_name):
if config.experimental.unpickle_gpu_on_cpu:
# directly return numpy array
warnings.warn(
"config.experimental.unpickle_gpu_on_cpu is set to True. "
"Unpickling GpuArray as numpy.ndarray")
return npa
elif pygpu:
ctx = get_context(ctx_name)
return pygpu.gpuarray.array(npa, copy=True, context=ctx)
else:
raise ImportError("pygpu not found. Cannot unpickle GpuArray")
copyreg.constructor(GpuArray_unpickler)
def GpuArray_pickler(cnda):
ctx_name = _name_for_ctx(cnda.context)
return (GpuArray_unpickler, (numpy.asarray(cnda), ctx_name))
# In case pygpu is not imported.
if pygpu is not None:
copyreg.pickle(pygpu.gpuarray.GpuArray,
GpuArray_pickler,
GpuArray_unpickler)
def kerasmodel_unpickler(s):
print("unpickling keras model")
modelstr, weightss = pickle.loads(s)
from .core import ConcatFixedStd
model = model_from_json(modelstr,
custom_objects={"ConcatFixedStd": ConcatFixedStd})
assert len(model.layers) == len(weightss)
for (layer, weights) in zip(model.layers, weightss):
layer.set_weights(weights)
return model
def kerasmodel_pickler(model):
print("pickling keras model")
modelstr = model.to_json()
weightss = []
for layer in model.layers:
weightss.append(layer.get_weights())
s = pickle.dumps((modelstr, weightss), -1)
return kerasmodel_unpickler, (s, )
def function_pickler(_):
raise RuntimeError("Trying to pickle theano function")
copyreg.pickle(Sequential, kerasmodel_pickler, kerasmodel_unpickler)
copyreg.pickle(theano.compile.function_module.Function, function_pickler)
"""
return self._reduction
class DynamicClasscallMetaclass(DynamicMetaclass, ClasscallMetaclass):
pass
class DynamicInheritComparisonMetaclass(DynamicMetaclass, InheritComparisonMetaclass):
pass
class DynamicInheritComparisonClasscallMetaclass(DynamicMetaclass, InheritComparisonClasscallMetaclass):
pass
# This registers the appropriate reduction methods (see Trac #5985)
from six.moves import copyreg
for M in [DynamicMetaclass,
DynamicClasscallMetaclass,
DynamicInheritComparisonMetaclass,
DynamicInheritComparisonClasscallMetaclass]:
copyreg.pickle(M, M.__reduce__)
class TestClass:
"""
A class used for checking that introspection works
"""
def bla():
"""
bla ...
"""
pass
# -*- coding: utf-8 -*- __author__ = 'Rami Al-Rfou' __email__ = '*****@*****.**' __version__ = '15.04.22' import types from six.moves import copyreg from .base import Sequence, TokenSequence from .utils import _pickle_method, _unpickle_method __all__ = ['Sequence', 'TokenSequence'] data_path = '~/' copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
# another, and what about __hash__ ? So the unpickled version doesn't
# equal the pickled version, and the cmodule cache is not happy with
# the situation. The old back-end have this same comment and use the
# same mechanism.
def GpuArray_unpickler(npa, ctx_name):
if config.experimental.unpickle_gpu_on_cpu:
# directly return numpy array
warnings.warn(
"config.experimental.unpickle_gpu_on_cpu is set to True. "
"Unpickling GpuArray as numpy.ndarray")
return npa
elif pygpu:
ctx = get_context(ctx_name)
return pygpu.gpuarray.array(npa, copy=True, context=ctx)
else:
raise ImportError("pygpu not found. Cannot unpickle GpuArray")
copyreg.constructor(GpuArray_unpickler)
def GpuArray_pickler(cnda):
ctx_name = _name_for_ctx(cnda.context)
return (GpuArray_unpickler, (np.asarray(cnda), ctx_name))
# In case pygpu is not imported.
if pygpu is not None:
copyreg.pickle(pygpu.gpuarray.GpuArray, GpuArray_pickler,
GpuArray_unpickler)
version=3,
)
# THIS WORKS But CudaNdarray instances don't compare equal to one
# another, and what about __hash__ ? So the unpickled version doesn't
# equal the pickled version, and the cmodule cache is not happy with
# the situation.
def CudaNdarray_unpickler(npa):
if config.experimental.unpickle_gpu_on_cpu:
# directly return numpy array
warnings.warn("config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray")
return npa
elif cuda:
return cuda.CudaNdarray(npa)
else:
raise ImportError("Cuda not found. Cannot unpickle CudaNdarray")
copyreg.constructor(CudaNdarray_unpickler)
def CudaNdarray_pickler(cnda):
return (CudaNdarray_unpickler, (numpy.asarray(cnda),))
# In case cuda is not imported.
if cuda is not None:
copyreg.pickle(cuda.CudaNdarray, CudaNdarray_pickler, CudaNdarray_unpickler)
def __index__( self ):
return self.unsanitized.index()
def __coerce__( self, other ):
while isinstance( other, SafeStringWrapper ):
other = other.unsanitized
return coerce( self.unsanitized, other )
def __enter__( self ):
return self.unsanitized.__enter__()
def __exit__( self, *args ):
return self.unsanitized.__exit__( *args )
class CallableSafeStringWrapper( SafeStringWrapper ):
def __call__( self, *args, **kwds ):
return self.__safe_string_wrapper_function__( self.unsanitized( *args, **kwds ) )
# Enable pickling/deepcopy
def pickle_SafeStringWrapper( safe_object ):
args = ( safe_object.unsanitized, )
cls = SafeStringWrapper
if isinstance( safe_object, CallableSafeStringWrapper ):
cls = CallableSafeStringWrapper
return ( cls, args )
copy_reg.pickle( SafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string )
copy_reg.pickle( CallableSafeStringWrapper, pickle_SafeStringWrapper, wrap_with_safe_string )
im_self)
return bound
except AttributeError:
# assert im_self is not None,"No recourse: no instance to guess from."
# Attempt a common fix before bailing -- if classes have
# changed around since we pickled this method, we may still be
# able to get it by looking on the instance's current class.
unbound = getattr(im_self.__class__, im_name)
if im_self is None:
return unbound
bound = types.MethodType(get_method_function(unbound),
im_self)
return bound
copyreg.pickle(types.MethodType,
pickleMethod,
unpickleMethod)
oldModules = {}
def pickleModule(module):
'support function for copyreg to pickle module refs'
return unpickleModule, (module.__name__,)
def unpickleModule(name):
'support function for copyreg to unpickle module refs'
if name in oldModules:
name = oldModules[name]
return __import__(name, {}, {}, 'x')
# -*- coding: utf-8 -*-
__author__ = 'Rami Al-Rfou'
__email__ = '*****@*****.**'
__version__ = '16.07.04'
import os
import sys
import types
from six.moves import copyreg
from .utils import _pickle_method, _unpickle_method
# On Windows, use %APPDATA%
if sys.platform == 'win32' and 'APPDATA' in os.environ:
data_path = os.environ['APPDATA']
else:
data_path = '~/'
data_path = os.environ.get('POLYGLOT_DATA_PATH', data_path)
data_path = os.path.abspath(os.path.expanduser(data_path))
polyglot_path = os.path.join(data_path, "polyglot_data")
copyreg.pickle(types.MethodType, _pickle_method, _unpickle_method)
"""
For backwards compatibility; please use helpers._check_for_bedtools()
"""
return helpers._check_for_bedtools(program_to_check, force_check)
# Allow Interval objects to be pickled -- required if you want to pass them
# across process boundaries
def interval_constructor(fields):
return create_interval_from_list(list(fields))
def interval_reducer(interval):
return interval_constructor, (tuple(interval.fields), )
copyreg.pickle(Interval, interval_reducer, interval_constructor)
def load_path_config(fn):
"""
You can use a config file to specify installation paths of various programs
used by pybedtools. This can be useful for testing, or using different
versions of programs.
`fn` is a config file with the following format. If an entry is blank,
then assume it's already on the path. All items must be lowercase::
[paths]
bedtools=/tools/BEDTools/bin
r=
tabix=
