def run(self, cell_model, param_values, sim=None, isolate=None):
"""Instantiate protocol"""
if isolate is None:
isolate = True
if isolate:
def _reduce_method(meth):
"""Overwrite reduce"""
return (getattr, (meth.__self__, meth.__func__.__name__))
import copy_reg
import types
copy_reg.pickle(types.MethodType, _reduce_method)
import multiprocessing
pool = multiprocessing.Pool(1, maxtasksperchild=1)
responses = pool.apply(
self._run_func,
kwds={
'cell_model': cell_model,
'param_values': param_values,
'sim': sim})
pool.terminate()
pool.join()
del pool
else:
responses = self._run_func(
cell_model=cell_model,
param_values=param_values,
sim=sim)
return responses
def __init__(self,N,ntree,maxfea,leafsize,N_proc=None):
self.N = N
self.ntree = ntree; self.maxfea = maxfea; self.leafsize = leafsize
self.N_proc = N_proc if N_proc is not None else max(1,multiprocessing.cpu_count()-1)
# fix pickling when using bound methods in classes
pickle(MethodType, _pickle_method, _pickle_method)
def test_factory_registered_with_copy_reg(self):
# Factories registered with copy_reg.pickle loose their __name__.
# We simply ignore those.
class AnonymousFactory(object):
def __new__(cls, name):
return object.__new__(cls)
def __init__(self, name):
self._name = name
def getName(self):
return self._name
sys.modules['module1'].AnonymousFactory = AnonymousFactory
sys.modules['module1'].AnonymousFactory.__module__ = 'module1'
sys.modules['module1'].Anonymous = AnonymousFactory('Anonymous')
import copy_reg
copy_reg.pickle(AnonymousFactory,
AnonymousFactory.getName,
AnonymousFactory)
self.root['test'] = sys.modules['module1'].Anonymous
transaction.commit()
updater = self.update()
self.assertEquals('module1', self.root['test'].__class__.__module__)
self.assertEquals('AnonymousFactory', self.root['test'].__class__.__name__)
renames = updater.processor.get_found_implicit_rules()
self.assertEquals({}, renames)
def install():
try:
import copy_reg
except ImportError:
import copyreg as copy_reg
copy_reg.pickle(TracebackType, pickle_traceback)
def initmessage(message_descriptor, cls):
"""constructs a new message instance (called before instance's __init__)."""
cls._extensions_by_name = {}
_addinitmethod(message_descriptor, cls)
_addmessagemethods(message_descriptor, cls)
_addpropertiesforextensions(message_descriptor, cls)
copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__()))
def _register_patched_dtype_reduce():
"""
Numpy < 1.7 has a bug when copying/pickling dtype objects with a
zero-width void type--i.e. ``np.dtype('V0')``. Specifically, although
creating a void type is perfectly valid, it crashes when instantiating
a dtype using a format string of 'V0', which is what is normally returned
by dtype.__reduce__() for these dtypes.
See https://github.com/astropy/astropy/pull/3283#issuecomment-81667461
"""
if NUMPY_LT_1_7:
import numpy as np
import copy_reg
# Originally this created an alternate constructor that fixed this
# issue, and returned that constructor from the new reduce_dtype;
# however that broke pickling since functions can't be pickled, so now
# we fix the issue directly within the custom __reduce__
def reduce_dtype(obj):
info = obj.__reduce__()
args = info[1]
if args[0] == 'V0':
args = ('V',) + args[1:]
info = (info[0], args) + info[2:]
return info
copy_reg.pickle(np.dtype, reduce_dtype)
def test_dynamic_class(self):
a = create_dynamic_class("my_dynamic_class", (object,))
copy_reg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
for proto in protocols:
s = self.dumps(a, proto)
b = self.loads(s)
self.assertEqual(a, b)
def rebuildCtype(type_, wrapper, length):
if length is not None:
type_ = type_ * length
if sys.platform == 'win32' and type_ not in copy_reg.dispatch_table:
copy_reg.pickle(type_, reduceCtype)
obj = type_.from_address(wrapper.getAddress())
obj._wrapper = wrapper
return obj
def enable(self):
if not self._enabled:
self._enabled = True
self._remap_modules({a: b for a, b in self._remap})
copy_reg.pickle(datetime_orig.date,
lambda d: (_WarpedDatetime.date,) + d.__reduce__()[1:])
copy_reg.pickle(datetime_orig.datetime,
lambda d: (_WarpedDatetime.datetime,) + d.__reduce__()[1:])
def reg_code_pickler():
"""
Called automatically when the module is loaded, this function will
ensure that the CodeType has pickle/unpickle functions registered
with copy_reg
"""
copy_reg.pickle(CodeType, _pickle_code, _unpickle_code)
def _registerEnumPicklers():
from copy_reg import constructor, pickle
def reduce_enum(e):
enum = type(e).__name__.split('.')[-1]
return ( _tuple2enum, ( enum, int(e) ) )
constructor( _tuple2enum)
for e in [ e for e in vars(ctimb).itervalues() if isEnumType(e) ]:
pickle(e, reduce_enum)
def mk_slots (*objs):
"""Make the classes that have __slots__ but not __dict__ serialisable.
Takes a number of types (new-style classes) to make serialisable.
"""
for cls in objs:
copy_reg.pickle(cls, _reduce_slots)
def __init__(self, package):
# Ensure that instance methods can be pickled.
def reduce_method(m):
return getattr, (m.__self__, m.__func__.__name__)
copy_reg.pickle(types.MethodType, reduce_method)
super(EventStorage, self).__init__('data/%s/' % package)
def _mpq_pickle_support():
"""Allow instances of gmpy.mpq to pickle."""
from gmpy import mpq
mpq_type = type(mpq(1, 10)) # gmpy doesn't appear to expose the type another way
import copy_reg
copy_reg.pickle(mpq_type, lambda q: (mpq, (q.digits(),)))
def Flags(cls):
if cls.__flags_class__ is not None:
return cls.__flags_class__
name = cls.__name__ + 'Flags'
flags_class = type(name, (cls.IntEnumFlags,), {})
flags_class.__enum_class__ = cls
cls.__flags_class__ = flags_class
copy_reg.pickle(flags_class, _int_enum_flags_pickler)
return flags_class
def main():
#Add this so we can pickle QTextDocuments
copy_reg.pickle(QTextDocument,pickle_QtD,unpickle_QtD)
app = QApplication(sys.argv)
mainWindow = MainWindow()
mainWindow.setWindowTitle("WireFrame Model")
mainWindow.show()
mainWindow.centralWidget().viewingWindow.animate()
app.exec_()
def __do_wrap( value ):
if isinstance( value, SafeStringWrapper ):
# Only ever wrap one-layer
return value
if callable( value ):
safe_class = CallableSafeStringWrapper
else:
safe_class = SafeStringWrapper
if isinstance( value, no_wrap_classes ):
return value
if isinstance( value, __DONT_WRAP_TYPES__ ):
return sanitize_lists_to_string( value, valid_characters=VALID_CHARACTERS, character_map=CHARACTER_MAP )
if isinstance( value, __WRAP_NO_SUBCLASS__ ):
return safe_class( value, safe_string_wrapper_function=__do_wrap )
for this_type in __WRAP_SEQUENCES__ + __WRAP_SETS__:
if isinstance( value, this_type ):
return this_type( map( __do_wrap, value ) )
for this_type in __WRAP_MAPPINGS__:
if isinstance( value, this_type ):
# Wrap both key and value
return this_type( map( lambda x: ( __do_wrap( x[0] ), __do_wrap( x[1] ) ), value.items() ) )
# Create a dynamic class that joins SafeStringWrapper with the object being wrapped.
# This allows e.g. isinstance to continue to work.
try:
wrapped_class_name = value.__name__
wrapped_class = value
except:
wrapped_class_name = value.__class__.__name__
wrapped_class = value.__class__
value_mod = inspect.getmodule( value )
if value_mod:
wrapped_class_name = "%s.%s" % ( value_mod.__name__, wrapped_class_name )
wrapped_class_name = "SafeStringWrapper(%s:%s)" % ( wrapped_class_name, ",".join( sorted( map( str, no_wrap_classes ) ) ) )
do_wrap_func_name = "__do_wrap_%s" % ( wrapped_class_name )
do_wrap_func = __do_wrap
global_dict = globals()
if wrapped_class_name in global_dict:
# Check to see if we have created a wrapper for this class yet, if so, reuse
wrapped_class = global_dict.get( wrapped_class_name )
do_wrap_func = global_dict.get( do_wrap_func_name, __do_wrap )
else:
try:
wrapped_class = type( wrapped_class_name, ( safe_class, wrapped_class, ), {} )
except TypeError, e:
# Fail-safe for when a class cannot be dynamically subclassed.
log.warning( "Unable to create dynamic subclass for %s, %s: %s", type( value), value, e )
wrapped_class = type( wrapped_class_name, ( safe_class, ), {} )
if wrapped_class not in ( SafeStringWrapper, CallableSafeStringWrapper ):
# Save this wrapper for reuse and pickling/copying
global_dict[ wrapped_class_name ] = wrapped_class
do_wrap_func.__name__ = do_wrap_func_name
global_dict[ do_wrap_func_name ] = do_wrap_func
def pickle_safe_object( safe_object ):
return ( wrapped_class, ( safe_object.unsanitized, do_wrap_func, ) )
# Set pickle and copy properties
copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func )
def _get_proxy_type(name):
clsname = "IPCProxy_" + name
mod = sys.modules[__name__]
if hasattr(mod, clsname):
cls = getattr(mod, clsname)
else:
cls = classobj(clsname, (IPCProxy,), {})
copy_reg.pickle(cls, _pickle_proxy, _unpickle_proxy)
setattr(mod, clsname, cls)
return cls
def int_enum_flags(name, enum_class, module=None):
assert issubclass(enum_class, IntEnum)
if enum_class.__flags_class__ is not None:
return enum_class.__flags_class__
dct = {'__module__': module or enum_class.__module__}
flags_class = type(name, (_IntEnumFlags,), dct)
flags_class.__enum_class__ = enum_class
enum_class.__flags_class__ = flags_class
copy_reg.pickle(flags_class, _int_enum_flags_pickler)
return flags_class
def _register_pickling():
crh = connected_region_handler
# Register picking functions for connected region
import copy_reg
def reduce_connected_region(obj):
return ConnectedRegion, (crh.get_shape(obj), crh.get_value(obj),
crh.get_start_row(obj), crh.get_rowptr(obj),
crh.get_colptr(obj))
copy_reg.pickle(ConnectedRegion, reduce_connected_region)
def install_cacheops():
"""
Installs cacheops by numerous monkey patches
"""
monkey_mix(Manager, ManagerMixin)
monkey_mix(QuerySet, QuerySetMixin)
QuerySet._cacheprofile = QuerySetMixin._cacheprofile
QuerySet._cloning = QuerySetMixin._cloning
# DateQuerySet existed in Django 1.7 and earlier
# Values*QuerySet existed in Django 1.8 and earlier
from django.db.models import query
for cls_name in ('ValuesQuerySet', 'ValuesListQuerySet', 'DateQuerySet'):
if hasattr(query, cls_name):
cls = getattr(query, cls_name)
monkey_mix(cls, QuerySetMixin, ['iterator'])
try:
# Use app registry in Django 1.7
from django.apps import apps
admin_used = apps.is_installed('django.contrib.admin')
get_models = apps.get_models
except ImportError:
# Introspect INSTALLED_APPS in older djangos
from django.conf import settings
admin_used = 'django.contrib.admin' in settings.INSTALLED_APPS
from django.db.models import get_models
# Install profile and signal handlers for any earlier created models
for model in get_models(include_auto_created=True):
model._default_manager._install_cacheops(model)
# Turn off caching in admin
if admin_used:
from django.contrib.admin.options import ModelAdmin
# Renamed queryset to get_queryset in Django 1.6
method_name = 'get_queryset' if hasattr(ModelAdmin, 'get_queryset') else 'queryset'
@monkey(ModelAdmin, name=method_name)
def get_queryset(self, request):
return get_queryset.original(self, request).nocache()
# Bind m2m changed handler
m2m_changed.connect(invalidate_m2m)
# Make buffers/memoryviews pickleable to serialize binary field data
if six.PY2:
import copy_reg
copy_reg.pickle(buffer, lambda b: (buffer, (bytes(b),)))
if six.PY3:
import copyreg
copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),)))
def test_copy_registry(self):
class C(object):
def __new__(cls, foo):
obj = object.__new__(cls)
obj.foo = foo
return obj
def pickle_C(obj):
return (C, (obj.foo,))
x = C(42)
self.assertRaises(TypeError, copy.copy, x)
copy_reg.pickle(C, pickle_C, C)
y = copy.copy(x)
def __init__(self, block_graph=None):
if block_graph is None:
block_graph = BlockGraph(#parents=dict(),
children=defaultdict(functools.partial(defaultdict, set)) )
# This last one is equal to children=defaultdict( lambda:defaultdict(set) ))
# Note the extra brackets at the end. TODO: Learn more FP.
# Make explicit the fact that the files, defs, and classes members
# together form the universal set of elements. There is redundancy
# here, but it adds semantic value to the data structure.
self.block_graph = block_graph
copy_reg.pickle(functools.partial, Children_ToReduce)
def fit(self, X, y=None):
self.X = X
self.y = y
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("Individual", list, fitness=creator.FitnessMax)
toolbox = base.Toolbox()
toolbox.register("attr_bool", random.randint, 0, 1)
toolbox.register("individual", tools.initRepeat, creator.Individual,
toolbox.attr_bool, n=self.individual_size)
toolbox.register("population", tools.initRepeat, list,
toolbox.individual)
toolbox.register("evaluate", self._evalFunction)
toolbox.register("mate", tools.cxTwoPoint)
toolbox.register("mutate", tools.mutFlipBit, indpb=self.mutation_prob)
toolbox.register("select", tools.selTournament, tournsize=self.tournament_size)
if self.n_jobs > 1:
copyreg.pickle(types.MethodType, _reduce_method)
pool = Pool(processes=self.n_jobs)
# self.toolbox.register("map", parmap)
toolbox.register("map", pool.map)
pop = toolbox.population(n=self.population_size)
hof = tools.HallOfFame(1)
stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("min", np.min)
stats.register("avg", np.mean)
stats.register("max", np.max)
if self.verbose:
print('--- Evolve in {0} possible combinations ---'.format(len(self.possible_params)))
pop, logbook = algorithms.eaSimple(pop, toolbox, cxpb=0.5, mutpb=0.2,
ngen=self.generations_number, stats=stats,
halloffame=hof, verbose=self.verbose)
self.best_score_ = hof[0].fitness
self.best_params_ = self._individual_to_params(BitArray(hof[0]).uint)
if self.verbose:
import json
print("Best individual is: %s\nwith fitness: %s" % (
json.dumps(self.best_params_), hof[0].fitness)
)
if self.refit:
self.best_estimator_ = clone(self.estimator)
if self.best_params_:
self.best_estimator_.set_params(**self.best_params_)
self.best_estimator_.fit(self.X, self.y)
def __init__(self, mesh, data, singularity, symmetry, indices_pack, name, derived):
self._mesh = mesh
self._data = data
self._singularity = singularity
self._symmetry = symmetry
self._indices_pack = indices_pack
self.name = name
self._derived = derived
self.indicesR, self.indicesL = indices_pack
copy_reg.pickle(derived, reductor, builder_cls_with_dict_arg )
def pickle_it(self, obj, path):
global pickle
try:
pickle
except NameError:
import cPickle as pickle
import copy_reg
copy_reg.pickle(slice, lambda s: (slice, (s.start, s.stop, s.step)))
sys.stderr.write("writing [%s]/%s\n" %
(self.package_name, os.path.basename(path)))
with open(path, 'wb') as f:
pickle.dump((pyke.version, pyke.compiler_version), f)
pickle.dump(obj, f)
def install_cacheops():
"""
Installs cacheops by numerous monkey patches
"""
monkey_mix(Manager, ManagerMixin)
monkey_mix(QuerySet, QuerySetMixin)
# Use app registry to introspect used apps
from django.apps import apps
# Install profile and signal handlers for any earlier created models
for model in apps.get_models(include_auto_created=True):
if family_has_profile(model):
if not isinstance(model._default_manager, Manager):
raise ImproperlyConfigured("Can't install cacheops for %s.%s model:"
" non-django model class or manager is used."
% (model._meta.app_label, model._meta.model_name))
model._default_manager._install_cacheops(model)
# Bind m2m changed handlers
rel_attr = 'remote_field' if django.VERSION >= (1, 9) else 'rel'
m2ms = (f for f in model._meta.get_fields(include_hidden=True) if f.many_to_many)
for m2m in m2ms:
rel = m2m if hasattr(m2m, 'through') else getattr(m2m, rel_attr, m2m)
opts = rel.through._meta
m2m_changed.connect(invalidate_m2m, sender=rel.through,
dispatch_uid=(opts.app_label, opts.model_name))
# Turn off caching in admin
if apps.is_installed('django.contrib.admin'):
from django.contrib.admin.options import ModelAdmin
@monkey(ModelAdmin)
def get_queryset(self, request):
return get_queryset.original(self, request).nocache()
# Make buffers/memoryviews pickleable to serialize binary field data
if six.PY2:
import copy_reg
copy_reg.pickle(buffer, lambda b: (buffer, (bytes(b),))) # noqa
if six.PY3:
import copyreg
copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),)))
# Fix random ordered dict keys producing different SQL for same QuerySet
if (3, 3) <= sys.version_info < (3, 6):
from django.db.models.query_utils import Q
def Q__init__(self, *args, **kwargs): # noqa
super(Q, self).__init__(children=list(args) + list(sorted(kwargs.items())))
Q.__init__ = Q__init__
def prepare():
""" Initialise the state module. """
global state_file, loaded
state_file = config.get('state')
if os.path.exists(state_name):
state_file = state_name
else:
state_file = os.path.join(plat.state_path, state_name)
# do not load any state file from a package
if config.package:
state_file = ''
# register the picklers for file and cStringIO
copy_reg.pickle(file, pickle_file)
copy_reg.pickle(cStringIO.OutputType, pickle_StringIO)
def _instance_method_pickle_support():
"""Allow instance methods to pickle."""
# CB: well, it seems to work - maybe there are cases where this
# wouldn't work?
# Alternative technique (totally different approach), but would
# only work with pickle (not cPickle):
# http://code.activestate.com/recipes/572213/
def _pickle_instance_method(mthd):
mthd_name = mthd.im_func.__name__
obj = mthd.im_self
return getattr, (obj,mthd_name)
import copy_reg, types
copy_reg.pickle(types.MethodType, _pickle_instance_method)
def _numpy_ufunc_pickle_support():
"""
Allow instances of numpy.ufunc to pickle.
"""
# Remove this when numpy.ufuncs themselves support pickling.
# Code from Robert Kern; see:
# http://news.gmane.org/find-root.php?group=gmane.comp.python.numeric.general&article=13400
from numpy import ufunc
import copy_reg
def ufunc_pickler(ufunc):
"""Return the ufunc's name"""
return ufunc.__name__
copy_reg.pickle(ufunc, ufunc_pickler)
# Attempt a last-ditch fix before giving up. If classes have changed
# around since we pickled this method, we may still be able to get it
# by looking on the instance's current class.
if im_self.__class__ is im_class:
raise
return unpickleMethod(im_name, im_self, im_self.__class__)
else:
if _PY3:
maybeClass = ()
else:
maybeClass = tuple([im_class])
bound = types.MethodType(methodFunction, im_self, *maybeClass)
return bound
copy_reg.pickle(types.MethodType, pickleMethod, unpickleMethod)
def _pickleFunction(f):
"""
Reduce, in the sense of L{pickle}'s C{object.__reduce__} special method, a
function object into its constituent parts.
@param f: The function to reduce.
@type f: L{types.FunctionType}
@return: a 2-tuple of a reference to L{_unpickleFunction} and a tuple of
its arguments, a 1-tuple of the function's fully qualified name.
@rtype: 2-tuple of C{callable, native string}
"""
if f.__name__ == '<lambda>':
##-----------------------------------------------------------------------------
def _pickle_method(method):
fct_name = method.im_func.__name__
obj = method.im_self
cl = method.im_class
## handle mangled function name
if(fct_name.startswith("__"))and(not fct_name.endswith("__")):
cl_name = cl.__name__.lstrip("_")
fct_name = "_" + cl_name + fct_name
return _unpickle_method, (fct_name, obj, cl)
##-----------------------------------------------------------------------------
def _unpickle_method(fct_name, obj, cl):
for oo in cl.__mro__:
try:
fct = oo.__dict__[fct_name]
except(KeyError):
pass
else:
break
return fct.__get__(obj, cl)
##-----------------------------------------------------------------------------
## enable instance methods pickling
cr.pickle(mt, _pickle_method, _unpickle_method)
Match = type(_pat.match(''))
del _pat
# We'll define an alias for the 'compile' function so that the repr of a
# pattern object is eval-able.
Regex = compile
# Register myself for pickling.
import copy_reg as _copy_reg
def _pickle(pattern):
return _regex.compile, pattern._pickled_data
_copy_reg.pickle(Pattern, _pickle)
if not hasattr(str, "format"):
# Strings don't have the .format method (below Python 2.6).
while True:
_start = __doc__.find(" subf")
if _start < 0:
break
_end = __doc__.find("\n", _start) + 1
while __doc__.startswith(" ", _end):
_end = __doc__.find("\n", _end) + 1
__doc__ = __doc__[:_start] + __doc__[_end:]
__all__ = [_name for _name in __all__ if not _name.startswith("subf")]
import marshal
def unpickle_code(ms):
co = marshal.loads(ms)
raise isinstance(co, types.CodeType) or AssertionError
return co
def pickle_code(co):
raise isinstance(co, types.CodeType) or AssertionError
ms = marshal.dumps(co)
return (unpickle_code, (ms, ))
copy_reg.pickle(types.CodeType, pickle_code, unpickle_code)
BUFSIZE = 8 * 1024
LOCALHOST = '127.0.0.1'
class RPCServer(SocketServer.TCPServer):
def __init__(self, addr, handlerclass=None):
if handlerclass is None:
handlerclass = RPCHandler
SocketServer.TCPServer.__init__(self, addr, handlerclass)
return
def server_bind(self):
"""Override TCPServer method, no bind() phase for connecting entity"""
pass
func_name = method.__name__
obj = method.__self__
cls = obj.__class__
return unpickle_method, (func_name, obj, cls)
def unpickle_method(func_name, obj, cls):
for cls in cls.mro():
try:
func = cls.__dict__[func_name]
except KeyError:
pass
else:
break
return func.__get__(obj, cls)
copy_reg.pickle(types.MethodType, pickle_method, unpickle_method)
# concatenates con. and disc. specs along axis 0, handling empty disc. specs
def concat_specs(c_specs, d_specs):
if len(d_specs):
return np.concatenate((c_specs, d_specs), axis=0)
else:
return c_specs
# handle Pool not being a context manager in Python < 3.4
@contextlib.contextmanager
def create_pool(*args, **kwargs):
if sys.version_info < (3, 4):
pool = multiprocessing.Pool(*args, **kwargs)
yield pool
pool.terminate()
def register_reduce(mcls):
"""Register __reduce__ as reduction function for mcls instances."""
copyreg.pickle(mcls, mcls.__reduce__)
return mcls
import types
import cumulative
import matplotlib
import time
matplotlib.use("Agg")
def _pickle_method(m):
if m.im_self is None:
return getattr, (m.im_class, m.im_func.func_name)
else:
return getattr, (m.im_self, m.im_func.func_name)
copy_reg.pickle(types.MethodType, _pickle_method)
# ---------------------
# DPGMM posterior class
# ---------------------
class DPGMMSkyPosterior(object):
"""
Dirichlet Process Gaussian Mixture model class
input parameters:
posterior_samples: posterior samples for which the density estimate needs to be calculated
dimension: the dimensionality of the problem. default = 3
from types import MethodType
from TarThread import TarThread
from mongodb_consistent_backup.Errors import Error, OperationError
from mongodb_consistent_backup.Pipeline import Task
# Allows pooled .apply_async()s to work on Class-methods:
def _reduce_method(m):
if m.im_self is None:
return getattr, (m.im_class, m.im_func.func_name)
else:
return getattr, (m.im_self, m.im_func.func_name)
pickle(MethodType, _reduce_method)
class Tar(Task):
def __init__(self, manager, config, timer, base_dir, backup_dir, **kwargs):
super(Tar, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir, **kwargs)
self.compression_method = self.config.archive.tar.compression
self.binary = "tar"
self._pool = None
self._pooled = []
def done(self, done_dir):
if done_dir in self._pooled:
logging.debug("Archiving completed for: %s" % done_dir)
self._pooled.remove(done_dir)
import copy_reg, pickle
from SpheralCompiledPackages import *
#-------------------------------------------------------------------------------
# Vector1d
#-------------------------------------------------------------------------------
def construct_Vector1d(x):
return Vector1d(x)
def reduce_Vector1d(obj):
return construct_Vector1d, (obj.x, )
copy_reg.pickle(type(Vector1d()), reduce_Vector1d, construct_Vector1d)
#-------------------------------------------------------------------------------
# Vector2d
#-------------------------------------------------------------------------------
def construct_Vector2d(x, y):
return Vector2d(x, y)
def reduce_Vector2d(obj):
return construct_Vector2d, (obj.x, obj.y)
copy_reg.pickle(type(Vector2d()), reduce_Vector2d, construct_Vector2d)
# We define _pattern_type here after all the support objects have been defined.
_pattern_type = type(_compile("", 0, {}))
# We'll define an alias for the 'compile' function so that the repr of a
# pattern object is eval-able.
Regex = compile
# Register myself for pickling.
import copy_reg as _copy_reg
def _pickle(pattern):
return _regex.compile, pattern._pickled_data
_copy_reg.pickle(_pattern_type, _pickle)
if not hasattr(str, "format"):
# Strings don't have the .format method (below Python 2.6).
while True:
_start = __doc__.find(" subf")
if _start < 0:
break
_end = __doc__.find("\n", _start) + 1
while __doc__.startswith(" ", _end):
_end = __doc__.find("\n", _end) + 1
__doc__ = __doc__[:_start] + __doc__[_end:]
__all__ = [_name for _name in __all__ if not _name.startswith("subf")]
}
""",
version=3)
# THIS WORKS But CudaNdarray instances don't compare equal to one
# another, and what about __hash__ ? So the unpickled version doesn't
# equal the pickled version, and the cmodule cache is not happy with
# the situation.
def CudaNdarray_unpickler(npa):
if config.experimental.unpickle_gpu_on_cpu:
# directly return numpy array
warnings.warn("config.experimental.unpickle_gpu_on_cpu is set to True. Unpickling CudaNdarray as numpy.ndarray")
return npa
elif cuda:
return cuda.CudaNdarray(npa)
else:
raise ImportError("Cuda not found. Cannot unpickle CudaNdarray")
copy_reg.constructor(CudaNdarray_unpickler)
def CudaNdarray_pickler(cnda):
return (CudaNdarray_unpickler, (numpy.asarray(cnda),))
# In case cuda is not imported.
if cuda is not None:
copy_reg.pickle(cuda.CudaNdarray, CudaNdarray_pickler,
CudaNdarray_unpickler)
from PYME.localization.cModels.gauss_app import *
#from scipy import weave
from PYME.Analysis._fithelpers import *
def pickleSlice(slice):
return unpickleSlice, (slice.start, slice.stop, slice.step)
def unpickleSlice(start, stop, step):
return slice(start, stop, step)
copy_reg.pickle(slice, pickleSlice, unpickleSlice)
def f_gauss2d(p, X, Y):
"""2D Gaussian model function with linear background - parameter vector [A, x0, y0, sigma, background, lin_x, lin_y]"""
A, x0, y0, s, b, b_x, b_y = p
#return A*scipy.exp(-((X-x0)**2 + (Y - y0)**2)/(2*s**2)) + b + b_x*X + b_y*Y
r = genGauss(X, Y, A, x0, y0, s, b, b_x, b_y)
r.strides = r.strides #Really dodgy hack to get around something which numpy is not doing right ....
return r
def f_gauss2dF(p, X, Y):
"""2D Gaussian model function with linear background - parameter vector [A, x0, y0, sigma, background, lin_x, lin_y] - uses fast exponential approx"""
A, x0, y0, s, b, b_x, b_y = p
#return A*scipy.exp(-((X-x0)**2 + (Y - y0)**2)/(2*s**2)) + b + b_x*X + b_y*Y
def filter(match, template=template):
return sre_parse.expand_template(template, match)
return filter
# register myself for pickling
import copy_reg
def _pickle(p):
return _compile, (p.pattern, p.flags)
copy_reg.pickle(_pattern_type, _pickle, _compile)
# --------------------------------------------------------------------
# experimental stuff (see python-dev discussions for details)
class Scanner:
def __init__(self, lexicon, flags=0):
from sre_constants import BRANCH, SUBPATTERN
self.lexicon = lexicon
# combine phrases into a compound pattern
p = []
s = sre_parse.Pattern()
s.flags = flags
for phrase, action in lexicon:
p.append(
def __getslice__(self, i, j):
return str(self)[i:j]
def __iter__(self):
for c in str(self):
yield c
def __len__(self):
return len(self.text)
def XML_unpickle(data):
return XML(marshal.loads(data))
def XML_pickle(data):
return XML_unpickle, (marshal.dumps(str(data)),)
copy_reg.pickle(XML, XML_pickle, XML_unpickle)
# ################################################################
# BEAUTIFY everything
# ################################################################
def BEAUTIFY(obj): # FIX ME, dealing with very large objects
if isinstance(obj, TAGGER):
return obj
elif isinstance(obj, list):
return UL(*[LI(BEAUTIFY(item)) for item in obj])
elif isinstance(obj, dict):
return TABLE(TBODY(*[TR(TH(XML(key)),TD(BEAUTIFY(value))) for key, value in obj.items()]))
else:
return XML(obj)
response.session_file.truncate()
finally:
self._close(response)
self.save_session_id_cookie()
return True
def _unlock(self, response):
if response and response.session_file and response.session_locked:
try:
portalocker.unlock(response.session_file)
response.session_locked = False
except: # this should never happen but happens in Windows
pass
def _close(self, response):
if response and response.session_file:
self._unlock(response)
try:
response.session_file.close()
del response.session_file
except:
pass
def pickle_session(s):
return Session, (dict(s), )
copy_reg.pickle(Session, pickle_session)
# use numpy's default reduction
return numpy_reduce(obj)
else:
# some other unknown buffer
# use numpy's default reduction
return numpy_reduce(obj)
def rebuild_array(wrapper, shape, dtype, order, strides, offset):
""" Rebuild an array with the given information.
"""
arr = np.asarray(BufferWrapperArray(wrapper, shape, dtype, order, strides, offset))
return arr
numpy_reduce = np.ndarray.__reduce__
copy_reg.pickle(np.ndarray, reduce_shared_array)
def shared_empty(shape, dtype=float, order='C'):
""" Create a shared-memory ndarray without initializing its contents.
"""
dtype = np.dtype(dtype)
if isinstance(shape, (int, long, np.integer)):
shape = (shape,)
shape = tuple(shape)
size = int(np.prod(shape))
nbytes = size * dtype.itemsize
wrapper = heap.BufferWrapper(nbytes)
strides = None
offset = 0
arr = rebuild_array(wrapper, shape, dtype, order, strides, offset)
finite_genome=finite_genome,
u=u,
v=v,
alternate_fg=alternate_fg)
#return self # comment out (returned for testing earlier)
# Allow TLSpectrum objects to be pickled.
# See http://effbot.org/librarybook/copy-reg.htm
try:
import copy_reg
except:
import copyreg
def TLSpectrum_pickler(fs):
# Collect all the info necessary to save the state of a TLSpectrum
return TLSpectrum_unpickler, (fs.data, fs.mask, fs.folded)
def TLSpectrum_unpickler(data, mask, folded):
# Use that info to recreate the TLSpectrum
return TLSpectrum(data, mask, mask_infeasible=False, data_folded=folded)
try:
copy_reg.pickle(TLSpectrum, TLSpectrum_pickler, TLSpectrum_unpickler)
except:
copyreg.pickle(TLSpectrum, TLSpectrum_pickler, TLSpectrum_unpickler)
if _coconut_sys.version_info < (2, 7):
import functools as _coconut_functools, copy_reg as _coconut_copy_reg
def _coconut_new_partial(func, args, keywords):
return _coconut_functools.partial(
func, *(args if args is not None else ()),
**(keywords if keywords is not None else {}))
_coconut_copy_reg.constructor(_coconut_new_partial)
def _coconut_reduce_partial(self):
return (_coconut_new_partial, (self.func, self.args,
self.keywords))
_coconut_copy_reg.pickle(_coconut_functools.partial,
_coconut_reduce_partial)
else:
py_chr, py_filter, py_hex, py_input, py_int, py_map, py_oct, py_open, py_print, py_range, py_str, py_zip = chr, filter, hex, input, int, map, oct, open, print, range, str, zip
class _coconut(object):
import collections, functools, imp, itertools, operator, types, copy, pickle
if _coconut_sys.version_info < (3, 3):
abc = collections
else:
import collections.abc as abc
IndexError, NameError, ValueError, map, zip, dict, filter, frozenset, getattr, hasattr, hash, isinstance, iter, len, list, min, max, next, object, range, reversed, set, slice, str, sum, super, tuple, bytearray, repr = IndexError, NameError, ValueError, map, zip, dict, filter, frozenset, getattr, hasattr, hash, isinstance, iter, len, list, min, max, next, object, range, reversed, set, slice, str, sum, super, tuple, bytearray, staticmethod(
repr)
class MatchError(Exception):
cls = method.im_class
return _unpickle_method, (func_name, obj, cls)
def _unpickle_method(func_name, obj, cls):
for cls in cls.mro():
try:
func = cls.__dict__[func_name]
except KeyError:
pass
else:
break
return func.__get__(obj, cls)
pickle(MethodType, _pickle_method, _unpickle_method)
gr.SetBatch(True) # NEEDS TO BE SET FOR MULTIPROCESSING OF plot.Draw()
Cut = namedtuple('Cut', ['name', 'cut'])
int_lumi = 41000.0 # pb #### FIXME
#int_lumi = 80000.0 # pb #### FIXME
## RICCARDO
# cuts.append(Cut('ttjetsloose', 'nbj>1'))
# cuts.append(Cut('zmmloose' , 'l1_pt>5 & l2_pt>5 & l1_q!=l2_q & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto'))
# cuts.append(Cut('zmmhighpt', 'l1_pt>15 & l2_pt>15 & l1_q!=l2_q & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto'))
# cuts.append(Cut('zmm' , 'l1_pt>10 & l2_pt>10 & l1_q!=l2_q & !l0_eid_mva_iso_loose & l0_reliso05>0.15 & l1_id_t & l2_id_t & l1_reliso05<0.2 & l2_reliso05<0.2 & abs(l1_dz)<0.2 & abs(l2_dz)<0.2 & abs(l1_dxy)<0.045 & abs(l2_dxy)<0.045 & nbj==0 & pass_e_veto & pass_m_veto'))
# cuts.append(Cut('inclusive' , 'l0_pt>30 & l1_pt>4 & l2_pt>4 & l1_q != l2_q & l0_eid_mva_iso_loose & l0_reliso05<0.15'))
# cuts.append(Cut('inclusive' , 'l0_pt>30 & l1_pt>4 & l2_pt>4 & l1_q != l2_q & l0_eid_mva_iso_loose & l0_reliso05<0.15 & l1_id_m & l2_id_m & l1_reliso05<0.2 & l2_reliso05<0.2'))
s = st
s += 'Layer {0}\n'.format(self.layer_index)
s += st
for n, model in enumerate(self):
s += '\n'
s += model.__repr__()
if n == len(self) - 1:
s += '\n'
return s
def add(self, model):
model.model_index = len(self)
self.append(model)
self.input_index_set.add(model.u1_index)
self.input_index_set.add(model.u2_index)
def delete(self, index):
self.pop(index)
for n in range(index, len(self)):
self[n].model_index = n
self.input_index_set.clear()
for model in self:
self.input_index_set.add(model.u1_index)
self.input_index_set.add(model.u2_index)
import sys
if sys.version_info.major == 2:
import copy_reg as cpr
cpr.pickle(types.MethodType, _pickle_method, _unpickle_method)
#proxy file starts first -- maybe change this
import socket
import signal
import time
from multiprocessing import Process, Manager
import copy_reg
from multiprocessing.reduction import rebuild_socket, reduce_socket
copy_reg.pickle(socket.socket, reduce_socket, rebuild_socket)
##function to handle incoming connections
def handleConnection(connections, c, addr):
print "handler started for %s" % (c)
name = c.recv(100)
print "Connection is from %s" % (name)
#add alice connection to the connection pool
print "type c = " + str(type(c))
############################ THIS PART DOES NOT WORK ############# THE SHARED MEMORY
item = connections[name] = list()
item.append(c)
connections[name] = item
print "connections : " + str(connections)
c.send('Thank you for connecting %s\n' % (name))
###### listen forever from client till client says to stop #####
while True:
message = ""
message = c.recv(1024)
if message.split()[0] == 'System':
print name + ' closed Connection'
def get_ilwdchar_class(tbl_name, col_name, namespace=globals()):
"""
Searches this module's namespace for a subclass of _ilwd.ilwdchar
whose table_name and column_name attributes match those provided.
If a matching subclass is found it is returned; otherwise a new
class is defined, added to this module's namespace, and returned.
Example:
>>> process_id = get_ilwdchar_class("process", "process_id")
>>> x = process_id(10)
>>> str(type(x))
"<class 'pycbc.ligolw.ilwd.process_process_id_class'>"
>>> str(x)
'process:process_id:10'
Retrieving and storing the class provides a convenient mechanism
for quickly constructing new ID objects.
Example:
>>> for i in range(10):
... print str(process_id(i))
...
process:process_id:0
process:process_id:1
process:process_id:2
process:process_id:3
process:process_id:4
process:process_id:5
process:process_id:6
process:process_id:7
process:process_id:8
process:process_id:9
"""
#
# if the class already exists, retrieve and return it
#
key = (str(tbl_name), str(col_name))
cls_name = "%s_%s_class" % key
assert cls_name != "get_ilwdchar_class"
try:
return namespace[cls_name]
except KeyError:
pass
#
# otherwise define a new class, and add it to the cache
#
class new_class(_ilwd.ilwdchar):
__slots__ = ()
table_name, column_name = key
index_offset = len("%s:%s:" % key)
new_class.__name__ = cls_name
namespace[cls_name] = new_class
#
# pickle support
#
copy_reg.pickle(new_class, lambda x: (ilwdchar, (unicode(x), )))
#
# return the new class
#
return new_class
import popen2
stdout, stdin = popen2.popen4(cmd, bufsize)
return stdin, stdout
__all__.append("popen4")
import copy_reg as _copy_reg
def _make_stat_result(tup, dict):
return stat_result(tup, dict)
def _pickle_stat_result(sr):
(type, args) = sr.__reduce__()
return (_make_stat_result, args)
try:
_copy_reg.pickle(stat_result, _pickle_stat_result, _make_stat_result)
except NameError: # stat_result may not exist
pass
def _make_statvfs_result(tup, dict):
return statvfs_result(tup, dict)
def _pickle_statvfs_result(sr):
(type, args) = sr.__reduce__()
return (_make_statvfs_result, args)
try:
_copy_reg.pickle(statvfs_result, _pickle_statvfs_result,
_make_statvfs_result)
except NameError: # statvfs_result may not exist
pass
"""
import sys
import array
import copy_reg
import numpy as np
__all__ = ['Vector', 'DenseVector', 'SparseVector', 'Vectors']
if sys.version_info[:2] == (2, 7):
# speed up pickling array in Python 2.7
def fast_pickle_array(ar):
return array.array, (ar.typecode, ar.tostring())
copy_reg.pickle(array.array, fast_pickle_array)
# Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,
# such as _dot and _serialize_double_vector, start to support scipy.sparse matrices.
try:
import scipy.sparse
_have_scipy = True
except:
# No SciPy in environment, but that's okay
_have_scipy = False
def _convert_to_vector(l):
if isinstance(l, Vector):
return l
return module.__spec__ is None
else:
# Backward compat for Python 2
import imp
try:
path = None
for part in module.__name__.split('.'):
if path is not None:
path = [path]
f, path, description = imp.find_module(part, path)
if f is not None:
f.close()
except ImportError:
return True
return False
""" Use copy_reg to extend global pickle definitions """
if sys.version_info < (3, 4): # pragma: no branch
method_descriptor = type(str.upper)
def _reduce_method_descriptor(obj):
return (getattr, (obj.__objclass__, obj.__name__))
try:
import copy_reg as copyreg
except ImportError:
import copyreg
copyreg.pickle(method_descriptor, _reduce_method_descriptor)
def copy_reg_pickle(type, function):
return copy_reg.pickle(type, function)
import copy_reg
__author__ = 'Daniel'
class State(object):
def __init__(self, level=0, lives=4, points=0):
self.level = level
self.lives = lives
self.points = points
def pickle_state(instance):
kwargs = instance.__dict__
return unpickle_state, (kwargs, )
def unpickle_state(kwargs):
return State(**kwargs)
copy_reg.pickle(State, pickle_state)
if not '!langcode!' in sentences:
sentences['!langcode!'] = ('en' if language in ('default',
'en') else language)
if not '!langname!' in sentences:
sentences['!langname!'] = ('English' if language in ('default', 'en')
else sentences['!langcode!'])
write_dict(lang_file, sentences)
### important to allow safe session.flash=T(....)
def lazyT_unpickle(data):
return marshal.loads(data)
def lazyT_pickle(data):
return lazyT_unpickle, (marshal.dumps(str(data)), )
copy_reg.pickle(lazyT, lazyT_pickle, lazyT_unpickle)
def update_all_languages(application_path):
path = ospath.join(application_path, 'languages/')
for language in listdir(path, regex_langfile):
findT(application_path, language[:-3])
if __name__ == '__main__':
import doctest
doctest.testmod()
from perfkitbenchmarker import vm_util
def PickleLock(lock):
return UnPickleLock, (lock.locked(),)
def UnPickleLock(locked, *args):
lock = threading.Lock()
if locked:
if not lock.acquire(False):
raise pickle.UnpicklingError('Cannot acquire lock')
return lock
copy_reg.pickle(thread.LockType, PickleLock)
SUPPORTED = 'strict'
NOT_EXCLUDED = 'permissive'
SKIP_CHECK = 'none'
FLAGS = flags.FLAGS
flags.DEFINE_enum('cloud', providers.GCP, providers.VALID_CLOUDS,
'Name of the cloud to use.')
flags.DEFINE_string('scratch_dir', None,
'Base name for all scratch disk directories in the VM. '
'Upon creation, these directories will have numbers '
'appended to them (for example /scratch0, /scratch1, etc).')
flags.DEFINE_string('startup_script', None,
'Script to run right after vm boot.')
self.level = 0
self.lives = 4
def pickle_obj(obj):
kwargs = obj.__dict__
return unpickle_obj, (kwargs, )
def unpickle_obj(kwargs):
return GameState(**kwargs)
import copy_reg
copy_reg.pickle(GameState, pickle_obj)
state = GameState()
state.level += 2
state.lives -= 3
print state.__dict__
import pickle
state_path = './game_state_v2.pickle'
with open(state_path, 'wb') as f:
pickle.dump(state, f)
class GameState(object):
