def duplicates(self, params=None):
if params:
params = sorted(params)
keyfunc = lambda ds: (
ds.experiment, list(
dumps(ds.p[k], protocol=-1) for k in params))
groups = itertools.groupby(sorted(
(ds for ds in self.datasets
if set(ds.p.dict.keys()) >= set(params)),
key=keyfunc),
key=keyfunc)
else:
keyfunc = lambda ds: (ds.experiment,
list(
sorted((k, dumps(v, protocol=-1))
for k, v in ds.p.dict.items())))
groups = itertools.groupby(sorted(self.datasets, key=keyfunc),
key=keyfunc)
def get_duplicates(groups):
for k, v in groups:
datasets = list(v)
if len(datasets) > 1:
p = datasets[0].p.dict
info = {
'experiment': datasets[0].experiment,
'params':
p if not params else {k: p[k]
for k in params}
}
yield [info, datasets]
class DuplicatesList(list):
def __str__(self):
formatter = lambda i, ds: ('experiment: ' + i[
'experiment'] + '\n' + 'params: ' + '\n'.join(
textwrap.wrap(
', '.join('{}={}'.format(k, v)
for k, v in sorted(i['params'].items())),
initial_indent='',
subsequent_indent=' ' * len('experiment: '),
width=100)) + '\n' + 'count: ' + str(len(ds))
+ '\n')
return '\n'.join(formatter(i, ds) for i, ds in self)
def delete_old(self):
for i, datasets in self:
for ds in datasets[:-1]:
ds.delete()
__repr__ = __str__
return DuplicatesList(get_duplicates(groups))
def set(self, key, value):
self._enforce_limits(value)
value = dumps(value)
if not key in self._keylist:
self._new_key(key, len(value))
with self._dbm_file(True) as dbm:
dbm[key] = value
def test_pickle_by_solving(discretization):
d = discretization
d2 = loads(dumps(d))
d.disable_caching()
d2.disable_caching()
for mu in d.parameter_space.sample_randomly(3, seed=234):
assert np.all(d.solve(mu).almost_equal(d2.solve(mu)))
def call(method, *args, **kwargs):
"""Execute method on all MPI ranks.
Assuming :func:`event_loop` is running on all MPI ranks
(except rank 0), this will execute `method` on all
ranks (including rank 0) with positional arguments
`args` and keyword arguments `kwargs`.
Parameters
----------
method
The function to execute on all ranks (must be picklable).
args
The positional arguments for `method`.
kwargs
The keyword arguments for `method`.
Returns
-------
The return value of `method` on rank 0.
"""
assert rank0
if finished:
return
comm.bcast(dumps((method, args, kwargs)), root=0)
return method(*args, **kwargs)
def test_pickle_by_solving(model):
m = model
m2 = loads(dumps(m))
m.disable_caching()
m2.disable_caching()
for mu in m.parameter_space.sample_randomly(3, seed=234):
assert np.all(almost_equal(m.solve(mu), m2.solve(mu)))
def test_pickle_by_evaluation(function):
f = function
f2 = loads(dumps(f))
mus = parameters_of_type(f.parameter_type, 47)
for arg in function_argument(f, 10, 42):
mu = next(mus)
assert np.all(f.evaluate(arg, mu) == f2.evaluate(arg, mu))
def call(method, *args, **kwargs):
"""Execute method on all MPI ranks.
Assuming :func:`event_loop` is running on all MPI ranks
(except rank 0), this will execute `method` on all
ranks (including rank 0) with sequential arguments
`args` and keyword arguments `kwargs`.
Parameters
----------
method
The function to execute on all ranks. Must be picklable.
args
The sequential arguments for `method`.
kwargs
The keyword arguments for `method`.
Returns
-------
The return value of `method` on rank 0.
"""
assert rank0
if finished:
return
comm.bcast(dumps((method, args, kwargs)), root=0)
return method(*args, **kwargs)
def build_cache_key(obj):
def transform_obj(obj):
t = type(obj)
if t in (NoneType, bool, int, float, str, bytes):
return obj
elif t is np.ndarray:
if obj.dtype == object:
raise CacheKeyGenerationError(
'Cannot generate cache key for provided arguments')
return obj
elif t in (list, tuple):
return tuple(transform_obj(o) for o in obj)
elif t in (set, frozenset):
return tuple(transform_obj(o) for o in sorted(obj))
elif t in (Mu, Parameters):
return tuple(
(transform_obj(k), transform_obj(v)) for k, v in obj.items())
elif t in (dict, Mu, Parameters):
return tuple((transform_obj(k), transform_obj(v))
for k, v in sorted(obj.items()))
elif isinstance(obj, Number):
# handle numpy number objects
return obj
else:
raise CacheKeyGenerationError(
'Cannot generate cache key for provided arguments')
obj = transform_obj(obj)
key = hashlib.sha256(dumps(obj, protocol=-1)).hexdigest()
return key
def test_pickle_by_evaluation(function):
f = function
f2 = loads(dumps(f))
mus = parameters_of_type(f.parameter_type, 47)
for arg in function_argument(f, 10, 42):
mu = next(mus)
assert np.all(f.evaluate(arg, mu) == f2.evaluate(arg, mu))
def quit():
"""Exit the event loop on all MPI ranks.
This will cause :func:`event_loop` to terminate on all
MPI ranks.
"""
global finished
comm.bcast(dumps(('QUIT', None, None)))
finished = True
def quit():
"""Exit the event loop on all MPI ranks.
This will cause :func:`event_loop` to terminate on all
MPI ranks.
"""
global finished
comm.bcast(dumps(('QUIT', None, None)))
finished = True
def assert_picklable_without_dumps_function(o):
def dumps_function_raise(function):
raise PicklingError('Cannot pickle function {}'.format(function))
old_code = dumps_function.__code__
dumps_function.__code__ = dumps_function_raise.__code__
try:
s = dumps(o)
o2 = loads(s)
assert_is_equal(o, o2)
finally:
dumps_function.__code__ = old_code
def assert_picklable_without_dumps_function(o):
def dumps_function_raise(function):
raise PicklingError("Cannot pickle function {}".format(function))
old_code = dumps_function.func_code
dumps_function.func_code = dumps_function_raise.func_code
try:
s = dumps(o)
o2 = loads(s)
assert_is_equal(o, o2)
finally:
dumps_function.func_code = old_code
def __getstate__(self):
s = self.__dict__.copy()
try:
pickled_mapping = dumps(self._mapping)
picklable = True
except PicklingError:
self.logger.warn('Mapping not picklable, trying pymor.core.pickle.dumps_function.')
pickled_mapping = dumps_function(self._mapping)
picklable = False
s['_mapping'] = pickled_mapping
s['_picklable'] = picklable
return s
def assert_picklable_without_dumps_function(o):
def dumps_function_raise(function):
raise PicklingError(f'Cannot pickle function {function}')
old_code = dumps_function.__code__
dumps_function.__code__ = dumps_function_raise.__code__
try:
s = dumps(o)
o2 = loads(s)
assert_is_equal(o, o2)
finally:
dumps_function.__code__ = old_code
def __getstate__(self):
s = self.__dict__.copy()
try:
pickled_mapping = dumps(self._mapping)
picklable = True
except PicklingError:
self.logger.warn(
'Mapping not picklable, trying pymor.core.pickle.dumps_function.'
)
pickled_mapping = dumps_function(self._mapping)
picklable = False
s['_mapping'] = pickled_mapping
s['_picklable'] = picklable
return s
def __call__(self, im_self, *args, **kwargs):
"""Via the magic that is partial functions returned from __get__, im_self is the instance object of the class
we're decorating a method of and [kw]args are the actual parameters to the decorated method"""
region = cache_regions[im_self.cache_region]
if not region.enabled:
return self.decorated_function(im_self, *args, **kwargs)
key = (self.decorated_function.__name__, getattr(im_self, 'sid', im_self.uid),
tuple(getattr(x, 'sid', x) for x in args),
tuple((k, getattr(v, 'sid', v)) for k, v in sorted(kwargs.iteritems())),
defaults_sid())
key = dumps(key)
found, value = region.get(key)
if found:
return value
else:
im_self.logger.debug('creating new cache entry for {}.{}'
.format(im_self.__class__.__name__, self.decorated_function.__name__))
value = self.decorated_function(im_self, *args, **kwargs)
region.set(key, value)
return value
def __call__(self, im_self, *args, **kwargs):
"""Via the magic that is partial functions returned from __get__, im_self is the instance object of the class
we're decorating a method of and [kw]args are the actual parameters to the decorated method"""
region = cache_regions[im_self.cache_region]
if not region.enabled:
return self.decorated_function(im_self, *args, **kwargs)
key = (self.decorated_function.__name__, getattr(im_self, 'sid', im_self.uid),
tuple(getattr(x, 'sid', x) for x in args),
tuple((k, getattr(v, 'sid', v)) for k, v in sorted(kwargs.iteritems())),
defaults_sid())
key = dumps(key)
found, value = region.get(key)
if found:
return value
else:
im_self.logger.debug('creating new cache entry for {}.{}'
.format(im_self.__class__.__name__, self.decorated_function.__name__))
value = self.decorated_function(im_self, *args, **kwargs)
region.set(key, value)
return value
def assert_picklable(o):
s = dumps(o)
o2 = loads(s)
assert_is_equal(o, o2)
