コード例 #1
0
ファイル: db.py プロジェクト: ftschindler-work/simdb
    def duplicates(self, params=None):
        if params:
            params = sorted(params)
            keyfunc = lambda ds: (
                ds.experiment, list(
                    dumps(ds.p[k], protocol=-1) for k in params))
            groups = itertools.groupby(sorted(
                (ds for ds in self.datasets
                 if set(ds.p.dict.keys()) >= set(params)),
                key=keyfunc),
                                       key=keyfunc)
        else:
            keyfunc = lambda ds: (ds.experiment,
                                  list(
                                      sorted((k, dumps(v, protocol=-1))
                                             for k, v in ds.p.dict.items())))
            groups = itertools.groupby(sorted(self.datasets, key=keyfunc),
                                       key=keyfunc)

        def get_duplicates(groups):
            for k, v in groups:
                datasets = list(v)
                if len(datasets) > 1:
                    p = datasets[0].p.dict
                    info = {
                        'experiment': datasets[0].experiment,
                        'params':
                        p if not params else {k: p[k]
                                              for k in params}
                    }
                    yield [info, datasets]

        class DuplicatesList(list):
            def __str__(self):
                formatter = lambda i, ds: ('experiment: ' + i[
                    'experiment'] + '\n' + 'params:     ' + '\n'.join(
                        textwrap.wrap(
                            ', '.join('{}={}'.format(k, v)
                                      for k, v in sorted(i['params'].items())),
                            initial_indent='',
                            subsequent_indent=' ' * len('experiment: '),
                            width=100)) + '\n' + 'count:      ' + str(len(ds))
                                           + '\n')
                return '\n'.join(formatter(i, ds) for i, ds in self)

            def delete_old(self):
                for i, datasets in self:
                    for ds in datasets[:-1]:
                        ds.delete()

            __repr__ = __str__

        return DuplicatesList(get_duplicates(groups))
コード例 #2
0
ファイル: dogpile_backends.py プロジェクト: ftalbrecht/pymor
 def set(self, key, value):
     self._enforce_limits(value)
     value = dumps(value)
     if not key in self._keylist:
         self._new_key(key, len(value))
     with self._dbm_file(True) as dbm:
         dbm[key] = value
コード例 #3
0
def test_pickle_by_solving(discretization):
    d = discretization
    d2 = loads(dumps(d))
    d.disable_caching()
    d2.disable_caching()
    for mu in d.parameter_space.sample_randomly(3, seed=234):
        assert np.all(d.solve(mu).almost_equal(d2.solve(mu)))
コード例 #4
0
    def call(method, *args, **kwargs):
        """Execute method on all MPI ranks.

        Assuming :func:`event_loop` is running on all MPI ranks
        (except rank 0), this will execute `method` on all
        ranks (including rank 0) with positional arguments
        `args` and keyword arguments `kwargs`.

        Parameters
        ----------
        method
            The function to execute on all ranks (must be picklable).
        args
            The positional arguments for `method`.
        kwargs
            The keyword arguments for `method`.

        Returns
        -------
        The return value of `method` on rank 0.
        """
        assert rank0
        if finished:
            return
        comm.bcast(dumps((method, args, kwargs)), root=0)
        return method(*args, **kwargs)
コード例 #5
0
def test_pickle_by_solving(model):
    m = model
    m2 = loads(dumps(m))
    m.disable_caching()
    m2.disable_caching()
    for mu in m.parameter_space.sample_randomly(3, seed=234):
        assert np.all(almost_equal(m.solve(mu), m2.solve(mu)))
コード例 #6
0
ファイル: function.py プロジェクト: renemilk/pyMor
def test_pickle_by_evaluation(function):
    f = function
    f2 = loads(dumps(f))
    mus = parameters_of_type(f.parameter_type, 47)
    for arg in function_argument(f, 10, 42):
        mu = next(mus)
        assert np.all(f.evaluate(arg, mu) == f2.evaluate(arg, mu))
コード例 #7
0
ファイル: mpi.py プロジェクト: denfromufa/pymor
    def call(method, *args, **kwargs):
        """Execute method on all MPI ranks.

        Assuming :func:`event_loop` is running on all MPI ranks
        (except rank 0), this will execute `method` on all
        ranks (including rank 0) with sequential arguments
        `args` and keyword arguments `kwargs`.

        Parameters
        ----------
        method
            The function to execute on all ranks. Must be picklable.
        args
            The sequential arguments for `method`.
        kwargs
            The keyword arguments for `method`.

        Returns
        -------
        The return value of `method` on rank 0.
        """
        assert rank0
        if finished:
            return
        comm.bcast(dumps((method, args, kwargs)), root=0)
        return method(*args, **kwargs)
コード例 #8
0
def build_cache_key(obj):
    def transform_obj(obj):
        t = type(obj)
        if t in (NoneType, bool, int, float, str, bytes):
            return obj
        elif t is np.ndarray:
            if obj.dtype == object:
                raise CacheKeyGenerationError(
                    'Cannot generate cache key for provided arguments')
            return obj
        elif t in (list, tuple):
            return tuple(transform_obj(o) for o in obj)
        elif t in (set, frozenset):
            return tuple(transform_obj(o) for o in sorted(obj))
        elif t in (Mu, Parameters):
            return tuple(
                (transform_obj(k), transform_obj(v)) for k, v in obj.items())
        elif t in (dict, Mu, Parameters):
            return tuple((transform_obj(k), transform_obj(v))
                         for k, v in sorted(obj.items()))
        elif isinstance(obj, Number):
            # handle numpy number objects
            return obj
        else:
            raise CacheKeyGenerationError(
                'Cannot generate cache key for provided arguments')

    obj = transform_obj(obj)
    key = hashlib.sha256(dumps(obj, protocol=-1)).hexdigest()

    return key
コード例 #9
0
def test_pickle_by_evaluation(function):
    f = function
    f2 = loads(dumps(f))
    mus = parameters_of_type(f.parameter_type, 47)
    for arg in function_argument(f, 10, 42):
        mu = next(mus)
        assert np.all(f.evaluate(arg, mu) == f2.evaluate(arg, mu))
コード例 #10
0
ファイル: mpi.py プロジェクト: denfromufa/pymor
    def quit():
        """Exit the event loop on all MPI ranks.

        This will cause :func:`event_loop` to terminate on all
        MPI ranks.
        """
        global finished
        comm.bcast(dumps(('QUIT', None, None)))
        finished = True
コード例 #11
0
    def quit():
        """Exit the event loop on all MPI ranks.

        This will cause :func:`event_loop` to terminate on all
        MPI ranks.
        """
        global finished
        comm.bcast(dumps(('QUIT', None, None)))
        finished = True
コード例 #12
0
def assert_picklable_without_dumps_function(o):
    def dumps_function_raise(function):
        raise PicklingError('Cannot pickle function {}'.format(function))

    old_code = dumps_function.__code__
    dumps_function.__code__ = dumps_function_raise.__code__
    try:
        s = dumps(o)
        o2 = loads(s)
        assert_is_equal(o, o2)
    finally:
        dumps_function.__code__ = old_code
コード例 #13
0
ファイル: pickle.py プロジェクト: sdrave/pymor
def assert_picklable_without_dumps_function(o):
    def dumps_function_raise(function):
        raise PicklingError("Cannot pickle function {}".format(function))

    old_code = dumps_function.func_code
    dumps_function.func_code = dumps_function_raise.func_code
    try:
        s = dumps(o)
        o2 = loads(s)
        assert_is_equal(o, o2)
    finally:
        dumps_function.func_code = old_code
コード例 #14
0
ファイル: functionals.py プロジェクト: andreasbuhr/pymor
 def __getstate__(self):
     s = self.__dict__.copy()
     try:
         pickled_mapping = dumps(self._mapping)
         picklable = True
     except PicklingError:
         self.logger.warn('Mapping not picklable, trying pymor.core.pickle.dumps_function.')
         pickled_mapping = dumps_function(self._mapping)
         picklable = False
     s['_mapping'] = pickled_mapping
     s['_picklable'] = picklable
     return s
コード例 #15
0
ファイル: pickling.py プロジェクト: pymor/pymor
def assert_picklable_without_dumps_function(o):

    def dumps_function_raise(function):
        raise PicklingError(f'Cannot pickle function {function}')

    old_code = dumps_function.__code__
    dumps_function.__code__ = dumps_function_raise.__code__
    try:
        s = dumps(o)
        o2 = loads(s)
        assert_is_equal(o, o2)
    finally:
        dumps_function.__code__ = old_code
コード例 #16
0
ファイル: functionals.py プロジェクト: michaellaier/pymor
 def __getstate__(self):
     s = self.__dict__.copy()
     try:
         pickled_mapping = dumps(self._mapping)
         picklable = True
     except PicklingError:
         self.logger.warn(
             'Mapping not picklable, trying pymor.core.pickle.dumps_function.'
         )
         pickled_mapping = dumps_function(self._mapping)
         picklable = False
     s['_mapping'] = pickled_mapping
     s['_picklable'] = picklable
     return s
コード例 #17
0
ファイル: cache.py プロジェクト: ftalbrecht/pymor
    def __call__(self, im_self, *args, **kwargs):
        """Via the magic that is partial functions returned from __get__, im_self is the instance object of the class
        we're decorating a method of and [kw]args are the actual parameters to the decorated method"""
        region = cache_regions[im_self.cache_region]
        if not region.enabled:
            return self.decorated_function(im_self, *args, **kwargs)

        key = (self.decorated_function.__name__, getattr(im_self, 'sid', im_self.uid),
               tuple(getattr(x, 'sid', x) for x in args),
               tuple((k, getattr(v, 'sid', v)) for k, v in sorted(kwargs.iteritems())),
               defaults_sid())
        key = dumps(key)
        found, value = region.get(key)
        if found:
            return value
        else:
            im_self.logger.debug('creating new cache entry for {}.{}'
                                 .format(im_self.__class__.__name__, self.decorated_function.__name__))
            value = self.decorated_function(im_self, *args, **kwargs)
            region.set(key, value)
            return value
コード例 #18
0
ファイル: cache.py プロジェクト: michaellaier/pymor
    def __call__(self, im_self, *args, **kwargs):
        """Via the magic that is partial functions returned from __get__, im_self is the instance object of the class
        we're decorating a method of and [kw]args are the actual parameters to the decorated method"""
        region = cache_regions[im_self.cache_region]
        if not region.enabled:
            return self.decorated_function(im_self, *args, **kwargs)

        key = (self.decorated_function.__name__, getattr(im_self, 'sid', im_self.uid),
               tuple(getattr(x, 'sid', x) for x in args),
               tuple((k, getattr(v, 'sid', v)) for k, v in sorted(kwargs.iteritems())),
               defaults_sid())
        key = dumps(key)
        found, value = region.get(key)
        if found:
            return value
        else:
            im_self.logger.debug('creating new cache entry for {}.{}'
                                 .format(im_self.__class__.__name__, self.decorated_function.__name__))
            value = self.decorated_function(im_self, *args, **kwargs)
            region.set(key, value)
            return value
コード例 #19
0
ファイル: pickling.py プロジェクト: prklVIP/pymor
def assert_picklable(o):
    s = dumps(o)
    o2 = loads(s)
    assert_is_equal(o, o2)