def test_get(): pool = MemoryPool() pa = empty(9) pc = empty(11) pool.add(pa) pool.add(pc) def func(v, b, bs, ba, bc, s, a, c): assert_equal(v.shape, s) if a: assert_aligned(v) if c: assert_contiguous(v) if a > ba or c and not bc or not bc and s != bs: assert address(pool._buffers) == address([pa, b]) else: assert address(pool._buffers) == address([pa, pc]) for b, ba, bc in zip(buffers, aligned, contiguous): with pool.set(b): for (s, a, c) in itertools.product([(10, ), (5, 2), (2, 5)], [False, True], [False, True]): with pool.get(s, float, a, c) as v: yield func, v, b, b.shape, ba, bc, s, a, c assert address(pool._buffers) == address([pa, b, pc]) assert address(pool._buffers) == address([pa, pc])
def test_get(): pool = MemoryPool() pa = empty(9) pc = empty(11) pool.add(pa) pool.add(pc) def func(v, b, bs, ba, bc, s, a, c): assert_equal(v.shape, s) if a: assert_aligned(v) if c: assert_contiguous(v) if a > ba or c and not bc or not bc and s != bs: assert address(pool._buffers) == address([pa, b]) else: assert address(pool._buffers) == address([pa, pc]) for b, ba, bc in zip(buffers, aligned, contiguous): with pool.set(b): for (s, a, c) in itertools.product([(10,), (5, 2), (2, 5)], [False, True], [False, True]): with pool.get(s, float, a, c) as v: yield func, v, b, b.shape, ba, bc, s, a, c assert address(pool._buffers) == address([pa, b, pc]) assert address(pool._buffers) == address([pa, pc])
def __init__(self, flib_id, shape, block_shape, nmax, sparse_axis, dtype=None, dtype_index=None, data=None, verbose=False): self._flib_id = flib_id if not isinstance(shape, (list, tuple)): raise TypeError("Invalid shape '{0}'.".format(shape)) if len(shape) != 2: raise ValueError("The number of dimensions is not 2.") if len(block_shape) != 2: raise ValueError( "The number of dimensions of the blocks is not 2.") straxes = ('row', 'column') if sparse_axis == 1: straxes = straxes[::-1] if any(s % b != 0 for s, b in zip(shape, block_shape)): raise ValueError( "The shape of the matrix '{0}' is incompatible with blocks of " "shape '{1}'.".format(shape, block_shape)) if data is None: if nmax is None: raise ValueError('The maximum number of non-zero {0}s per {1} ' 'is not specified.'.format(*straxes)) shape_data = (shape[1-sparse_axis] // block_shape[0], nmax) if dtype is None: dtype = float if dtype_index is None: dtype_index = int dtype_data = self._get_dtype_data(dtype, dtype_index, block_shape) data = empty(shape_data, dtype_data, verbose=verbose) elif 'index' not in data.dtype.names: raise TypeError('The structured array has no field index.') elif len(data.dtype.names) == 1: raise TypeError('The structured array has no data field.') elif product(data.shape[:-1]) * block_shape[1-sparse_axis] != \ shape[1-sparse_axis]: raise ValueError( "The shape of the matrix '{0}' is incompatible with that of th" "e structured array '{1}'.".format(shape, data.shape)) elif nmax not in (None, data.shape[-1]): raise ValueError( "The n{0}max keyword value '{1}' is incompatible with the shap" "e of the structured array '{2}'.".format( straxes[0][:3], nmax, data.shape)) else: dtype_index = data.dtype['index'] dtype = data.dtype[1] if dtype.type == np.void: dtype = dtype.subdtype[0].type expected = self._get_dtype_data(dtype, dtype_index, block_shape) if data.dtype != expected: raise TypeError( 'The input dtype {0} is invalid. Expected dtype is {1}.'. format(data.dtype, expected)) if nmax is None: nmax = data.shape[-1] self.dtype = np.dtype(dtype) self.data = data.view(np.recarray) self.ndim = 2 self.shape = tuple(shape) self.block_shape = tuple(block_shape) setattr(self, 'n' + straxes[0][:3] + 'max', int(nmax))
def empty(cls, shape, mask=None, header=None, unit=None, derived_units=None, dtype=None, order=None, **keywords): if dtype is None: dtype = cls.default_dtype return cls(empty(shape, dtype, order), mask=mask, header=header, unit=unit, derived_units=derived_units, dtype=dtype, copy=False, **keywords)
def empty(self): """ Return a new array as described by the scene, without initializing entries. """ return empty(self.shape, self.dtype)
def restrict(self, mask, inplace=False): """ Restrict the operator to a subspace defined by a mask (True means that the element is kept). Indices are renumbered in-place if the inplace keyword is set to True. """ if not isinstance(self.matrix, (FSRMatrix, FSRBlockMatrix, FSRRotation2dMatrix, FSRRotation3dMatrix)): raise NotImplementedError( 'Restriction is not implemented for {0} sparse storage.'. format(type(self.matrix).__name__)) mask = np.asarray(mask) if mask.dtype != bool: raise TypeError('The mask is not boolean.') if isinstance(self.matrix, FSRMatrix): block_shapein = self.broadcastable_shapein else: block_shapein = self.shapein[:-1] if mask.shape != block_shapein: raise ValueError( "Invalid shape '{}'. Expected value is '{}'.".format( mask.shape, block_shapein)) if inplace: matrix = self.matrix else: matrix = self.matrix.copy() itype = matrix.data.dtype['index'] if itype.type in (np.int8, np.int16, np.int32, np.int64): f = 'fsr_restrict_i{0}'.format(itype.itemsize) func = getattr(fsp, f) block_shape = matrix.block_shape[0] ncol = func( matrix.data.view(np.int8).ravel(), mask.ravel(), matrix.ncolmax, matrix.shape[0] // block_shape, matrix.data.strides[-1]) else: ncol = np.sum(mask) new_index = empty(mask.shape, itype) new_index[...] = -1 new_index[mask] = np.arange(ncol, dtype=itype) undef = matrix.data.index < 0 matrix.data.index = new_index[matrix.data.index] matrix.data.index[undef] = -1 out = self.copy() matrix.shape = matrix.shape[0], ncol * matrix.block_shape[1] out.matrix = matrix if isinstance(self.matrix, FSRMatrix): out.broadcastable_shapein = (ncol, ) if self.shapein is not None: ndims_in = len(self.broadcastable_shapein) out.shapein = (ncol, ) + self.shapein[ndims_in:] else: out.shapein = (ncol, out.matrix.block_shape[1]) if inplace: self.delete() return out
def restrict(self, mask, inplace=False): """ Restrict the operator to a subspace defined by a mask (True means that the element is kept). Indices are renumbered in-place if the inplace keyword is set to True. """ if not isinstance(self.matrix, (FSRMatrix, FSRBlockMatrix, FSRRotation2dMatrix, FSRRotation3dMatrix)): raise NotImplementedError( 'Restriction is not implemented for {0} sparse storage.' .format(type(self.matrix).__name__)) mask = np.asarray(mask) if mask.dtype != bool: raise TypeError('The mask is not boolean.') if isinstance(self.matrix, FSRMatrix): block_shapein = self.broadcastable_shapein else: block_shapein = self.shapein[:-1] if mask.shape != block_shapein: raise ValueError("Invalid shape '{}'. Expected value is '{}'.". format(mask.shape, block_shapein)) if inplace: matrix = self.matrix else: matrix = self.matrix.copy() itype = matrix.data.dtype['index'] if itype.type in (np.int8, np.int16, np.int32, np.int64): f = 'fsr_restrict_i{0}'.format(itype.itemsize) func = getattr(fsp, f) block_shape = matrix.block_shape[0] ncol = func(matrix.data.view(np.int8).ravel(), mask.ravel(), matrix.ncolmax, matrix.shape[0] // block_shape, matrix.data.strides[-1]) else: ncol = np.sum(mask) new_index = empty(mask.shape, itype) new_index[...] = -1 new_index[mask] = np.arange(ncol, dtype=itype) undef = matrix.data.index < 0 matrix.data.index = new_index[matrix.data.index] matrix.data.index[undef] = -1 out = self.copy() matrix.shape = matrix.shape[0], ncol * matrix.block_shape[1] out.matrix = matrix if isinstance(self.matrix, FSRMatrix): out.broadcastable_shapein = (ncol,) if self.shapein is not None: ndims_in = len(self.broadcastable_shapein) out.shapein = (ncol,) + self.shapein[ndims_in:] else: out.shapein = (ncol, out.matrix.block_shape[1]) if inplace: self.delete() return out
def _fold_psd(p): """ Convert even two-sided PSD into one-sided PSD. """ p = np.asarray(p) n = p.shape[-1] // 2 + 1 out = empty(p.shape[:-1] + (n, )) out[..., 0] = p[..., 0] np.multiply(p[..., 1:n - 1], 2, out[..., 1:-1]) out[..., -1] = p[..., n] return out
def empty(cls, shape, header=None, unit=None, derived_units=None, coverage=None, error=None, origin=None, dtype=None, order=None, **keywords): if dtype is None: dtype = cls.default_dtype return cls(empty(shape, dtype, order), header=header, unit=unit, derived_units=derived_units, coverage=coverage, error=error, origin=origin, copy=False, dtype=dtype, **keywords)
def _fold_psd(p): """ Convert even two-sided PSD into one-sided PSD. """ p = np.asarray(p) n = p.shape[-1] // 2 + 1 out = empty(p.shape[:-1] + (n,)) out[..., 0] = p[..., 0] np.multiply(p[..., 1:n-1], 2, out[..., 1:-1]) out[..., -1] = p[..., n] return out
def _unfold_psd(p): """ Convert one-sided PSD into even two-sided PSD. """ p = np.asarray(p) n = p.shape[-1] out = empty(p.shape[:-1] + (2 * (n - 1),)) out[..., 0] = p[..., 0] np.multiply(p[..., 1:n-1], 0.5, out[..., 1:n-1]) out[..., n-1] = p[..., n-1] out[..., n:] = out[..., n-2:0:-1] return out
def test_new_entry(): pool = MemoryPool() a = empty(12) b = empty(20) pool.add(a) pool.add(b) shapes = ((4,), (15,), (30,)) def func(i, s, d=-1): assert_equal(len(pool), 3 + i) for i, s in enumerate(shapes): with pool.get(s, float): pass yield func, i, s for s in [a.shape, b.shape]: for d in [0, 1, 2]: with pool.get(s[0] - d, float): pass yield func, i, s, d
def _unfold_psd(p): """ Convert one-sided PSD into even two-sided PSD. """ p = np.asarray(p) n = p.shape[-1] out = empty(p.shape[:-1] + (2 * (n - 1), )) out[..., 0] = p[..., 0] np.multiply(p[..., 1:n - 1], 0.5, out[..., 1:n - 1]) out[..., n - 1] = p[..., n - 1] out[..., n:] = out[..., n - 2:0:-1] return out
def test_new_entry(): pool = MemoryPool() a = empty(12) b = empty(20) pool.add(a) pool.add(b) shapes = ((4, ), (15, ), (30, )) def func(i, s, d=-1): assert_equal(len(pool), 3 + i) for i, s in enumerate(shapes): with pool.get(s, float): pass yield func, i, s for s in [a.shape, b.shape]: for d in [0, 1, 2]: with pool.get(s[0] - d, float): pass yield func, i, s, d
def test_empty(): shapes = (10, (10,), (2, 10), (3, 3, 3)) dtypes = (float, np.int8, complex) def func(v, s, d): assert_equal(v.shape, tointtuple(s)) assert_equal(v.dtype, d) assert_aligned(v) assert_contiguous(v) for s in shapes: for d in dtypes: v = empty(s, d) yield func, v, s, d
def _logloginterp_psd(f, bandwidth, psd, out=None): """ Loglog-interpolation of one-sided PSD. """ f = np.asarray(f) psd = np.asarray(psd) frequency = np.arange(psd.shape[-1], dtype=float) * bandwidth if out is None: out = empty(psd.shape[:-1] + (f.size,)) out[..., 0] = psd[..., 0] _interp(np.log(f[1:]), np.log(frequency[1:-1]), np.log(psd[..., 1:-1]), out=out[..., 1:]) np.exp(out[..., 1:], out[..., 1:]) out[..., -1] /= 2 return out
def test_empty(): shapes = (10, (10, ), (2, 10), (3, 3, 3)) dtypes = (float, np.int8, complex) def func(v, s, d): assert_equal(v.shape, tointtuple(s)) assert_equal(v.dtype, d) assert_aligned(v) assert_contiguous(v) for s in shapes: for d in dtypes: v = empty(s, d) yield func, v, s, d
def _logloginterp_psd(f, bandwidth, psd, out=None): """ Loglog-interpolation of one-sided PSD. """ f = np.asarray(f) psd = np.asarray(psd) frequency = np.arange(psd.shape[-1], dtype=float) * bandwidth if out is None: out = empty(psd.shape[:-1] + (f.size, )) out[..., 0] = psd[..., 0] _interp(np.log(f[1:]), np.log(frequency[1:-1]), np.log(psd[..., 1:-1]), out=out[..., 1:]) np.exp(out[..., 1:], out[..., 1:]) out[..., -1] /= 2 return out
def empty(cls, shape, unit=None, derived_units=None, dtype=None, order=None, **keywords): if dtype is None: dtype = cls.default_dtype return cls(empty(shape, dtype, order), dtype=dtype, unit=unit, derived_units=derived_units, copy=False, **keywords)
def get_noise(self, out=None, operation=operation_assignment): """ Return the noise realization according the instrument's noise model. Parameters ---------- out : ndarray, optional Placeholder for the output noise. """ if out is None: if operation is not operation_assignment: raise ValueError('The output buffer is not specified.') out = empty((len(self.instrument), len(self.sampling))) for b in self.block: self.instrument.get_noise(self.sampling[b], out=out[:, b], operation=operation) return out
def _interp(z, x, y, out=None): """ Interpolate / extrapolate y(x) in z. x and z increasing. """ z = np.asarray(z) x = np.asarray(x) y = np.asarray(y) if out is None: out = empty(y.shape[:-1] + (z.size, )) z = np.array(z, ndmin=1, copy=False) ix = 1 x1 = x[0] x2 = x[1] for iz, z_ in enumerate(z): while z_ > x2 and ix < x.size - 1: ix += 1 x1 = x2 x2 = x[ix] out[..., iz] = ((z_ - x1) * y[..., ix] + (x2 - z_) * y[..., ix-1]) / \ (x2 - x1) return out
def _interp(z, x, y, out=None): """ Interpolate / extrapolate y(x) in z. x and z increasing. """ z = np.asarray(z) x = np.asarray(x) y = np.asarray(y) if out is None: out = empty(y.shape[:-1] + (z.size,)) z = np.array(z, ndmin=1, copy=False) ix = 1 x1 = x[0] x2 = x[1] for iz, z_ in enumerate(z): while z_ > x2 and ix < x.size - 1: ix += 1 x1 = x2 x2 = x[ix] out[..., iz] = ((z_ - x1) * y[..., ix] + (x2 - z_) * y[..., ix-1]) / \ (x2 - x1) return out
def _gaussian_sample(nsamples, sampling_frequency, psd, twosided=False, out=None, fftw_flag='FFTW_MEASURE'): """ Generate a gaussian N-sample sampled at fs from a one- or two-sided Power Spectrum Density sampled at fs/N. Parameter --------- nsamples : int Number of time samples. sampling_frequency : float Sampling frequency [Hz]. psd : array-like One- or two-sided Power Spectrum Density [signal unit**2/Hz]. twosided : boolean, optional Whether or not the input psd is one-sided (only positive frequencies) or two-sided (positive and negative frequencies). out : ndarray Placeholder for the output buffer. """ psd = np.asarray(psd) if out is None: out = empty(psd.shape[:-1] + (nsamples, )) if not twosided: psd = _unfold_psd(psd) shape = psd.shape[:-1] + (nsamples, ) gauss = np.random.randn(*shape) nthreads = multiprocessing.cpu_count() ftgauss = fft.fft(gauss, planner_effort=fftw_flag, threads=nthreads) ftgauss[..., 0] = 0 spec = ftgauss * np.sqrt(psd) out[...] = fft.ifft(spec, planner_effort=fftw_flag, threads=nthreads).real * np.sqrt(sampling_frequency) return out
def _gaussian_sample(nsamples, sampling_frequency, psd, twosided=False, out=None, fftw_flag='FFTW_MEASURE'): """ Generate a gaussian N-sample sampled at fs from a one- or two-sided Power Spectrum Density sampled at fs/N. Parameter --------- nsamples : int Number of time samples. sampling_frequency : float Sampling frequency [Hz]. psd : array-like One- or two-sided Power Spectrum Density [signal unit**2/Hz]. twosided : boolean, optional Whether or not the input psd is one-sided (only positive frequencies) or two-sided (positive and negative frequencies). out : ndarray Placeholder for the output buffer. """ psd = np.asarray(psd) if out is None: out = empty(psd.shape[:-1] + (nsamples,)) if not twosided: psd = _unfold_psd(psd) shape = psd.shape[:-1] + (nsamples,) gauss = np.random.randn(*shape) nthreads = multiprocessing.cpu_count() ftgauss = fft.fft(gauss, planner_effort=fftw_flag, threads=nthreads) ftgauss[..., 0] = 0 spec = ftgauss * np.sqrt(psd) out[...] = fft.ifft(spec, planner_effort=fftw_flag, threads=nthreads).real * np.sqrt(sampling_frequency) return out
from __future__ import division import itertools import numpy as np from numpy.testing import assert_equal from pyoperators.config import MEMORY_ALIGNMENT from pyoperators.memory import MemoryPool, empty from pyoperators.utils import tointtuple buffers = [ empty(10), empty((5, 2)), empty(20)[::2], empty(11)[1:], empty(21)[1:].reshape((10, 2))[::2, :] ] aligned = 3 * [True] + [False, False] contiguous = [_.flags.contiguous for _ in buffers] def assert_contiguous(x): assert x.flags.contiguous def assert_aligned(x): assert address(x) % MEMORY_ALIGNMENT == 0 def address(l): if isinstance(l, np.ndarray): return l.__array_interface__['data'][0]
from __future__ import division import itertools import numpy as np from numpy.testing import assert_equal from pyoperators.config import MEMORY_ALIGNMENT from pyoperators.memory import MemoryPool, empty from pyoperators.utils import tointtuple buffers = [empty(10), empty((5, 2)), empty(20)[::2], empty(11)[1:], empty(21)[1:].reshape((10, 2))[::2, :]] aligned = 3 * [True] + [False, False] contiguous = [_.flags.contiguous for _ in buffers] def assert_contiguous(x): assert x.flags.contiguous def assert_aligned(x): assert address(x) % MEMORY_ALIGNMENT == 0 def address(l): if isinstance(l, np.ndarray): return l.__array_interface__['data'][0] return [address(_) for _ in l] def test_empty(): shapes = (10, (10,), (2, 10), (3, 3, 3))
def __init__(self, flib_id, shape, block_shape, nmax, sparse_axis, dtype=None, dtype_index=None, data=None, verbose=False): self._flib_id = flib_id if not isinstance(shape, (list, tuple)): raise TypeError("Invalid shape '{0}'.".format(shape)) if len(shape) != 2: raise ValueError("The number of dimensions is not 2.") if len(block_shape) != 2: raise ValueError( "The number of dimensions of the blocks is not 2.") straxes = ('row', 'column') if sparse_axis == 1: straxes = straxes[::-1] if any(s % b != 0 for s, b in zip(shape, block_shape)): raise ValueError( "The shape of the matrix '{0}' is incompatible with blocks of " "shape '{1}'.".format(shape, block_shape)) if data is None: if nmax is None: raise ValueError('The maximum number of non-zero {0}s per {1} ' 'is not specified.'.format(*straxes)) shape_data = (shape[1 - sparse_axis] // block_shape[0], nmax) if dtype is None: dtype = float if dtype_index is None: dtype_index = int dtype_data = self._get_dtype_data(dtype, dtype_index, block_shape) data = empty(shape_data, dtype_data, verbose=verbose) elif 'index' not in data.dtype.names: raise TypeError('The structured array has no field index.') elif len(data.dtype.names) == 1: raise TypeError('The structured array has no data field.') elif product(data.shape[:-1]) * block_shape[1-sparse_axis] != \ shape[1-sparse_axis]: raise ValueError( "The shape of the matrix '{0}' is incompatible with that of th" "e structured array '{1}'.".format(shape, data.shape)) elif nmax not in (None, data.shape[-1]): raise ValueError( "The n{0}max keyword value '{1}' is incompatible with the shap" "e of the structured array '{2}'.".format( straxes[0][:3], nmax, data.shape)) else: dtype_index = data.dtype['index'] dtype = data.dtype[1] if dtype.type == np.void: dtype = dtype.subdtype[0].type expected = self._get_dtype_data(dtype, dtype_index, block_shape) if data.dtype != expected: raise TypeError( 'The input dtype {0} is invalid. Expected dtype is {1}.'. format(data.dtype, expected)) if nmax is None: nmax = data.shape[-1] self.dtype = np.dtype(dtype) self.data = data.view(np.recarray) self.ndim = 2 self.shape = tuple(shape) self.block_shape = tuple(block_shape) setattr(self, 'n' + straxes[0][:3] + 'max', int(nmax))
def get_noise(self, sampling, psd=None, bandwidth=None, twosided=False, sigma=None, nep=None, fknee=0, fslope=1, out=None, operation=operation_assignment): """ Return the noise realization following a given PSD. The input Power Spectrum Density can either be fully specified by using the 'bandwidth' and 'psd' keywords, or by providing the parameters of the gaussian distribution: psd = sigma**2 * (1 + (fknee/f)**fslope) / B where B is equal to sampling_frequency / N. Parameters ---------- sampling : Sampling The temporal sampling. psd : array-like, optional The one-sided or two-sided Power Spectrum Density, [signal unit**2/Hz]. bandwidth : float, optional The PSD frequency increment [Hz]. twosided : boolean, optional Whether or not the output psd is one-sided (only positive frequencies) or two-sided (positive and negative frequencies). sigma : float, optional Standard deviation of the white noise component. fknee : float, optional The 1/f noise knee frequency [Hz]. fslope : float, optional The 1/f noise slope. out : ndarray, optional Placeholder for the output noise. """ if bandwidth is None and psd is not None or \ bandwidth is not None and psd is None: raise ValueError('The bandwidth or the PSD is not specified.') if nep is not None: sigma = nep / np.sqrt(2 * sampling.period) if bandwidth is None and psd is None and sigma is None: raise ValueError('The noise model is not specified.') if out is None and operation is not operation_assignment: raise ValueError('The output buffer is not specified.') shape = (len(self), len(sampling)) # handle non-correlated case first if bandwidth is None and fknee == 0: sigma = np.atleast_1d(sigma) noise = np.random.standard_normal(shape) * sigma[:, None] if out is None: return noise operation(out, noise) return out if out is None: out = empty((len(self), len(sampling))) sampling_frequency = 1 / sampling.period # fold two-sided input PSD if bandwidth is not None and psd is not None: if twosided: psd = _fold_psd(psd) twosided = False else: twosided = True n = len(sampling) if bandwidth is None and psd is None: p = _gaussian_psd_1f(n, sampling_frequency, sigma, fknee, fslope, twosided=twosided) else: # log-log interpolation of one-sided PSD f = np.arange(n // 2 + 1, dtype=float) * (sampling_frequency / n) p = _logloginterp_psd(f, bandwidth, psd) # looping over the detectors for out_ in out: operation(out_, _gaussian_sample(n, sampling_frequency, p, twosided=twosided)) return out