def test_special_attribute_func4(): class P(PackedTable): def __init__(self, shape, ordering, val): PackedTable.__init__(self, shape, ordering=ordering, val=val, key1=None, key2=None, key3=None) myscalar1 = 2 _myscalar2 = 3 def key1(self): return self.val * self.myscalar1 * self._myscalar2 @property def key2(self): return self.val * self.myscalar1 * self._myscalar2 def key3(self, x): return x * self.val * self.myscalar1 * self._myscalar2 shape = (6, 6) ordering = np.arange(product(shape))[::-1].reshape(shape) val = np.arange(product(shape)).reshape(shape) layout = P(shape, ordering, val) assert_same(layout.key1, val.ravel()[::-1] * 6) assert_same(layout.all.key1, val * 6) assert_same(layout.key2, val.ravel()[::-1] * 6) assert_same(layout.all.key2, val * 6) assert_same(layout.key3(2), val.ravel()[::-1] * 12) assert_same(layout.all.key3(2), val * 12)
def get_pointing_matrix(self, header, npixels_per_sample=0, method=None, downsampling=False, section=None, comm=MPI.COMM_WORLD): # let's get the pointing matrix associated to the section if npixels_per_sample not in (0, self.info.npixels_per_sample): raise ValueError('The npixels_per_sample value is incompatible wi' \ 'th the MADMAP1 file.') npixels_per_sample = self.info.npixels_per_sample # if no slice is provided, return a pointing matrix for each of them if section is None: return super(MadMap1Observation, self).get_pointing_matrix(header, npixels_per_sample, method) if method is None: method = 'default' method = method.lower() if method != 'default': raise ValueError("Invalid pointing matrix method '" + method + "'.") header = gather_fitsheader_if_needed(header, comm=comm) shape_input = tuple(header['NAXIS' + str(i+1)] for i in range(header['NAXIS']))[::-1] if product(shape_input) > np.iinfo(np.int32).max: raise RuntimeError('The map is too large: pixel indices cannot be s' 'stored using 32 bits: {0}>{1}'.format(product(shape_input), np.iinfo(np.int32).max)) pointing = self.pointing[section.start:section.stop] ndetectors = self.get_ndetectors() nsamples = int(np.sum(~pointing.removed)) tod = np.empty((ndetectors, nsamples)) try: islice = list(self.slice.start).index(section.start) if self.slice[islice].stop != section.stop: raise ValueError() except ValueError: raise RuntimeError('Only observation slice can be specified through' ' the section keyword.') shape = (ndetectors, nsamples, npixels_per_sample) info = {'header' : header, 'method' : method, 'outside': False, 'npixels_per_sample_min': npixels_per_sample} try: pmatrix = PointingMatrix.empty(shape, shape_input, info=info, verbose=False) except MemoryError: gc.collect() pmatrix = PointingMatrix.empty(shape, shape_input, info=info, verbose=False) status = tmf.madmap1_read_tod(self.info.todfile, self.info.invnttfile, self.info.convert, npixels_per_sample, islice + 1, tod.T, pmatrix.ravel().view(np.int64)) if status != 0: raise RuntimeError() return pmatrix
def test_special_attribute_func3(): shape = (6, 6) ordering = np.arange(product(shape))[::-1].reshape(shape) val = np.arange(product(shape)).reshape(shape) func = lambda s, x: x * s.val * s.myscalar1 * s._myscalar2 layout = PackedTable(shape, ordering=ordering, val=val, key=func) layout.myscalar1 = 2 layout._myscalar2 = 3 assert_same(layout.key(2), val.ravel()[::-1] * 12) assert_same(layout.all.key(2), val * 12)
def func(s1, s2, sm1, sm2): shapein = broadcast_shapes(s1 + sm2[1:], s2 + sm2[1:]) data1 = np.arange(product(s1 + sm1)).reshape(s1 + sm1) data2 = np.arange(product(s2 + sm2)).reshape(s2 + sm2) op1 = DenseBlockDiagonalOperator(data1) op2 = DenseBlockDiagonalOperator(data2) comp1 = op1 * op2 assert_is_instance(comp1, DenseBlockDiagonalOperator) with rule_manager(none=True): comp2 = op1 * op2 assert_equal(comp1.todense(shapein), comp2.todense(shapein))
def func(shapein, shapeout, extrainput): datashape = shapeout + shapein inputshape = extrainput + shapein d = np.arange(product(datashape)).reshape(datashape) b = DenseOperator(d, naxesin=len(shapein), naxesout=len(shapeout), shapein=inputshape) bdense = b.todense() n = product(extrainput) d_ = d.reshape((product(shapeout), product(shapein))) expected = BlockDiagonalOperator(n * [d_], axisin=0).todense() assert_equal(bdense, expected)
def plot_tod(tod, mask=None, **kw): """Plot the signal timelines in a Tod and show masked samples. Plotting every detector timelines may be time consuming, so it is recommended to use this method on one or few detectors like this: >>> plot_tod(tod[idetector]) """ import matplotlib.pyplot as mp if mask is None: mask = getattr(tod, 'mask', None) ndetectors = product(tod.shape[0:-1]) tod = tod.view().reshape((ndetectors, -1)) if mask is not None: mask = mask.view().reshape((ndetectors, -1)) if np.all(mask): print('There is no valid sample.') return for idetector in range(ndetectors): mp.plot(tod[idetector], **kw) if mask is not None: index = np.where(mask[idetector]) mp.plot(index, tod[idetector, index], 'ro') unit = getattr(tod, 'unit', '') if unit: mp.ylabel('Signal [' + unit + ']') else: mp.ylabel('Signal') mp.xlabel('Time sample')
def __init__(self, flib_id, shape, block_shape, nmax, sparse_axis, dtype=None, dtype_index=None, data=None, verbose=False): self._flib_id = flib_id if not isinstance(shape, (list, tuple)): raise TypeError("Invalid shape '{0}'.".format(shape)) if len(shape) != 2: raise ValueError("The number of dimensions is not 2.") if len(block_shape) != 2: raise ValueError( "The number of dimensions of the blocks is not 2.") straxes = ('row', 'column') if sparse_axis == 1: straxes = straxes[::-1] if any(s % b != 0 for s, b in zip(shape, block_shape)): raise ValueError( "The shape of the matrix '{0}' is incompatible with blocks of " "shape '{1}'.".format(shape, block_shape)) if data is None: if nmax is None: raise ValueError('The maximum number of non-zero {0}s per {1} ' 'is not specified.'.format(*straxes)) shape_data = (shape[1-sparse_axis] // block_shape[0], nmax) if dtype is None: dtype = float if dtype_index is None: dtype_index = int dtype_data = self._get_dtype_data(dtype, dtype_index, block_shape) data = empty(shape_data, dtype_data, verbose=verbose) elif 'index' not in data.dtype.names: raise TypeError('The structured array has no field index.') elif len(data.dtype.names) == 1: raise TypeError('The structured array has no data field.') elif product(data.shape[:-1]) * block_shape[1-sparse_axis] != \ shape[1-sparse_axis]: raise ValueError( "The shape of the matrix '{0}' is incompatible with that of th" "e structured array '{1}'.".format(shape, data.shape)) elif nmax not in (None, data.shape[-1]): raise ValueError( "The n{0}max keyword value '{1}' is incompatible with the shap" "e of the structured array '{2}'.".format( straxes[0][:3], nmax, data.shape)) else: dtype_index = data.dtype['index'] dtype = data.dtype[1] if dtype.type == np.void: dtype = dtype.subdtype[0].type expected = self._get_dtype_data(dtype, dtype_index, block_shape) if data.dtype != expected: raise TypeError( 'The input dtype {0} is invalid. Expected dtype is {1}.'. format(data.dtype, expected)) if nmax is None: nmax = data.shape[-1] self.dtype = np.dtype(dtype) self.data = data.view(np.recarray) self.ndim = 2 self.shape = tuple(shape) self.block_shape = tuple(block_shape) setattr(self, 'n' + straxes[0][:3] + 'max', int(nmax))
def func(shapein, shapeout, extradata, extrainput): datashape = extradata + shapeout + shapein d = np.arange(product(datashape)).reshape(datashape) b = DenseBlockDiagonalOperator(d, naxesin=len(shapein), naxesout=len(shapeout)) new_shape = broadcast_shapes(extradata, extrainput) bdense = b.todense(shapein=new_shape + shapein) d_ = reshape_broadcast(d, new_shape + shapeout + shapein) d_ = d_.reshape(-1, product(shapeout), product(shapein)) expected = BlockDiagonalOperator( [_ for _ in d_], axisin=0).todense(shapein=product(new_shape + shapein)) assert_same(bdense, expected) bTdense = b.T.todense(shapein=new_shape + shapeout) assert_same(bTdense, expected.T)
def validateout(self, shape): size = product(shape) if isinstance(self.matrix, (FSCMatrix, FSRMatrix)): if size % self.matrix.shape[0] != 0: raise ValueError( "The output shape '{0}' is incompatible with the expected " "number of rows '{1}'.".format(shape, self.matrix.shape[0])) self.matrix._validateout(shape)
def func(shape, axis): dX = DifferenceOperator(axis=axis, shapein=shape) a = np.arange(product(shape)).reshape(shape) assert_eq(dX(a), np.diff(a, axis=axis)) dX_dense = dX.todense() dXT_dense = dX.T.todense() assert_eq(dX_dense.T, dXT_dense)
def validatein(self, shape): size = product(shape) if isinstance(self.matrix, (FSCMatrix, FSRMatrix)): if size % self.matrix.shape[1] != 0: raise ValueError( "The input shape '{0}' is incompatible with the expected n" "umber of column '{1}'.".format(shape, self.matrix.shape[1])) self.matrix._validatein(shape)
def test_special_attribute_func2(): shape = (6, 6) ordering = np.arange(product(shape))[::-1].reshape(shape) val = np.arange(product(shape)).reshape(shape) layout = PackedTable(shape, ordering=ordering, val=val, key=None) layout.myscalar1 = 2 layout._myscalar2 = 3 layout_funcs = (lambda: val.ravel()[::-1] * 6, lambda s: s.val * s.myscalar1 * s._myscalar2) def func(v): setattr(layout, 'key', v) assert_equal(layout.all.key, val * 6) assert_equal(layout.key, val.ravel()[::-1] * 6) assert_same(layout.all.key.shape, layout.shape) assert_same(layout.key.shape, (len(layout),)) for v in layout_funcs: yield func, v
def func(op, shape): vec = np.arange(product(shape)).reshape(shape) vec_ = vec.reshape((-1, 3)) expected = np.empty(shape) expected_ = expected.reshape((-1, 3)) for i in range(expected.size // 3): expected_[i] = op(vec_[i]) actual = op(vec) assert_same(actual, expected)
def func(x): x = np.asarray(x) out = np.empty((ntot, ) + x.shape[1:], x.dtype) nbytes = product(x.shape[1:]) * x.itemsize self.comm.Allgatherv(x.view(np.byte), [ out.view(np.byte), ([_ * nbytes for _ in counts], [_ * nbytes for _ in offsets]) ]) return out
def test_dense_error(): shapes = ((2, ), (3, 2)) data = (np.arange(product(s)).reshape(s) for s in shapes) def func(d): b = DenseOperator(d) assert_raises(ValueError, b, np.ones(3)) for d in data: yield func, d
def _validateout(self, shape): size = product(shape) if not isinstance(self, (FSCMatrix, FSRMatrix)): if size == self.shape[0]: return elif size % self.shape[0] == 0: return raise ValueError( "The shape '{0}' is incompatible with the number of rows of the sp" "arse matrix '{1}'.".format(shape, self.shape[0]))
def func(x): x = np.asarray(x) out = np.empty((ntot,) + x.shape[1:], x.dtype) nbytes = product(x.shape[1:]) * x.itemsize self.comm.Allgatherv( x.view(np.byte), [out.view(np.byte), ([_ * nbytes for _ in counts], [_ * nbytes for _ in offsets])]) return out
def __init__(self, mask, operator=MPI.SUM, commin=MPI.COMM_WORLD, **keywords): commout = commin shapeout = mask.size - int(np.sum(mask)) shapein = distribute_shape(mask.shape, comm=commin) Operator.__init__(self, shapein=shapein, shapeout=shapeout, commin=commin, commout=commout, **keywords) self.mask = mask self.chunk_size = product(mask.shape[1:]) self.operator = operator
def func(shape, degrees): angle = np.arange(product(shape)).reshape(shape) if degrees: angle_ = np.radians(angle) else: angle_ = angle angle_ = angle_.reshape(angle.size) r = Rotation2dOperator(angle, degrees=degrees) actual = r([1, 0]).reshape((angle.size, 2)) expected = np.array([np.cos(angle_), np.sin(angle_)]).T assert_same(actual, expected)
def _median(x, mask=None, axis=None): x = np.asarray(x) if mask is not None: mask = np.asarray(mask, dtype=bool) x = x.copy() x[mask] = np.nan if axis is None: return np.median(x[np.isfinite(x)]) if np.all(np.isfinite(x)): return np.median(x, axis=axis) slow = product(x.shape[:axis]) fast = product(x.shape[axis + 1:]) out = np.empty(x.shape[:axis] + x.shape[axis + 1:]) out_ = out.reshape((slow, fast)) x_ = x.reshape((slow, -1, fast)) for i in range(slow): for j in range(fast): b = x_[i, :, j] out_[i, j] = np.median(b[np.isfinite(b)]) return out
def _median(x, mask=None, axis=None): x = np.asarray(x) if mask is not None: mask = np.asarray(mask, dtype=bool) x = x.copy() x[mask] = np.nan if axis is None: return np.median(x[np.isfinite(x)]) if np.all(np.isfinite(x)): return np.median(x, axis=axis) slow = product(x.shape[:axis]) fast = product(x.shape[axis+1:]) out = np.empty(x.shape[:axis] + x.shape[axis+1:]) out_ = out.reshape((slow,fast)) x_ = x.reshape((slow,-1,fast)) for i in range(slow): for j in range(fast): b = x_[i,:,j] out_[i,j] = np.median(b[np.isfinite(b)]) return out
def gaussian(shape, sigma=None, fwhm=None, center=None, dtype=float): """ Returns an array whose values are the distances to a given center. Parameters ---------- shape : tuple of integer dimensions of the output array. For a 2d array, the first integer is for the Y-axis and the second one for the X-axis. fwhm : array-like The Full Width Half Maximum of the gaussian (fwhm_x, fwhm_y, ...). sigma : array-like The sigma parameter (sigma_x, sigma_y, ...) in pixel units. center : array-like, optional Center (x0, y0, ...) of the gaussian, in pixel units. By convention, the coordinates of the center of the pixel [0, 0] are (0, 0). Default is the image center. dtype : np.dtype, optional The output data type. """ if sigma is None and fwhm is None: raise ValueError('The shape of the gaussian is not specified.') shape = tointtuple(shape) n = len(shape) if sigma is None: sigma = fwhm / np.sqrt(8 * np.log(2)) if center is None: center = (np.array(shape[::-1], dtype) - 1) / 2 else: center = np.ascontiguousarray(center, dtype) if isscalarlike(sigma): sigma = np.resize(sigma, n).astype(dtype) else: sigma = np.ascontiguousarray(sigma, dtype) dtype = np.dtype(dtype) if n == 2 and dtype in (np.float32, np.float64): out = np.empty(shape, dtype) func = getattr(flib.datautils, 'gaussian_2d_r{0}'.format(dtype.itemsize)) func(out.T, center, sigma) else: scale = 1 / (np.sqrt(2) * sigma[::-1]) axes = np.ogrid[[ slice(-o * sc, (sh - 1 - o) * sc, complex(sh)) for o, sh, sc in zip(center[::-1], shape, scale) ]] out = 1 / ((2 * np.pi)**(n / 2) * product(sigma)) for a in axes: out = out * np.exp(-a**2) return out
def func(dtype, shape): n = max(product(shape), 1) size = np.arange(1, n + 1, dtype=dtype).reshape(shape) actual = create_square(size, center=origin, dtype=dtype) expected = np.empty(shape + (4, 2)) expected[..., 0, 0] = origin[0] + size / 2 expected[..., 0, 1] = origin[1] + size / 2 expected[..., 1, 0] = origin[0] - size / 2 expected[..., 1, 1] = origin[1] + size / 2 expected[..., 2, 0] = origin[0] - size / 2 expected[..., 2, 1] = origin[1] - size / 2 expected[..., 3, 0] = origin[0] + size / 2 expected[..., 3, 1] = origin[1] - size / 2 assert_same(actual, expected)
def gaussian(shape, sigma=None, fwhm=None, center=None, dtype=float): """ Returns an array whose values are the distances to a given center. Parameters ---------- shape : tuple of integer dimensions of the output array. For a 2d array, the first integer is for the Y-axis and the second one for the X-axis. fwhm : array-like The Full Width Half Maximum of the gaussian (fwhm_x, fwhm_y, ...). sigma : array-like The sigma parameter (sigma_x, sigma_y, ...) in pixel units. center : array-like, optional Center (x0, y0, ...) of the gaussian, in pixel units. By convention, the coordinates of the center of the pixel [0, 0] are (0, 0). Default is the image center. dtype : np.dtype, optional The output data type. """ if sigma is None and fwhm is None: raise ValueError('The shape of the gaussian is not specified.') shape = tointtuple(shape) n = len(shape) if sigma is None: sigma = fwhm / np.sqrt(8 * np.log(2)) if center is None: center = (np.array(shape[::-1], dtype) - 1) / 2 else: center = np.ascontiguousarray(center, dtype) if isscalarlike(sigma): sigma = np.resize(sigma, n).astype(dtype) else: sigma = np.ascontiguousarray(sigma, dtype) dtype = np.dtype(dtype) if n == 2 and dtype in (np.float32, np.float64): out = np.empty(shape, dtype) func = getattr(flib.datautils, 'gaussian_2d_r{0}'.format(dtype.itemsize)) func(out.T, center, sigma) else: scale = 1 / (np.sqrt(2) * sigma[::-1]) axes = np.ogrid[[slice(-o * sc, (sh - 1 - o) * sc, complex(sh)) for o, sh, sc in zip(center[::-1], shape, scale)]] out = 1 / ((2*np.pi)**(n / 2) * product(sigma)) for a in axes: out = out * np.exp(-a**2) return out
def todense(self, shapein=None, shapeout=None, inplace=False): shapein, shapeout = self._validate_shapes(shapein, shapeout) if shapein is None: raise ValueError("The operator's input shape is not explicit. Spec" "ify it with the 'shapein' keyword.") extra_size = product(shapein) // self.matrix.shape[1] if self.shapein is None and extra_size > 1: shapein = shapein[:-1] broadcasting = True else: broadcasting = False out = SparseBase.todense(self, shapein=shapein, inplace=inplace) if broadcasting: out = np.kron(out, np.eye(extra_size)) return out
def func(dtype, shape): n = max(product(shape), 1) size_x = np.arange(1, n + 1, dtype=dtype).reshape(shape) size_y = size_x / 2 actual = create_rectangle(np.asarray([size_x.T, size_y.T]).T, center=origin, dtype=dtype) expected = np.empty(shape + (4, 2)) expected[..., 0, 0] = origin[0] + size_x / 2 expected[..., 0, 1] = origin[1] + size_y / 2 expected[..., 1, 0] = origin[0] - size_x / 2 expected[..., 1, 1] = origin[1] + size_y / 2 expected[..., 2, 0] = origin[0] - size_x / 2 expected[..., 2, 1] = origin[1] - size_y / 2 expected[..., 3, 0] = origin[0] + size_x / 2 expected[..., 3, 1] = origin[1] - size_y / 2 assert_same(actual, expected)
def diffTdiff(input, output, axis=0, scalar=1., comm=None): """ Inplace discrete difference transpose times discrete difference """ if not isinstance(input, np.ndarray): raise TypeError('Input array is not an ndarray.') if input.dtype != var.FLOAT_DTYPE: raise TypeError('The data type of the input array is not ' + \ str(var.FLOAT_DTYPE.type) + '.') if axis < 0: raise ValueError("Invalid negative axis '" + str(axis) + "'.") if comm is None: comm = MPI.COMM_WORLD scalar = np.asarray(scalar, var.FLOAT_DTYPE) ndim = input.ndim if ndim == 0: output.flat = 0 return if axis >= ndim: raise ValueError("Invalid axis '" + str(axis) + "'. Expected value is" \ ' less than ' + str(ndim) + '.') inplace = input.__array_interface__['data'][0] == \ output.__array_interface__['data'][0] if axis != 0 or comm.size == 1: if input.size == 0: return tmf.operators.difftdiff(input.ravel(), output.ravel(), ndim-axis, np.asarray(input.T.shape), scalar, inplace) return if product(input.shape[1:]) == 0: return with filter_comm(input.shape[0] > 0, comm) as fcomm: if fcomm is not None: status = tmf.operators_mpi.difftdiff(input.ravel(), output.ravel(), ndim-axis, np.asarray(input.T.shape), scalar, inplace, fcomm.py2f()) if status != 0: raise RuntimeError()
def func(c, s, b, sameshapein, sameshapeout): x = np.arange(product(s)).reshape(s) op = get_operator(c, x, broadcast=b) if len(s) == 0 or c in (HomothetyOperator, IdentityOperator, ZeroOperator): assert_equal(op.broadcast, 'scalar') assert_is_none(op.shapein) assert_is_none(op.shapeout) elif b in ('leftward', 'rightward'): assert_equal(op.broadcast, b) assert_is_none(op.shapein) assert_is_none(op.shapeout) else: assert_equal(op.broadcast, 'disabled') if sameshapein: assert_equal(op.shapein, s) if sameshapeout: assert_equal(op.shapeout, s)
def _get_index(self, selection, ordering): selection = self._normalize_selection(selection, self._shape_actual) if ordering is not None: ordering = np.asarray(ordering) if ordering.shape != self._shape_actual: raise ValueError('Invalid ordering dimensions.') if ordering is None: # assuming row major storage for the ordering if selection is Ellipsis or isinstance(selection, slice): return selection if isinstance(selection, np.ndarray) and selection.dtype == int: out = np.asarray(selection, self._dtype_index) return self._normalize_int_selection(out, self._size_actual) if isinstance(selection, tuple): s = selection[0] if isinstance(s, slice) and \ (self.ndim == 1 or len(selection) == 1 and s.step == 1): n = product(self._shape_actual[1:]) return slice(s.start * n, s.stop * n, s.step) selection = self._selection2bool(selection) return self._normalize_int_selection( np.asarray(np.where(selection.ravel())[0], dtype=self._dtype_index), self._size_actual) if selection is not Ellipsis and \ (not isinstance(selection, np.ndarray) or selection.dtype == int): selection = self._selection2bool(selection) if selection is not Ellipsis: ordering = ordering.copy() ordering[~selection] = -1 npacked = np.sum(ordering >= 0) if npacked == 0: return np.array([], self._dtype_index) out = np.asarray( np.argsort(ordering.ravel(), kind='mergesort')[-npacked:], self._dtype_index) return self._normalize_int_selection(out, self._size_actual)
def _get_index(self, selection, ordering): selection = self._normalize_selection(selection, self._shape_actual) if ordering is not None: ordering = np.asarray(ordering) if ordering.shape != self._shape_actual: raise ValueError('Invalid ordering dimensions.') if ordering is None: # assuming row major storage for the ordering if selection is Ellipsis or isinstance(selection, slice): return selection if isinstance(selection, np.ndarray) and selection.dtype == int: out = np.asarray(selection, self._dtype_index) return self._normalize_int_selection(out, self._size_actual) if isinstance(selection, tuple): s = selection[0] if isinstance(s, slice) and \ (self.ndim == 1 or len(selection) == 1 and s.step == 1): n = product(self._shape_actual[1:]) return slice(s.start * n, s.stop * n, s.step) selection = self._selection2bool(selection) return self._normalize_int_selection( np.asarray(np.where(selection.ravel())[0], dtype=self._dtype_index), self._size_actual) if selection is not Ellipsis and \ (not isinstance(selection, np.ndarray) or selection.dtype == int): selection = self._selection2bool(selection) if selection is not Ellipsis: ordering = ordering.copy() ordering[~selection] = -1 npacked = np.sum(ordering >= 0) if npacked == 0: return np.array([], self._dtype_index) out = np.asarray(np.argsort( ordering.ravel(), kind='mergesort')[-npacked:], self._dtype_index) return self._normalize_int_selection(out, self._size_actual)
def test_gather(): n = 4 shapes = (n,), (n, n), (n, n, 2) ndims = 1, 2, 2 def func(shape, ndim, ordering, value): table = PackedTable(shape, ndim=ndim, ordering=ordering, value=value) table_local = table.scatter() table_global = table_local.gather() assert_equal(table_global.shape, table.shape) assert_equal(table_global.ndim, table.ndim) assert_equal(table_global._index, table._index) assert_equal(table_global.removed, table.removed) assert_equal(table_global.value, table.value) for shape, ndim in zip(shapes, ndims): bshape = shape[:ndim] ntot = product(bshape) o = np.arange(ntot).reshape(bshape) os = o, o.copy(), o.copy(), o.copy(), o.copy() os[1].ravel()[...] = os[1].ravel()[::-1] os[2].ravel()[-2:] = -1 os[3].ravel()[::2] = -1 os[4].ravel()[[0, 2, 3]] = -1 value1 = None value2 = 2.3 value3 = np.array(2.4) value4 = np.arange(product(bshape), dtype=np.int8).reshape(bshape) value5 = np.arange(product(bshape), dtype=complex).reshape(bshape) value6 = np.arange(product(bshape), dtype=complex).reshape(bshape) \ .view([('re', float), ('im', float)]) value7 = np.arange(product(bshape) * 3, dtype=np.int8) \ .reshape(bshape + (3,)) value8 = np.arange(product(bshape) * 3, dtype=complex) \ .reshape(bshape + (3,)) value9 = np.arange(product(bshape) * 3, dtype=complex) \ .reshape(bshape + (3,)).view([('re', float), ('im', float)]) for ordering in os: for value in (value1, value2, value3, value4, value5, value6, value7, value8, value9): yield func, shape, ndim, ordering, value
def test_gather(): n = 4 shapes = (n, ), (n, n), (n, n, 2) ndims = 1, 2, 2 def func(shape, ndim, ordering, value): table = PackedTable(shape, ndim=ndim, ordering=ordering, value=value) table_local = table.scatter() table_global = table_local.gather() assert_equal(table_global.shape, table.shape) assert_equal(table_global.ndim, table.ndim) assert_equal(table_global._index, table._index) assert_equal(table_global.removed, table.removed) assert_equal(table_global.value, table.value) for shape, ndim in zip(shapes, ndims): bshape = shape[:ndim] ntot = product(bshape) o = np.arange(ntot).reshape(bshape) os = o, o.copy(), o.copy(), o.copy(), o.copy() os[1].ravel()[...] = os[1].ravel()[::-1] os[2].ravel()[-2:] = -1 os[3].ravel()[::2] = -1 os[4].ravel()[[0, 2, 3]] = -1 value1 = None value2 = 2.3 value3 = np.array(2.4) value4 = np.arange(product(bshape), dtype=np.int8).reshape(bshape) value5 = np.arange(product(bshape), dtype=complex).reshape(bshape) value6 = np.arange(product(bshape), dtype=complex).reshape(bshape) \ .view([('re', float), ('im', float)]) value7 = np.arange(product(bshape) * 3, dtype=np.int8) \ .reshape(bshape + (3,)) value8 = np.arange(product(bshape) * 3, dtype=complex) \ .reshape(bshape + (3,)) value9 = np.arange(product(bshape) * 3, dtype=complex) \ .reshape(bshape + (3,)).view([('re', float), ('im', float)]) for ordering in os: for value in (value1, value2, value3, value4, value5, value6, value7, value8, value9): yield func, shape, ndim, ordering, value
def _normalize_selection(self, selection, shape): # return the selection as an array of int or bool, a slice, an Ellipsis # or a tuple. if isinstance(selection, list) and len(selection) == 0: return np.array([], self._dtype_index) if selection is None or selection is Ellipsis or \ isinstance(selection, tuple) and len(selection) == 0: return Ellipsis if isscalarlike(selection): return np.asarray([selection], self._dtype_index) if isinstance(selection, slice): return self._normalize_slice(selection, product(shape)) if isinstance(selection, (list, tuple)): selection_ = np.asarray(selection) if selection_.dtype == object: if len(selection) > len(shape): raise ValueError('Invalid selection dimensions.') selection = tuple( self._normalize_slice(_, s) if isinstance(_, slice) else _ for _, s in zip(selection, shape)) try: return selection[:ilast_is_not(selection, Ellipsis) + 1] except ValueError: return Ellipsis else: selection = selection_ elif not isinstance(selection, np.ndarray): raise TypeError('Invalid selection.') if selection.dtype not in (bool, int): raise TypeError('Invalid selection.') if selection.dtype == int: if selection.ndim != 1: raise ValueError('Index selection is not 1-dimensional.') elif selection.shape != shape: raise ValueError('Invalid boolean selection dimensions.') return selection
def _normalize_selection(self, selection, shape): # return the selection as an array of int or bool, a slice, an Ellipsis # or a tuple. if isinstance(selection, list) and len(selection) == 0: return np.array([], self._dtype_index) if selection is None or selection is Ellipsis or \ isinstance(selection, tuple) and len(selection) == 0: return Ellipsis if isscalarlike(selection): return np.asarray([selection], self._dtype_index) if isinstance(selection, slice): return self._normalize_slice(selection, product(shape)) if isinstance(selection, (list, tuple)): selection_ = np.asarray(selection) if selection_.dtype == object: if len(selection) > len(shape): raise ValueError('Invalid selection dimensions.') selection = tuple(self._normalize_slice(_, s) if isinstance(_, slice) else _ for _, s in zip(selection, shape)) try: return selection[:ilast_is_not(selection, Ellipsis)+1] except ValueError: return Ellipsis else: selection = selection_ elif not isinstance(selection, np.ndarray): raise TypeError('Invalid selection.') if selection.dtype not in (bool, int): raise TypeError('Invalid selection.') if selection.dtype == int: if selection.ndim != 1: raise ValueError('Index selection is not 1-dimensional.') elif selection.shape != shape: raise ValueError('Invalid boolean selection dimensions.') return selection
def _get_projection_operator(rotation, scene, nu, position, synthbeam, horn, primary_beam, verbose=True): ndetectors = position.shape[0] ntimes = rotation.data.shape[0] nside = scene.nside thetas, phis, vals = QubicInstrument._peak_angles( scene, nu, position, synthbeam, horn, primary_beam) ncolmax = thetas.shape[-1] thetaphi = _pack_vector(thetas, phis) # (ndetectors, ncolmax, 2) direction = Spherical2CartesianOperator('zenith,azimuth')(thetaphi) e_nf = direction[:, None, :, :] if nside > 8192: dtype_index = np.dtype(np.int64) else: dtype_index = np.dtype(np.int32) cls = { 'I': FSRMatrix, 'QU': FSRRotation2dMatrix, 'IQU': FSRRotation3dMatrix }[scene.kind] ndims = len(scene.kind) nscene = len(scene) nscenetot = product(scene.shape[:scene.ndim]) s = cls((ndetectors * ntimes * ndims, nscene * ndims), ncolmax=ncolmax, dtype=synthbeam.dtype, dtype_index=dtype_index, verbose=verbose) index = s.data.index.reshape((ndetectors, ntimes, ncolmax)) c2h = Cartesian2HealpixOperator(nside) if nscene != nscenetot: table = np.full(nscenetot, -1, dtype_index) table[scene.index] = np.arange(len(scene), dtype=dtype_index) def func_thread(i): # e_nf[i] shape: (1, ncolmax, 3) # e_ni shape: (ntimes, ncolmax, 3) e_ni = rotation.T(e_nf[i].swapaxes(0, 1)).swapaxes(0, 1) if nscene != nscenetot: np.take(table, c2h(e_ni).astype(int), out=index[i]) else: index[i] = c2h(e_ni) with pool_threading() as pool: pool.map(func_thread, xrange(ndetectors)) if scene.kind == 'I': value = s.data.value.reshape(ndetectors, ntimes, ncolmax) value[...] = vals[:, None, :] shapeout = (ndetectors, ntimes) else: if str(dtype_index) not in ('int32', 'int64') or \ str(synthbeam.dtype) not in ('float32', 'float64'): raise TypeError( 'The projection matrix cannot be created with types: {0} a' 'nd {1}.'.format(dtype_index, synthbeam.dtype)) func = 'matrix_rot{0}d_i{1}_r{2}'.format(ndims, dtype_index.itemsize, synthbeam.dtype.itemsize) getattr(flib.polarization, func)(rotation.data.T, direction.T, s.data.ravel().view(np.int8), vals.T) if scene.kind == 'QU': shapeout = (ndetectors, ntimes, 2) else: shapeout = (ndetectors, ntimes, 3) return ProjectionOperator(s, shapeout=shapeout)
def median(x, mask=None, axis=None, out=None): """ Return median of array. Parameters ---------- x : sequence or ndarray The input array. NaN values are discarded. Complex and floats of precision greater than 64 are not handled mask : ndarray, optional Boolean array mask whose True values indicate an element to be discarded. axis : {None, int}, optional Axis along which the medians are computed. The default (axis=None) is to compute the median along a flattened version of the array. out : ndarray, optional Alternative output array in which to place the result. It must have the same shape and buffer length as the expected output, but the type (of the output) will be cast if necessary. Returns ------- median : ndarray A new array holding the result (unless `out` is specified, in which case that array is returned instead). Examples -------- >>> a = np.array([[10, 7, 4], [3, 2, 1]]) >>> a array([[10, 7, 4], [ 3, 2, 1]]) >>> median(a) 3.0 >>> median(a, axis=1) array([[ 7.], [ 2.]]) """ x = np.array(x, copy=False, order='c', subok=True) shape = x.shape dtype = np.find_common_type([np.float64, x.dtype], []) if dtype != np.float64: raise TypeError("Invalid input type '{0}'.".format(dtype)) x = np.asanyarray(x, dtype) if mask is None and hasattr(x, 'mask'): mask = x.mask if mask is not None and mask.shape != x.shape: raise ValueError('Incompatible mask shape.') if mask is not None: mask = np.array(mask, dtype=bool, order='c', copy=False).view(np.int8) if axis is not None: slow = product(shape[:axis]) fast = product(shape[axis+1:]) x = x.reshape((slow,-1,fast)) if mask is not None: mask = mask.reshape((slow,-1,fast)).view(np.int8) if out is not None: if out.nbytes != slow * fast * dtype.itemsize: raise ValueError('Incompatible output buffer length.') if out.shape != shape[:axis] + shape[axis+1:]: raise ValueError('Incompatible output shape.') out.dtype = dtype else: out = np.empty(shape[:axis] + shape[axis+1:], dtype) out_ = out.reshape((slow,fast)) else: out = np.empty((), dtype) out_ = out if mask is axis is None: tmf.math.median(x.ravel(), out) elif axis is None: tmf.math.median_mask(x.ravel(), mask.ravel(), out) elif mask is None: tmf.math.median_axis(x.T, out_.T) else: tmf.math.median_mask_axis(x.T, mask.T, out_.T) if out.ndim == 0: out = out.flat[0] return out
def get_pointing_matrix(self, header, npixels_per_sample=0, method=None, downsampling=False, section=None, comm=MPI.COMM_WORLD): # let's get the pointing matrix associated to the section if npixels_per_sample not in (0, self.info.npixels_per_sample): raise ValueError('The npixels_per_sample value is incompatible wi' \ 'th the MADMAP1 file.') npixels_per_sample = self.info.npixels_per_sample # if no slice is provided, return a pointing matrix for each of them if section is None: return super(MadMap1Observation, self).get_pointing_matrix(header, npixels_per_sample, method) if method is None: method = 'default' method = method.lower() if method != 'default': raise ValueError("Invalid pointing matrix method '" + method + "'.") header = gather_fitsheader_if_needed(header, comm=comm) shape_input = tuple(header['NAXIS' + str(i + 1)] for i in range(header['NAXIS']))[::-1] if product(shape_input) > np.iinfo(np.int32).max: raise RuntimeError( 'The map is too large: pixel indices cannot be s' 'stored using 32 bits: {0}>{1}'.format(product(shape_input), np.iinfo(np.int32).max)) pointing = self.pointing[section.start:section.stop] ndetectors = self.get_ndetectors() nsamples = int(np.sum(~pointing.removed)) tod = np.empty((ndetectors, nsamples)) try: islice = list(self.slice.start).index(section.start) if self.slice[islice].stop != section.stop: raise ValueError() except ValueError: raise RuntimeError( 'Only observation slice can be specified through' ' the section keyword.') shape = (ndetectors, nsamples, npixels_per_sample) info = { 'header': header, 'method': method, 'outside': False, 'npixels_per_sample_min': npixels_per_sample } try: pmatrix = PointingMatrix.empty(shape, shape_input, info=info, verbose=False) except MemoryError: gc.collect() pmatrix = PointingMatrix.empty(shape, shape_input, info=info, verbose=False) status = tmf.madmap1_read_tod(self.info.todfile, self.info.invnttfile, self.info.convert, npixels_per_sample, islice + 1, tod.T, pmatrix.ravel().view(np.int64)) if status != 0: raise RuntimeError() return pmatrix
def _reshapeout(self, shape): extra_size = product(shape) // self.shape[0] if extra_size > 1 or shape[-1] == 1: return self.shape[1], extra_size return self.shape[1]
def func(dtype, shape): n = max(product(shape), 1) radius = np.arange(1, n + 1, dtype=dtype).reshape(shape) circle = create_circle(radius, center=origin, dtype=dtype) actual = np.sqrt(np.sum((circle - origin)**2, axis=-1)) assert_same(actual, radius[..., None], broadcasting=True)
def write_fits(filename, data, header, extension, extname, comm): """ Collectively write local arrays into a single FITS file. Parameters ---------- filename : str The FITS file name. data : ndarray The array to be written. header : pyfits.Header The data FITS header. None can be set, in which case a minimal FITS header will be inferred from the data. extension : boolean If True, the data will be written as an extension to an already existing FITS file. extname : str The FITS extension name. Use None to write the primary HDU. comm : mpi4py.Comm The MPI communicator of the local arrays. Use MPI.COMM_SELF if the data are not meant to be combined into a global array. Make sure that the MPI processes are not executing this routine with the same file name. """ # check if the file name is the same for all MPI jobs files = comm.allgather(filename+str(extname)) all_equal = all(f == files[0] for f in files) if comm.size > 1 and not all_equal: raise ValueError('The file name is not the same for all MPI jobs.') ndims = comm.allgather(data.ndim) if any(n != ndims[0] for n in ndims): raise ValueError("The arrays have an incompatible number of dimensions:" " '{0}'.".format(', '.join(str(n) for n in ndims))) ndim = ndims[0] shapes = comm.allgather(data.shape) if any(s[1:] != shapes[0][1:] for s in shapes): raise ValueError("The arrays have incompatible shapes: '{0}'.".format( strshape(shapes))) # get header if header is None: header = create_fitsheader(fromdata=data, extname=extname) else: header = header.copy() if extname is not None: header.update('extname', extname) # we remove the file first to avoid an annoying pyfits informative message if not extension: if comm.rank == 0: try: os.remove(filename) except OSError: pass # case without MPI communication if comm.size == 1: if not extension: hdu = pyfits.PrimaryHDU(data, header) hdu.writeto(filename, clobber=True) else: pyfits.append(filename, data, header) return # get global/local parameters nglobal = sum(s[0] for s in shapes) s = distribute_slice(nglobal) nlocal = s.stop - s.start if data.shape[0] != nlocal: raise ValueError("On rank {}, the local array shape '{}' is invalid. Th" "e first dimension does not match the expected local number '{}' gi" "ven the global number '{}'.{}".format(comm.rank, data.shape, nlocal, nglobal, '' if comm.rank > 0 else ' Shapes are: {}.'.format( shapes))) # write FITS header if comm.rank == 0: header['NAXIS' + str(ndim)] = nglobal shdu = pyfits.StreamingHDU(filename, header) data_loc = shdu._datLoc shdu.close() else: data_loc = None data_loc = comm.bcast(data_loc) # get a communicator excluding the processes which have no work to do # (Create_subarray does not allow 0-sized subarrays) chunk = product(data.shape[1:]) rank_nowork = min(comm.size, nglobal) group = comm.Get_group() group.Incl(range(rank_nowork)) newcomm = comm.Create(group) # collectively write data if comm.rank < rank_nowork: # mpi4py 1.2.2: pb with viewing data as big endian KeyError '>d' if sys.byteorder == 'little' and data.dtype.byteorder == '=' or \ data.dtype.byteorder == '<': data = data.byteswap() data = data.newbyteorder('=') mtype = DTYPE_MAP[data.dtype] ftype = mtype.Create_subarray([nglobal*chunk], [nlocal*chunk], [s.start*chunk]) ftype.Commit() f = MPI.File.Open(newcomm, filename, amode=MPI.MODE_APPEND | MPI.MODE_WRONLY | MPI.MODE_CREATE) f.Set_view(data_loc, mtype, ftype, 'native', MPI.INFO_NULL) f.Write_all(data) f.Close() ftype.Free() newcomm.Free() # pad FITS file with zeros if comm.rank == 0: datasize = nglobal * chunk * data.dtype.itemsize BLOCK_SIZE = 2880 padding = BLOCK_SIZE - (datasize % BLOCK_SIZE) with open(filename, 'a') as f: if f.tell() - data_loc != datasize: raise RuntimeError('Unexpected file size.') f.write(padding * '\0') comm.Barrier()
def __init__(self, flib_id, shape, block_shape, nmax, sparse_axis, dtype=None, dtype_index=None, data=None, verbose=False): self._flib_id = flib_id if not isinstance(shape, (list, tuple)): raise TypeError("Invalid shape '{0}'.".format(shape)) if len(shape) != 2: raise ValueError("The number of dimensions is not 2.") if len(block_shape) != 2: raise ValueError( "The number of dimensions of the blocks is not 2.") straxes = ('row', 'column') if sparse_axis == 1: straxes = straxes[::-1] if any(s % b != 0 for s, b in zip(shape, block_shape)): raise ValueError( "The shape of the matrix '{0}' is incompatible with blocks of " "shape '{1}'.".format(shape, block_shape)) if data is None: if nmax is None: raise ValueError('The maximum number of non-zero {0}s per {1} ' 'is not specified.'.format(*straxes)) shape_data = (shape[1 - sparse_axis] // block_shape[0], nmax) if dtype is None: dtype = float if dtype_index is None: dtype_index = int dtype_data = self._get_dtype_data(dtype, dtype_index, block_shape) data = empty(shape_data, dtype_data, verbose=verbose) elif 'index' not in data.dtype.names: raise TypeError('The structured array has no field index.') elif len(data.dtype.names) == 1: raise TypeError('The structured array has no data field.') elif product(data.shape[:-1]) * block_shape[1-sparse_axis] != \ shape[1-sparse_axis]: raise ValueError( "The shape of the matrix '{0}' is incompatible with that of th" "e structured array '{1}'.".format(shape, data.shape)) elif nmax not in (None, data.shape[-1]): raise ValueError( "The n{0}max keyword value '{1}' is incompatible with the shap" "e of the structured array '{2}'.".format( straxes[0][:3], nmax, data.shape)) else: dtype_index = data.dtype['index'] dtype = data.dtype[1] if dtype.type == np.void: dtype = dtype.subdtype[0].type expected = self._get_dtype_data(dtype, dtype_index, block_shape) if data.dtype != expected: raise TypeError( 'The input dtype {0} is invalid. Expected dtype is {1}.'. format(data.dtype, expected)) if nmax is None: nmax = data.shape[-1] self.dtype = np.dtype(dtype) self.data = data.view(np.recarray) self.ndim = 2 self.shape = tuple(shape) self.block_shape = tuple(block_shape) setattr(self, 'n' + straxes[0][:3] + 'max', int(nmax))
def _get_projection_operator( rotation, scene, nu, position, synthbeam, horn, primary_beam, verbose=True): if len(position.shape) == 2: position = position[None, ...] ndetectors = position.shape[0] npoints = position.shape[1] ntimes = rotation.data.shape[0] nside = scene.nside thetas, phis, vals = MultiQubicInstrument._peak_angles( scene, nu, position, synthbeam, horn, primary_beam) ncolmax = thetas.shape[-1] thetaphi = _pack_vector(thetas, phis) # (ndetectors, npoints, ncolmax, 2) direction = Spherical2CartesianOperator('zenith,azimuth')(thetaphi) e_nf = direction[..., None, :, :] if nside > 8192: dtype_index = np.dtype(np.int64) else: dtype_index = np.dtype(np.int32) cls = {'I': FSRMatrix, 'QU': FSRRotation2dMatrix, 'IQU': FSRRotation3dMatrix}[scene.kind] ndims = len(scene.kind) nscene = len(scene) nscenetot = product(scene.shape[:scene.ndim]) s = cls((ndetectors * npoints * ntimes * ndims, nscene * ndims), ncolmax=ncolmax, dtype=synthbeam.dtype, dtype_index=dtype_index, verbose=verbose) index = s.data.index.reshape((ndetectors, npoints, ntimes, ncolmax)) c2h = Cartesian2HealpixOperator(nside) if nscene != nscenetot: table = np.full(nscenetot, -1, dtype_index) table[scene.index] = np.arange(len(scene), dtype=dtype_index) e_nf = e_nf.reshape(-1, 1, ncolmax, 3) index = index.reshape(-1, ntimes, ncolmax) def func_thread(i): # e_nf[i] shape: (1, ncolmax, 3) # e_ni shape: (ntimes, ncolmax, 3) e_ni = rotation.T(e_nf[i].swapaxes(0, 1)).swapaxes(0, 1) if nscene != nscenetot: np.take(table, c2h(e_ni).astype(int), out=index[i]) else: index[i] = c2h(e_ni) with pool_threading() as pool: pool.map(func_thread, xrange(ndetectors * npoints)) e_nf = e_nf.reshape(ndetectors, npoints, 1, ncolmax, 3) index = index.reshape(ndetectors, npoints, ntimes, ncolmax) if scene.kind == 'I': value = s.data.value.reshape( ndetectors, npoints, ntimes, ncolmax) value[...] = vals[..., None, :] shapeout = (ndetectors, npoints, ntimes) else: if str(dtype_index) not in ('int32', 'int64') or \ str(synthbeam.dtype) not in ('float32', 'float64'): raise TypeError( 'The projection matrix cannot be created with types:' '{0} and {1}.'.format(dtype_index, synthbeam.dtype)) direction_ = direction.reshape( ndetectors * npoints, ncolmax, 3) vals_ = vals.reshape(ndetectors * npoints, ncolmax) func = 'matrix_rot{0}d_i{1}_r{2}'.format( ndims, dtype_index.itemsize, synthbeam.dtype.itemsize) getattr(flib.polarization, func)( rotation.data.T, direction_.T, s.data.ravel().view(np.int8), vals_.T) if scene.kind == 'QU': shapeout = (ndetectors, npoints, ntimes, 2) else: shapeout = (ndetectors, npoints, ntimes, 3) return ProjectionOperator(s, shapeout=shapeout)
def test_product(): def func(o): assert o == 1 for o in ([], (), (1,), [1], [2, 0.5], (2, 0.5), np.array(1), np.array([2, 0.5])): yield func, product(o)
def func(shape): vec = np.arange(product(shape)).reshape(shape) exp = vec / np.sqrt(np.sum(vec ** 2, axis=-1))[..., None] assert_same(n(vec), exp)