def _get_unit_conversion_operator(self, nu): """ Convert sky temperature into W / m^2 / Hz / pixel. If the scene has been initialised with the 'absolute' keyword, the scene is assumed to include the CMB background and the fluctuations (in Kelvin) and the operator follows the non-linear Planck law. Otherwise, the scene only includes the fluctuations (in microKelvin) and the operator is linear (i.e. the output also corresponds to power fluctuations). """ if not self.temperature: return IdentityOperator() # solid angle of a sky pixel omega = 4 * np.pi / self.shape[0] a = 2 * omega * h * nu**3 / c**2 if self.absolute: hnu_k = h * nu / k return a / asoperator(np.expm1)(hnu_k * ReciprocalOperator()) T = self.T_cmb hnu_kT = h * nu / (k * T) val = 1e-6 * a * hnu_kT * np.exp(hnu_kT) / (np.expm1(hnu_kT)**2 * T) return asoperator(val)
def __init__(self, shape, topixel=None, ndim=None, dtype=float, **keywords): """ Parameters ---------- shape : tuple of integers The surface shape. ndim : int, optional The number of splittable (indexable) dimensions. It is the actual number of dimensions of the layout. It can be lower than that specified by the layout shape, in which case the extra dimensions are instructed not to be split. topixel : Operator, optional World-to-pixel coordinate transform. """ PackedTable.__init__(self, shape, ndim=ndim) if topixel is None: topixel = IdentityOperator(self.shape[:self.ndim]) else: topixel = asoperator(topixel) self.dtype = np.dtype(dtype) self.topixel = topixel self.toworld = topixel.I for k, v in keywords.items(): setattr(self, k, v)
def test_rule_right(): ids = (IdentityOperator(classout=ndarray2, attrout=attr2), IdentityOperator(shapein=4, classout=ndarray2, attrout=attr2)) def func(id_, op_): op = id_(op_) assert_is_type(op, type(op_)) attr = {} assert_is(op.classout, id_.classout) attr.update(op_.attrout) attr.update(id_.attrout) assert_eq(op.attrout, attr) assert_eq(op.flags.linear, op_.flags.linear) assert_eq(op.flags.contiguous_input, op_.flags.contiguous_input) for id_ in ids: for op_ in ops: yield func, id_, op_
def get_filter_operator(self): """ Return the filter operator. Convert units from W/Hz to W. """ if self.filter.bandwidth == 0: return IdentityOperator() return HomothetyOperator(self.filter.bandwidth)
def func(opout, opin, idin): if opin is not None and idin is not None and opin != idin: return p = Op(shapeout=opout, shapein=opin) * IdentityOperator(shapein=idin) if idin is None: idin = opin assert_is_instance(p, Op) assert_eq(p.shapein, idin) assert_eq(p.shapeout, opout)
def get_hwp_operator(self, sampling, scene): """ Return the rotation matrix for the half-wave plate. """ shape = (len(self), len(sampling)) if scene.kind == 'I': return IdentityOperator(shapein=shape) if scene.kind == 'QU': return Rotation2dOperator(-4 * sampling.angle_hwp, degrees=True, shapein=shape + (2,)) return Rotation3dOperator('X', -4 * sampling.angle_hwp, degrees=True, shapein=shape + (3,))
def get_detector_response_operator(self, sampling, tau=None): """ Return the operator for the bolometer responses. """ if tau is None: tau = self.detector.tau sampling_period = sampling.period shapein = len(self), len(sampling) if sampling_period == 0: return IdentityOperator(shapein) return ConvolutionTruncatedExponentialOperator(tau / sampling_period, shapein=shapein)
def test_diagonal2(): ops = (DiagonalOperator([1., 2], broadcast='rightward'), DiagonalOperator([[2., 3, 4], [5, 6, 7]], broadcast='rightward'), DiagonalOperator([1., 2, 3, 4, 5], broadcast='leftward'), DiagonalOperator(np.arange(20).reshape(4, 5), broadcast='leftward'), DiagonalOperator(np.arange(120.).reshape(2, 3, 4, 5)), HomothetyOperator(7.), IdentityOperator()) x = np.arange(120.).reshape(2, 3, 4, 5) / 2 def func(cls, d1, d2): op = {AdditionOperator: operator.add, CompositionOperator: operator.mul, MultiplicationOperator: operator.mul}[cls] d = cls([d1, d2]) if type(d1) is DiagonalOperator: assert_is_type(d, DiagonalOperator) elif type(d1) is HomothetyOperator: assert_is_type(d, HomothetyOperator) elif op is CompositionOperator: assert_is_type(d, IdentityOperator) else: assert_is_type(d, HomothetyOperator) data = op(d1.data.T, d2.data.T).T \ if 'rightward' in (d1.broadcast, d2.broadcast) \ else op(d1.data, d2.data) assert_same(d.data, data) if cls is CompositionOperator: assert_same(d(x), d1(d2(x))) else: assert_same(d(x), op(d1(x), d2(x))) for op in (AdditionOperator, CompositionOperator):#, MultiplicationOperator): for d1, d2 in itertools.combinations(ops, 2): if set((d1.broadcast, d2.broadcast)) == \ set(('leftward', 'rightward')): continue yield func, op, d1, d2
def __init__(self, band, scene, true_sky=None, factor=1, fwhm=0, mask=None, convolution_operator=None): """ Parameters ---------- band : int The band 150 or 220. scene : Scene The acquisition scene. true_sky : array of shape (npixel,) or (npixel, 3) The true CMB sky (temperature or polarized). The Planck observation will be this true sky plus a random independent gaussian noise realization. factor : 1 or 3 floats, optional The factor by which the Planck standard deviation is multiplied. fwhm : float, optional, !not used! The fwhm of the Gaussian used to smooth the map [radians]. mask : array of shape (npixel, ) or None The boolean mask, which is equal True at pixels where we need Planck, that is outside the QUBIC field. """ if band not in (150, 220): raise ValueError("Invalid band '{}'.".format(band)) if true_sky is None: raise ValueError('The Planck Q & U maps are not released yet.') if scene.kind == 'IQU' and true_sky.shape[-1] != 3: raise TypeError('The Planck sky shape is not (npix, 3).') true_sky = np.array(hp.ud_grade(true_sky.T, nside_out=scene.nside), copy=False).T if scene.kind == 'IQU' and true_sky.shape[-1] != 3: raise TypeError('The Planck sky shape is not (npix, 3).') self.scene = scene self.fwhm = fwhm self._true_sky = true_sky if mask is not None: self.mask = mask else: self.mask = np.ones(scene.npixel, dtype=np.bool) if band == 150: filename = 'Variance_Planck143GHz_Kcmb2_ns256.fits' else: filename = 'Variance_Planck217GHz_Kcmb2_ns256.fits' sigma = 1e6 * factor * np.sqrt(FitsArray(PATH + filename)) if scene.kind == 'I': sigma = sigma[:, 0] elif scene.kind == 'QU': sigma = sigma[:, :2] if self.scene.nside != 256: sigma = np.array(hp.ud_grade(sigma.T, self.scene.nside, power=2), copy=False).T self.sigma = sigma if convolution_operator is None: self.C = IdentityOperator() else: self.C = convolution_operator
def test_block_row1(): I2 = IdentityOperator(2) I3 = IdentityOperator(3) assert_raises(ValueError, BlockRowOperator, [I2, 2 * I3], axisin=0) assert_raises(ValueError, BlockRowOperator, [I2, 2 * I3], new_axisin=0)
Operator, AdditionOperator, CompositionOperator, DiagonalOperator, HomothetyOperator, IdentityOperator, MultiplicationOperator, PyOperatorsWarning) from pyoperators.flags import linear from pyoperators.rules import BinaryRule, UnaryRule, RuleManager, rule_manager from pyoperators.utils import ndarraywrap from pyoperators.utils.testing import ( assert_eq, assert_is, assert_is_none, assert_is_not_none, assert_is_instance) from .common import OPS, ndarray2, attr2 op = Operator() ops = [OP() for OP in OPS] ids_left = (IdentityOperator(classout=ndarray2, attrout=attr2), IdentityOperator(shapein=4, classout=ndarray2, attrout=attr2)) ids_right = (IdentityOperator(classout=ndarray2, attrout=attr2), IdentityOperator(shapein=3, classout=ndarray2, attrout=attr2)) class Operator1(Operator): pass class Operator2(Operator): pass class Operator3(Operator): pass
def func(s, v): solution = s(IdentityOperator(shapein=v.shape), v, x0=v) assert_same(solution['nit'], 0) assert_same(solution['x'], v)
import scipy from pyoperators import DiagonalOperator, IdentityOperator, MaskOperator from tamasis import (PacsObservation, CompressionAverageOperator, UnpackOperator, mapper_ls, mapper_naive) pyoperators.memory.verbose = False profile = None#'test_ls.png' solver = scipy.sparse.linalg.bicgstab tol = 1.e-6 if profile else 1.e-4 maxiter = 10 data_dir = os.path.dirname(__file__) + '/data/' obs = PacsObservation(data_dir + 'frames_blue.fits', fine_sampling_factor=1, reject_bad_line=False) tod = obs.get_tod() telescope = IdentityOperator() projection = obs.get_projection_operator(downsampling=True,npixels_per_sample=6) compression = CompressionAverageOperator(obs.slice.compression_factor) masking_tod = MaskOperator(tod.mask) masking_map = MaskOperator(projection.get_mask()) model = masking_tod * projection * telescope * masking_map # naive map map_naive = mapper_naive(tod, model) # iterative map, restricting oneself to observed map pixels unpacking = UnpackOperator(projection.get_mask()) old_settings = np.seterr(divide='ignore') M = DiagonalOperator(unpacking.T(1./map_naive.coverage)) np.seterr(**old_settings)
def mapper_rls(y, H, invntt=None, unpacking=None, hyper=1.0, x0=None, tol=1.e-5, maxiter=300, M=None, solver=None, verbose=True, callback=None, criterion=True, profile=None): """ Solve the linear equation y = H(x) where H is an Operator representing the acquisition, and y is the observed time series. x is the regularised least square solution as given by: x = argmin (Hx-y)^T N^-1 (Hx-y) + h ||D(x)||^2 or: x = (H^T N^-1 H + h D1^T D1 + h D2^T D2)^-1 H^T N^-1 y """ comm_map = H.commin or MPI.COMM_WORLD comm_tod = H.commout or comm_map if solver is None: solver = cg new_solver = solver in (pcg, ) tod = _validate_tod(y) ntods_ = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \ else tod.size nmaps_ = unpacking.shape[1] if unpacking is not None else None if nmaps_ is None: nmaps_ = H.shape[1] if nmaps_ is None: raise ValueError('The model H has not an explicit input shape.') ntods = comm_tod.allreduce(ntods_) nmaps = comm_map.allreduce(nmaps_) # get A if invntt is None: invntt = IdentityOperator() A = H.T * invntt * H npriors = len(H.shapein) priors = [ DiscreteDifferenceOperator(axis=axis, shapein=H.shapein, commin=comm_map) for axis in range(npriors) ] if comm_map.rank == 0 or comm_map.size > 1: A += sum((hyper * ntods / nmaps) * p.T * p for p in priors) # get b b = (H.T * invntt)(tod) if not np.all(np.isfinite(b)): raise ValueError('RHS contains non-finite values.') if b.shape != A.shapein: raise ValueError("Incompatible size for RHS: '" + str(b.size) + "' instead of '" + str(A.shape[1]) + "'.") if np.min(b) == np.max(b) == 0: print('Warning: in equation Ax=b, b is zero.') # unpack input if unpacking is None: unpacking = IdentityOperator() A = unpacking.T * A * unpacking b = unpacking.T(b) if x0 is not None: x0 = unpacking.T(x0) if not new_solver and solver is not cg: b = b.ravel() if x0 is not None: x0 = x0.ravel() if M is not None: if isinstance(M, DiagonalOperator): filter_nonfinite(M.data, out=M.data) M = unpacking.T * M * unpacking H_ = H * unpacking priors = [p * unpacking for p in priors] # criterion if hyper != 0: hc = np.hstack([1, npriors * [hyper]]) / ntods else: hc = [1 / ntods] norms = [norm2_ellipsoid(invntt)] + npriors * [norm2] comms = [comm_tod] + npriors * [comm_map] def criter(x): rs = [H_.matvec(x) - tod.view(np.ndarray).ravel() ] + [p.matvec(x) for p in priors] Js = [h * n(r, comm=c) for h, n, r, c in zip(hc, norms, rs, comms)] return Js if callback is None: if new_solver: callback = NewCgCallback(disp=verbose, comm=comm_map) else: callback = CgCallback(verbose=verbose, objfunc=criter if criterion else None) if (verbose or profile) and comm_map.rank == 0: print('') print('H.T * N^-1 * H:') print(repr(A)) if M is not None: print('Preconditioner:') print(M) time0 = time.time() if profile is not None: def run(): result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter) if new_solver: solution = result['x'] if not result['success']: print('Solver failure: ' + result['message']) else: solution, info = result if info != 0: print('Solver failure: info=' + str(info)) cProfile.runctx('run()', globals(), locals(), profile + '.prof') print('Profile time: ' + str(time.time() - time0)) os.system('python -m gprof2dot -f pstats -o ' + profile + '.dot ' + profile + '.prof') os.system('dot -Tpng ' + profile + '.dot > ' + profile) os.system('rm -f ' + profile + '.prof' + ' ' + profile + '.dot') return None result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter, callback=callback) time0 = time.time() - time0 if new_solver: solution = result['x'] if not result['success']: print('Solver failure: ' + result['message']) niterations = result['niterations'] error = result['error'] else: solution, info = result if info < 0: raise RuntimeError('Solver failure (code=' + str(info) + ' after ' + str(callback.niterations) + ' iterations).') if info > 0 and comm_map.rank == 0: print( 'Warning: Solver reached maximum number of iterations without' ' reaching specified tolerance.') niterations = getattr(callback, 'niterations', None) error = getattr(callback, 'residual', None) Js = criter(solution) if isinstance(unpacking, IdentityOperator): solution = solution.reshape(H.shapein) else: solution = unpacking(solution) tod[...] = 1 coverage = H.T(tod) unit = getattr(coverage, 'unit', None) derived_units = getattr(coverage, 'derived_units', None) coverage = coverage.view(Map) header = getattr(coverage, 'header', None) if header is None: header = create_fitsheader(fromdata=coverage) header.update('likeliho', Js[0]) header.update('criter', sum(Js)) header.update('hyper', hyper) header.update('nsamples', ntods) header.update('npixels', nmaps) header.update('time', time0) if niterations is not None: header.update('niter', niterations) header.update('maxiter', maxiter) if error is not None: header.update('error', error) header.update('tol', tol) header.update('solver', solver.__name__) output = Map(solution, header=header, coverage=coverage, unit=unit, derived_units=derived_units, copy=False) return output
def mapper_naive(tod, model, unit=None): """ Returns a naive map, i.e.: map = model.T(tod) / model.T(1) This equation is valid for a map and a Time Ordered Data (TOD) expressed as a surface brightness, so when the TOD does not meet this requirement and is a quantity per detector, a unit conversion is attempted. Parameters ---------- tod : Tod The input Time Ordered Data model : Operator The instrument model such as tod = model(map) unit : string Output map unit. By default, the output map unit is chosen to be compatible with the model (usually pixel^-1) """ # apply mask if hasattr(tod, 'mask') and tod.mask is not None: mask = MaskOperator(tod.mask) else: mask = IdentityOperator() tod = mask(tod) # get tod units if not hasattr(tod, '_unit') or len(tod._unit) == 0: attr = {'_unit': {'?': 1.}} model.propagate_attributes(None, attr) u = getattr(attr, '_unit', {}) if 'detector' in u and u['detector'] == -1: u = {'detector': -1.} elif u == {'?': 1.}: u = {} elif len(u) > 1: raise ValueError( 'The timeline units are not known and cannot be in' 'ferred from the model.') tod_du = getattr(attr, '_derived_units', {}) tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False) else: attr = {'_unit': {'?': 1}} model.T.propagate_attributes(None, attr) u = attr['_unit'] if 'detector' not in tod._unit and 'detector' in u and u[ 'detector'] == 1: raise ValueError("The model is incompatible with input units '{0}'"\ .format(tod.unit)) tod_unit = tod._unit tod_du = tod._derived_units # make sure the input is a surface brightness if 'detector' in tod._unit: tod.inunit(tod.unit + ' detector / arcsec^2') elif 'detector_reference' in tod._unit: tod.inunit(tod.unit + ' detector_reference / arcsec^2') # compute model.T(tod)/model.T(one) mymap = model.T(tod.magnitude) tod[...] = 1 mask(tod, tod) map_weights = model.T(tod.magnitude) old_settings = np.seterr(divide='ignore', invalid='ignore') mymap /= map_weights mymap.unit = tod.unit np.seterr(**old_settings) mymap.coverage = Map(map_weights.magnitude, header=mymap.header, copy=False) if unit is not None: mymap.inunit(unit) return mymap # set map units according to model attr = {'_unit': tod_unit, '_derived_units': tod_du} model.T.propagate_attributes(None, attr) if '_derived_units' in attr: mymap.derived_units = attr['_derived_units'] if '_unit' in attr: mymap.inunit(attr['_unit']) return mymap
def test_block_column1(): I2 = IdentityOperator(2) I3 = IdentityOperator(3) assert_raises(ValueError, BlockColumnOperator, [I2, 2 * I3], axisout=0) assert_raises(ValueError, BlockColumnOperator, [I2, 2 * I3], new_axisout=0)
def cg(A, b, x0=None, tol=1.e-5, maxiter=300, M=None, disp=False, callback=None): """ OpenMPI/MPI hybrid conjugate gradient solver with preconditioning. """ A = asoperator(A) comm = A.commin or MPI.COMM_WORLD if M is None: M = IdentityOperator() M = asoperator(M) maxRelError = tol**2 shape = b.shape x = np.empty(shape) d = np.empty(shape) q = np.empty(shape) r = np.empty(shape) s = np.empty(shape) xfinal = np.zeros(shape) if x0 is None: x[...] = 0 else: x[...] = x0 norm = norm2(b, comm=comm) if norm == 0: return xfinal, 0 r[...] = b r -= A(x) epsilon = norm2(r, comm=comm) / norm minEpsilon = epsilon M(r, d) delta0 = dot(r, d, comm=comm) deltaNew = delta0 for iter_ in xrange(maxiter): if epsilon <= maxRelError: break A(d, q) alpha = deltaNew / dot(d, q, comm=comm) x += alpha * d r -= alpha * q epsilon = norm2(r, comm=comm) / norm if disp: print '{0:4}: {1}'.format(iter_ + 1, np.sqrt(epsilon)) if callback is not None: resid = np.sqrt(epsilon) callback(x) if epsilon < minEpsilon: xfinal[...] = x minEpsilon = epsilon M(r, s) deltaOld = deltaNew deltaNew = dot(r, s, comm=comm) beta = deltaNew / deltaOld d *= beta d += s minEpsilon = np.sqrt(minEpsilon) maxRelError = np.sqrt(maxRelError) return xfinal.reshape(b.shape), int(minEpsilon > maxRelError)
def _rule_left_unpack(self, op): if self.mask.shape != op.mask.shape: return if np.any(self.mask != op.mask): return return IdentityOperator()
datadir = os.getenv('PACS_DATA', '') + '/transpScan/' datafile = [ datadir + '1342184598_blue_PreparedFrames.fits', datadir + '1342184599_blue_PreparedFrames.fits' ] if not all(map(os.path.exists, datafile)): print('The data files are not found: ' + ', '.join(datafile)) exit(0) pacs = PacsObservation( [datafile[0] + '[6065:20000]', datafile[1] + '[6066:20001]'], fine_sampling_factor=1, calblock_extension_time=0.) telescope = IdentityOperator() projection = pacs.get_projection_operator(resolution=3.2, npixels_per_sample=5) multiplexing = CompressionAverageOperator(1) crosstalk = IdentityOperator() compression = CompressionAverageOperator(4) model = compression * crosstalk * multiplexing * projection * telescope # read the Tod off the disk tod40Hz = pacs.get_tod(flatfielding=True, subtraction_mean=True) # remove drift tod40Hz_filtered = filter_polynomial(tod40Hz, 6) drift = tod40Hz - tod40Hz_filtered tod40Hz = filter_median(tod40Hz_filtered, 10000)
def _rule_identity(o1, o2): if o1.nside == o2.nside and o1.nest == o2.nest: return IdentityOperator()