def func(d, e): m = Map(np.array([1, 2, 3], dtype=d), coverage=np.array([0, 1, 0], dtype=d), error=np.array([2, 2, 2], dtype=d)) m2 = m.astype(e) assert m2.dtype == e assert m2.coverage.dtype == e assert m2.error.dtype == e
def test(): assert map_rls.header['NITER'] < 49 ref = Map(data_dir + 'frames_blue_map_rls_cgs_tol1e-6.fits') ref.derived_units = map_rls.derived_units cov = ref.coverage > 80 assert all_eq(ref[cov], map_rls[cov], 1.e-1) cov = ref.coverage > 125 assert all_eq(ref[cov], map_rls[cov], 1.e-2)
def test_detector_policy(): map_naive_ref = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') obs = PacsObservation(data_dir + 'frames_blue.fits', reject_bad_line=False) obs.pointing.chop[:] = 0 projection = obs.get_projection_operator(header=map_naive_ref.header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() masking = MaskOperator(tod.mask) model = masking * projection map_naive = mapper_naive(tod, model) assert all_eq(map_naive, map_naive_ref, tol) obs_rem = PacsObservation(data_dir + 'frames_blue.fits', policy_bad_detector='remove', reject_bad_line=False) obs_rem.pointing.chop[:] = 0 projection_rem = obs_rem.get_projection_operator(header=map_naive.header, downsampling=True, npixels_per_sample=7) tod_rem = obs_rem.get_tod() masking_rem = MaskOperator(tod_rem.mask) model_rem = masking_rem * projection_rem map_naive_rem = mapper_naive(tod_rem, model_rem) assert all_eq(map_naive, map_naive_rem, tol)
def test_header(): obs = PacsObservation(data_dir+'frames_blue.fits', reject_bad_line=False) obs.pointing.chop = 0 header = obs.get_map_header() projection = obs.get_projection_operator(header=header, downsampling=True, npixels_per_sample=6) tod = obs.get_tod() map_naive = mapper_naive(tod, projection) header2 = header.copy() header2['NAXIS1'] += 500 header2['CRPIX1'] += 250 projection2 = obs.get_projection_operator(header=header2, downsampling=True) map_naive2 = mapper_naive(tod, MaskOperator(tod.mask) * projection2) map_naive2.inunit('Jy/arcsec^2') map_naive3 = map_naive2[:,250:header['NAXIS1']+250] assert all_eq(map_naive.tounit('Jy/arcsec^2'), map_naive3, 2.e-7) # test compatibility with photproject tod = obs.get_tod() map_naive4 = mapper_naive(tod, projection) hdu_ref = pyfits.open(data_dir + 'frames_blue_map_hcss_photproject.fits')[1] map_ref = Map(hdu_ref.data, hdu_ref.header, unit=hdu_ref.header['qtty____']+'/pixel') std_naive = np.std(map_naive4[40:60,40:60]) std_ref = np.std(map_ref[40:60,40:60]) assert abs(std_naive-std_ref) / std_ref < 0.025
def test_ls(): H = projection m = pcg(H.T * invntt * H, (H.T*invntt)(tod), M=M, tol=1e-7) map_ls_packed = Map(m['x']) map_ls_packed.header['TIME'] = m['time'] #print('Elapsed time:', map_ls_packed.header['TIME']) #from tamasis import mapper_ls #map_ls_packed = mapper_ls(tod, projection, invntt=invntt, tol=1e-7, M=M, # callback=callback, criterion=False, # profile=profile) if profile: return print('Elapsed time:', map_ls_packed.header['TIME']) assert m['nit'] < 50 ref = packing(Map(path + 'madmapSpirePSW.fits')) assert_allclose(map_ls_packed, ref, atol=1e-5)
from tamasis import (PacsInstrument, PacsObservation, UnpackOperator, mapper_naive) from tamasis.utils import assert_all_eq data_dir = os.path.dirname(__file__) + '/data/' rank = MPI.COMM_WORLD.rank size = MPI.COMM_WORLD.size tamasis.var.verbose = True PacsInstrument.info.CALFILE_BADP = tamasis.var.path + '/pacs/PCalPhotometer_Ba'\ 'dPixelMask_FM_v5.fits' PacsInstrument.info.CALFILE_RESP = tamasis.var.path + '/pacs/PCalPhotometer_Re'\ 'sponsivity_FM_v5.fits' map_ref_local = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits') mask_ref_local = map_ref_local.coverage == 0 header_ref_local = map_ref_local.header map_ref_global = Map(data_dir + '../../../core/test/data/frames_blue_map_naive.fits', comm=MPI.COMM_SELF) mask_ref_global = map_ref_global.coverage == 0 header_ref_global = map_ref_global.header def check_map_global(m): assert_all_eq(m.magnitude, map_ref_global.magnitude, 1e-10) assert_all_eq(m.coverage, map_ref_global.coverage, 1e-10) assert_all_eq(m.tounit('Jy/arcsec^2'), map_ref_global.tounit('Jy/arcsec^2'), 1e-10) def check_map_local(m): assert_all_eq(m.magnitude, map_ref_local.magnitude, 1e-10)
def mapper_nl(tod, model, unpacking=None, priors=[], hypers=[], norms=[], comms=[], x0=None, tol=1.e-6, maxiter=300, solver=None, linesearch=None, descent_method='pr', M=None, verbose=True, callback=None, profile=None): comm_map = model.commin or MPI.COMM_WORLD comm_tod = model.commout or comm_map tod = _validate_tod(tod) if len(priors) == 0 and len(hypers) != 0: npriors = len(model.shapein) priors = [ DiscreteDifferenceOperator(axis=axis, shapein=model.shapein, commin=comm_map) for axis in range(npriors) ] else: npriors = len(priors) if np.isscalar(hypers): hypers = npriors * [hypers] elif npriors != len(hypers): raise ValueError('There should be one hyperparameter per prior.') if len(norms) == 0: norms = (npriors + 1) * [norm2] if len(comms) == 0: comms = [comm_tod] + npriors * [comm_map] if isinstance(M, DiagonalOperator): filter_nonfinite(M.data, out=M.data) hypers = np.asarray(hypers, dtype=var.FLOAT_DTYPE) ntods = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \ else tod.size ntods = comm_tod.allreduce(ntods) nmaps = comm_map.allreduce(model.shape[1]) hypers /= nmaps hypers = np.hstack([1 / ntods, hypers]) y = tod.view(np.ndarray).ravel() class ObjectiveFunction(Function): def __init__(self): pass def __call__(self, x, inwork=None, outwork=None, out=None): outwork = self.set_outwork(outwork, self.get_work(x)) return self.set_out(out, [ h * n(r,comm=c) for h, n, r, c in \ zip(hypers, norms, outwork, comms) ]) def D(self, x, inwork=None, outwork=None, out=None): inwork = self.get_work(x, inwork) return self.set_out( out, sum([ h * M.T * n.D(r, comm=c) for h, M, n, r, c in zip(hypers, [model] + priors, norms, inwork, comms) ])) def get_work(self, x, work=None): if work is not None: return work if unpacking is not None: x = unpacking.matvec(x) return [model * x - y] + [p * x for p in priors] objfunc = ObjectiveFunction() if linesearch is None: linesearch = QuadraticStep(hypers, norms, [model] + priors, comms, unpacking, comm_map) if callback is None: callback = CgCallback(verbose=verbose) time0 = time.time() solution = nlcg(objfunc, model.shape[1], M=M, maxiter=maxiter, tol=tol, descent_method=descent_method, linesearch=linesearch, callback=callback, comm=comm_map) time0 = time.time() - time0 Js = objfunc(solution) solution = unpacking(solution) tod[...] = 1 coverage = model.T(tod) header = getattr(coverage, 'header', None) if header is None: header = create_fitsheader(fromdata=coverage) unit = getattr(coverage, 'unit', None) derived_units = getattr(coverage, 'derived_units', None) coverage = Map(coverage.magnitude, header=header, copy=False) header.update('likeliho', Js[0]) header.update('criter', sum(Js)) header.update('hyper', str(hypers)) header.update('nsamples', ntods) header.update('npixels', nmaps) header.update('time', time0) if hasattr(callback, 'niterations'): header.update('niter', callback.niterations) header.update('maxiter', maxiter) if hasattr(callback, 'residual'): header.update('residual', callback.residual) header.update('tol', tol) header.update('solver', 'nlcg') output = Map(solution.reshape(model.shapein), header=header, coverage=coverage, unit=unit, derived_units=derived_units, copy=False) return output
def mapper_naive(tod, model, unit=None): """ Returns a naive map, i.e.: map = model.T(tod) / model.T(1) This equation is valid for a map and a Time Ordered Data (TOD) expressed as a surface brightness, so when the TOD does not meet this requirement and is a quantity per detector, a unit conversion is attempted. Parameters ---------- tod : Tod The input Time Ordered Data model : Operator The instrument model such as tod = model(map) unit : string Output map unit. By default, the output map unit is chosen to be compatible with the model (usually pixel^-1) """ # apply mask if hasattr(tod, 'mask') and tod.mask is not None: mask = MaskOperator(tod.mask) else: mask = IdentityOperator() tod = mask(tod) # get tod units if not hasattr(tod, '_unit') or len(tod._unit) == 0: attr = {'_unit': {'?': 1.}} model.propagate_attributes(None, attr) u = getattr(attr, '_unit', {}) if 'detector' in u and u['detector'] == -1: u = {'detector': -1.} elif u == {'?': 1.}: u = {} elif len(u) > 1: raise ValueError( 'The timeline units are not known and cannot be in' 'ferred from the model.') tod_du = getattr(attr, '_derived_units', {}) tod = Tod(tod.magnitude, unit=u, derived_units=tod_du, copy=False) else: attr = {'_unit': {'?': 1}} model.T.propagate_attributes(None, attr) u = attr['_unit'] if 'detector' not in tod._unit and 'detector' in u and u[ 'detector'] == 1: raise ValueError("The model is incompatible with input units '{0}'"\ .format(tod.unit)) tod_unit = tod._unit tod_du = tod._derived_units # make sure the input is a surface brightness if 'detector' in tod._unit: tod.inunit(tod.unit + ' detector / arcsec^2') elif 'detector_reference' in tod._unit: tod.inunit(tod.unit + ' detector_reference / arcsec^2') # compute model.T(tod)/model.T(one) mymap = model.T(tod.magnitude) tod[...] = 1 mask(tod, tod) map_weights = model.T(tod.magnitude) old_settings = np.seterr(divide='ignore', invalid='ignore') mymap /= map_weights mymap.unit = tod.unit np.seterr(**old_settings) mymap.coverage = Map(map_weights.magnitude, header=mymap.header, copy=False) if unit is not None: mymap.inunit(unit) return mymap # set map units according to model attr = {'_unit': tod_unit, '_derived_units': tod_du} model.T.propagate_attributes(None, attr) if '_derived_units' in attr: mymap.derived_units = attr['_derived_units'] if '_unit' in attr: mymap.inunit(attr['_unit']) return mymap
def mapper_rls(y, H, invntt=None, unpacking=None, hyper=1.0, x0=None, tol=1.e-5, maxiter=300, M=None, solver=None, verbose=True, callback=None, criterion=True, profile=None): """ Solve the linear equation y = H(x) where H is an Operator representing the acquisition, and y is the observed time series. x is the regularised least square solution as given by: x = argmin (Hx-y)^T N^-1 (Hx-y) + h ||D(x)||^2 or: x = (H^T N^-1 H + h D1^T D1 + h D2^T D2)^-1 H^T N^-1 y """ comm_map = H.commin or MPI.COMM_WORLD comm_tod = H.commout or comm_map if solver is None: solver = cg new_solver = solver in (pcg, ) tod = _validate_tod(y) ntods_ = int(np.sum(~tod.mask)) if getattr(tod, 'mask', None) is not None \ else tod.size nmaps_ = unpacking.shape[1] if unpacking is not None else None if nmaps_ is None: nmaps_ = H.shape[1] if nmaps_ is None: raise ValueError('The model H has not an explicit input shape.') ntods = comm_tod.allreduce(ntods_) nmaps = comm_map.allreduce(nmaps_) # get A if invntt is None: invntt = IdentityOperator() A = H.T * invntt * H npriors = len(H.shapein) priors = [ DiscreteDifferenceOperator(axis=axis, shapein=H.shapein, commin=comm_map) for axis in range(npriors) ] if comm_map.rank == 0 or comm_map.size > 1: A += sum((hyper * ntods / nmaps) * p.T * p for p in priors) # get b b = (H.T * invntt)(tod) if not np.all(np.isfinite(b)): raise ValueError('RHS contains non-finite values.') if b.shape != A.shapein: raise ValueError("Incompatible size for RHS: '" + str(b.size) + "' instead of '" + str(A.shape[1]) + "'.") if np.min(b) == np.max(b) == 0: print('Warning: in equation Ax=b, b is zero.') # unpack input if unpacking is None: unpacking = IdentityOperator() A = unpacking.T * A * unpacking b = unpacking.T(b) if x0 is not None: x0 = unpacking.T(x0) if not new_solver and solver is not cg: b = b.ravel() if x0 is not None: x0 = x0.ravel() if M is not None: if isinstance(M, DiagonalOperator): filter_nonfinite(M.data, out=M.data) M = unpacking.T * M * unpacking H_ = H * unpacking priors = [p * unpacking for p in priors] # criterion if hyper != 0: hc = np.hstack([1, npriors * [hyper]]) / ntods else: hc = [1 / ntods] norms = [norm2_ellipsoid(invntt)] + npriors * [norm2] comms = [comm_tod] + npriors * [comm_map] def criter(x): rs = [H_.matvec(x) - tod.view(np.ndarray).ravel() ] + [p.matvec(x) for p in priors] Js = [h * n(r, comm=c) for h, n, r, c in zip(hc, norms, rs, comms)] return Js if callback is None: if new_solver: callback = NewCgCallback(disp=verbose, comm=comm_map) else: callback = CgCallback(verbose=verbose, objfunc=criter if criterion else None) if (verbose or profile) and comm_map.rank == 0: print('') print('H.T * N^-1 * H:') print(repr(A)) if M is not None: print('Preconditioner:') print(M) time0 = time.time() if profile is not None: def run(): result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter) if new_solver: solution = result['x'] if not result['success']: print('Solver failure: ' + result['message']) else: solution, info = result if info != 0: print('Solver failure: info=' + str(info)) cProfile.runctx('run()', globals(), locals(), profile + '.prof') print('Profile time: ' + str(time.time() - time0)) os.system('python -m gprof2dot -f pstats -o ' + profile + '.dot ' + profile + '.prof') os.system('dot -Tpng ' + profile + '.dot > ' + profile) os.system('rm -f ' + profile + '.prof' + ' ' + profile + '.dot') return None result = solver(A, b, x0=x0, M=M, tol=tol, maxiter=maxiter, callback=callback) time0 = time.time() - time0 if new_solver: solution = result['x'] if not result['success']: print('Solver failure: ' + result['message']) niterations = result['niterations'] error = result['error'] else: solution, info = result if info < 0: raise RuntimeError('Solver failure (code=' + str(info) + ' after ' + str(callback.niterations) + ' iterations).') if info > 0 and comm_map.rank == 0: print( 'Warning: Solver reached maximum number of iterations without' ' reaching specified tolerance.') niterations = getattr(callback, 'niterations', None) error = getattr(callback, 'residual', None) Js = criter(solution) if isinstance(unpacking, IdentityOperator): solution = solution.reshape(H.shapein) else: solution = unpacking(solution) tod[...] = 1 coverage = H.T(tod) unit = getattr(coverage, 'unit', None) derived_units = getattr(coverage, 'derived_units', None) coverage = coverage.view(Map) header = getattr(coverage, 'header', None) if header is None: header = create_fitsheader(fromdata=coverage) header.update('likeliho', Js[0]) header.update('criter', sum(Js)) header.update('hyper', hyper) header.update('nsamples', ntods) header.update('npixels', nmaps) header.update('time', time0) if niterations is not None: header.update('niter', niterations) header.update('maxiter', maxiter) if error is not None: header.update('error', error) header.update('tol', tol) header.update('solver', solver.__name__) output = Map(solution, header=header, coverage=coverage, unit=unit, derived_units=derived_units, copy=False) return output
def ProjectionOperator(input, method=None, header=None, resolution=None, npixels_per_sample=0, units=None, derived_units=None, downsampling=False, packed=False, commin=MPI.COMM_WORLD, commout=MPI.COMM_WORLD, onfly_func=None, onfly_ids=None, onfly_shapeouts=None, **keywords): """ Projection operator factory, to handle operations by one or more pointing matrices. It is assumed that each pointing matrix row has at most 'npixels_per_sample' non-null elements, or in other words, an output element can only intersect a fixed number of input elements. Given an input vector x and an output vector y, y = P(x) translates into: y[i] = sum(P.matrix[i,:].value * x[P.matrix[i,:].index]) If the input is not MPI-distributed unlike the output, the projection operator is automatically multiplied by the operator MPIDistributionIdenti- tyOperator, to enable MPI reductions. If the input is MPI-distributed, this operator is automatically packed (see below) and multiplied by the operator MPIDistributionLocalOperator, which takes the local input as argument. Arguments --------- input : pointing matrix (or sequence of) or observation (deprecated) method : deprecated header : deprecated resolution : deprecated npixels_per_sample : deprecated downsampling : deprecated packed deprecated """ # check if there is only one pointing matrix isonfly = input is None isobservation = hasattr(input, 'get_pointing_matrix') if isobservation: if hasattr(input, 'slice'): nmatrices = len(input.slice) else: nmatrices = 1 commout = input.instrument.comm elif isonfly: nmatrices = len(onfly_ids) else: if isinstance(input, PointingMatrix): input = (input,) if any(not isinstance(i, PointingMatrix) for i in input): raise TypeError('The input is not a PointingMatrix, (nor a sequence' ' of).') nmatrices = len(input) ismapdistributed = commin.size > 1 istoddistributed = commout.size > 1 # get the pointing matrix from the input observation if isobservation: if header is None: if not hasattr(input, 'get_map_header'): raise AttributeError("No map header has been specified and " "the observation has no 'get_map_header' method.") header_global = input.get_map_header(resolution=resolution, downsampling=downsampling) else: if isinstance(header, str): header = str2fitsheader(header) header_global = gather_fitsheader_if_needed(header, comm=commin) #XXX we should hand over the local header input = input.get_pointing_matrix(header_global, npixels_per_sample, method=method, downsampling=downsampling, comm=commin) if isinstance(input, PointingMatrix): input = (input,) elif isonfly: header_global = header else: header_global = input[0].info['header'] # check shapein if not isonfly: shapeins = [i.shape_input for i in input] if any(s != shapeins[0] for s in shapeins): raise ValueError('The pointing matrices do not have the same input ' "shape: {0}.".format(', '.join(str(shapeins)))) shapein = shapeins[0] else: shapein = fitsheader2shape(header_global) # the output is simply a ProjectionOperator instance if nmatrices == 1 and not ismapdistributed and not istoddistributed \ and not packed: return ProjectionInMemoryOperator(input[0], units=units, derived_units= derived_units, commin=commin, commout=commout, **keywords) if packed or ismapdistributed: if isonfly: raise NotImplementedError() # compute the map mask before this information is lost while packing mask_global = Map.ones(shapein, dtype=np.bool8, header=header_global) for i in input: i.get_mask(out=mask_global) for i in input: i.pack(mask_global) if ismapdistributed: shapes = distribute_shapes(mask_global.shape, comm=commin) shape = shapes[commin.rank] mask = np.empty(shape, bool) commin.Reduce_scatter([mask_global, MPI.BYTE], [mask, MPI.BYTE], [product(s) for s in shapes], op=MPI.BAND) else: mask = mask_global if isonfly: place_holder = {} operands = [ProjectionOnFlyOperator(place_holder, id, onfly_func, shapein=shapein, shapeout=shapeout, units=units, derived_units=derived_units) for id, shapeout in zip(onfly_ids, onfly_shapeouts)] else: operands = [ProjectionInMemoryOperator(i, units=units, derived_units= derived_units, commout=commout, **keywords) for i in input] result = BlockColumnOperator(operands, axisout=-1) if nmatrices > 1: def apply_mask(self, mask): mask = np.asarray(mask, np.bool8) dest = 0 if mask.shape != self.shapeout: raise ValueError("The mask shape '{0}' is incompatible with tha" "t of the projection operator '{1}'.".format(mask.shape, self.shapeout)) if any(isinstance(p, ProjectionOnFlyOperator) for p in self.operands): blocks = self.copy() self.__class__ = CompositionOperator self.__init__([MaskOperator(mask), blocks]) return for p in self.operands: n = p.matrix.shape[1] p.apply_mask(mask[...,dest:dest+n]) dest += n def get_mask(self, out=None): for p in self.operands: out = p.get_mask(out=out) return out def get_pTp(self, out=None): for p in self.operands: out = p.get_pTp(out=out) return out def get_pTx_pT1(self, x, out=None, mask=None): dest = 0 for p in self.operands: n = p.matrix.shape[1] out = p.get_pTx_pT1(x[...,dest:dest+n], out=out, mask=mask) dest += n return out def intersects(self, out=None): raise NotImplementedError('email-me') result.apply_mask = apply_mask.__get__(result) result.get_mask = get_mask.__get__(result) result.get_pTp = get_pTp.__get__(result) result.get_pTx_pT1 = get_pTx_pT1.__get__(result) result.intersects = intersects.__get__(result) if not istoddistributed and not ismapdistributed and not packed: return result if packed or ismapdistributed: def get_mask(self, out=None): if out is not None: out &= mask else: out = mask return out if ismapdistributed: header = scatter_fitsheader(header_global, comm=commin) result *= MPIDistributionLocalOperator(mask_global, commin=commin, attrin={'header':header}) elif packed: result *= PackOperator(mask) result.get_mask = get_mask.__get__(result) if istoddistributed and not ismapdistributed: def get_mask(self, out=None): out = self.operands[0].get_mask(out=out) commout.Allreduce(MPI.IN_PLACE, [out, MPI.BYTE], op=MPI.BAND) return out result *= MPIDistributionIdentityOperator(commout=commout) result.get_mask = get_mask.__get__(result) def not_implemented(out=None): raise NotImplementedError('email-me') def apply_mask(self, mask): self.operands[0].apply_mask(mask) result.apply_mask = apply_mask.__get__(result) result.get_pTp = not_implemented result.get_pTx_pT1 = not_implemented result.intersects = not_implemented return result
def ProjectionOperator(input, method=None, header=None, resolution=None, npixels_per_sample=0, units=None, derived_units=None, downsampling=False, packed=False, commin=MPI.COMM_WORLD, commout=MPI.COMM_WORLD, onfly_func=None, onfly_ids=None, onfly_shapeouts=None, **keywords): """ Projection operator factory, to handle operations by one or more pointing matrices. It is assumed that each pointing matrix row has at most 'npixels_per_sample' non-null elements, or in other words, an output element can only intersect a fixed number of input elements. Given an input vector x and an output vector y, y = P(x) translates into: y[i] = sum(P.matrix[i,:].value * x[P.matrix[i,:].index]) If the input is not MPI-distributed unlike the output, the projection operator is automatically multiplied by the operator MPIDistributionIdenti- tyOperator, to enable MPI reductions. If the input is MPI-distributed, this operator is automatically packed (see below) and multiplied by the operator MPIDistributionLocalOperator, which takes the local input as argument. Arguments --------- input : pointing matrix (or sequence of) or observation (deprecated) method : deprecated header : deprecated resolution : deprecated npixels_per_sample : deprecated downsampling : deprecated packed deprecated """ # check if there is only one pointing matrix isonfly = input is None isobservation = hasattr(input, 'get_pointing_matrix') if isobservation: if hasattr(input, 'slice'): nmatrices = len(input.slice) else: nmatrices = 1 commout = input.instrument.comm elif isonfly: nmatrices = len(onfly_ids) else: if isinstance(input, PointingMatrix): input = (input, ) if any(not isinstance(i, PointingMatrix) for i in input): raise TypeError( 'The input is not a PointingMatrix, (nor a sequence' ' of).') nmatrices = len(input) ismapdistributed = commin.size > 1 istoddistributed = commout.size > 1 # get the pointing matrix from the input observation if isobservation: if header is None: if not hasattr(input, 'get_map_header'): raise AttributeError( "No map header has been specified and " "the observation has no 'get_map_header' method.") header_global = input.get_map_header(resolution=resolution, downsampling=downsampling) else: if isinstance(header, str): header = str2fitsheader(header) header_global = gather_fitsheader_if_needed(header, comm=commin) #XXX we should hand over the local header input = input.get_pointing_matrix(header_global, npixels_per_sample, method=method, downsampling=downsampling, comm=commin) if isinstance(input, PointingMatrix): input = (input, ) elif isonfly: header_global = header else: header_global = input[0].info['header'] # check shapein if not isonfly: shapeins = [i.shape_input for i in input] if any(s != shapeins[0] for s in shapeins): raise ValueError( 'The pointing matrices do not have the same input ' "shape: {0}.".format(', '.join(str(shapeins)))) shapein = shapeins[0] else: shapein = fitsheader2shape(header_global) # the output is simply a ProjectionOperator instance if nmatrices == 1 and not ismapdistributed and not istoddistributed \ and not packed: return ProjectionInMemoryOperator(input[0], units=units, derived_units=derived_units, commin=commin, commout=commout, **keywords) if packed or ismapdistributed: if isonfly: raise NotImplementedError() # compute the map mask before this information is lost while packing mask_global = Map.ones(shapein, dtype=np.bool8, header=header_global) for i in input: i.get_mask(out=mask_global) for i in input: i.pack(mask_global) if ismapdistributed: shapes = distribute_shapes(mask_global.shape, comm=commin) shape = shapes[commin.rank] mask = np.empty(shape, bool) commin.Reduce_scatter([mask_global, MPI.BYTE], [mask, MPI.BYTE], [product(s) for s in shapes], op=MPI.BAND) else: mask = mask_global if isonfly: place_holder = {} operands = [ ProjectionOnFlyOperator(place_holder, id, onfly_func, shapein=shapein, shapeout=shapeout, units=units, derived_units=derived_units) for id, shapeout in zip(onfly_ids, onfly_shapeouts) ] else: operands = [ ProjectionInMemoryOperator(i, units=units, derived_units=derived_units, commout=commout, **keywords) for i in input ] result = BlockColumnOperator(operands, axisout=-1) if nmatrices > 1: def apply_mask(self, mask): mask = np.asarray(mask, np.bool8) dest = 0 if mask.shape != self.shapeout: raise ValueError( "The mask shape '{0}' is incompatible with tha" "t of the projection operator '{1}'.".format( mask.shape, self.shapeout)) if any( isinstance(p, ProjectionOnFlyOperator) for p in self.operands): blocks = self.copy() self.__class__ = CompositionOperator self.__init__([MaskOperator(mask), blocks]) return for p in self.operands: n = p.matrix.shape[1] p.apply_mask(mask[..., dest:dest + n]) dest += n def get_mask(self, out=None): for p in self.operands: out = p.get_mask(out=out) return out def get_pTp(self, out=None): for p in self.operands: out = p.get_pTp(out=out) return out def get_pTx_pT1(self, x, out=None, mask=None): dest = 0 for p in self.operands: n = p.matrix.shape[1] out = p.get_pTx_pT1(x[..., dest:dest + n], out=out, mask=mask) dest += n return out def intersects(self, out=None): raise NotImplementedError('email-me') result.apply_mask = apply_mask.__get__(result) result.get_mask = get_mask.__get__(result) result.get_pTp = get_pTp.__get__(result) result.get_pTx_pT1 = get_pTx_pT1.__get__(result) result.intersects = intersects.__get__(result) if not istoddistributed and not ismapdistributed and not packed: return result if packed or ismapdistributed: def get_mask(self, out=None): if out is not None: out &= mask else: out = mask return out if ismapdistributed: header = scatter_fitsheader(header_global, comm=commin) result *= MPIDistributionLocalOperator(mask_global, commin=commin, attrin={'header': header}) elif packed: result *= PackOperator(mask) result.get_mask = get_mask.__get__(result) if istoddistributed and not ismapdistributed: def get_mask(self, out=None): out = self.operands[0].get_mask(out=out) commout.Allreduce(MPI.IN_PLACE, [out, MPI.BYTE], op=MPI.BAND) return out result *= MPIDistributionIdentityOperator(commout=commout) result.get_mask = get_mask.__get__(result) def not_implemented(out=None): raise NotImplementedError('email-me') def apply_mask(self, mask): self.operands[0].apply_mask(mask) result.apply_mask = apply_mask.__get__(result) result.get_pTp = not_implemented result.get_pTx_pT1 = not_implemented result.intersects = not_implemented return result
keywords_types = (keywords_q, keywords_f, keywords_m, keywords_t) dtypes = (bool, int, np.float32, np.float64, np.complex64, np.complex128) def teardown(): for file in glob.glob(filename + '*'): os.remove(file) a = np.ones((4, 3)) a[1, 2] = 4 q = Quantity(a, unit='myunit', derived_units={'myunit': Quantity(2., 'Jy')}) header = create_fitsheader(fromdata=q, cdelt=0.5, crval=(4., 8.)) header['BUNIT'] = 'myunit' f = FitsArray(q, header=header) m = Map(f, origin='upper', error=a * 2, coverage=a * 3) mask = np.zeros((4, 3), np.bool8) mask[0, 2] = True t = Tod(f, mask=mask) del mask def test_copy_false_subok_true(): def func(obj1, t): obj2 = t(obj1, copy=False, subok=True) if isinstance(obj, t): assert obj1 is obj2 else: assert obj1 is not obj2 assert_equal(obj1, obj2)