def generate_model(ra0, dec0, pointing_params, repeats=1, cross_scan=False, span_angles=False, band="red", map_header=None, npixels_per_sample=0): """ Generate a PACS projector. """ pointing1 = tm.pacs_create_scan(ra0, dec0, **pointing_params) # create obs object obs1 = tm.PacsSimulation(pointing1, band) if map_header is None: map_header = obs1.get_map_header() # create projector projection1 = tm.Projection(obs1, header=map_header, npixels_per_sample=npixels_per_sample) P = lo.aslinearoperator(projection1) # cross scan if cross_scan: pointing_params["scan_angle"] += 90. pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params) obs2 = tm.PacsSimulation(pointing2, band) projection2 = tm.Projection(obs2, header=map_header, npixels_per_sample=npixels_per_sample) P2 = lo.aslinearoperator(projection2) P = lo.concatenate((P, P2)) # repeats if repeats > 1: P = lo.concatenate((P, ) * repeats) if span_angles: if cross_scan: raise ValueError("Span_angles and cross_scan are incompatible.") # equally spaced but exclude 0 and 90 angles = np.linspace(0, 90, repeats + 2)[1:-1] for a in angles: pointing_params["scan_angle"] = a pointing2 = tm.pacs_create_scan(ra0, dec0, **pointing_params) obs2 = tm.PacsSimulation(pointing2, band) projection2 = tm.Projection(obs2, header=map_header, npixels_per_sample=npixels_per_sample) P2 = lo.aslinearoperator(projection2) P = lo.concatenate((P, P2)) # noise N = generate_noise_filter(obs1, P, band) # covered area map_shape = map_header['NAXIS2'], map_header['NAXIS1'] coverage = (P.T * np.ones(P.shape[0])).reshape(map_shape) seen = coverage != 0 M = lo.decimate(coverage < 10) # global model H = N * P * M.T return H
def pipeline_dli(filename, output_file, keywords, verbose=False): import numpy as np import linear_operators as lo import tamasis as tm # verbosity tm.var.verbose = verbose # define observation obs = tm.PacsObservation(filename, **keywords["PacsObservation"]) # extra masking tm.step_scanline_masking(obs, **keywords["scanline_masking"]) # get data tod = obs.get_tod(**keywords["get_tod"]) # deglitching tm.step_deglitching(obs, tod, **keywords["deglitching"]) # median filtering tod = tm.filter_median(tod, **keywords["filter_median"]) # define projector projection = tm.Projection(obs, **keywords["Projection"]) # build instrument model response = tm.ResponseTruncatedExponential(obs.pack( obs.instrument.detector.time_constant) / obs.SAMPLING_PERIOD) compression = tm.CompressionAverage(obs.slice.compression_factor) masking = tm.Masking(tod.mask) model = masking * compression * response * projection # set tod masked values to zero tod = masking(tod) # N^-1 operator invntt = tm.InvNtt(obs) # for the dli algorithm M = map_mask(tod, model, **keywords["map_mask"]) # recast model as a new-style LinearOperator from linear_operators package H = lo.aslinearoperator(model) * M.T N = lo.aslinearoperator(invntt) # vectorize data so it is accepted by LinearOperators y = tod.ravel() Ds = [tm.DiscreteDifference(axis=i, shapein=projection.shapein) for i in (0, 1)] Ds = [lo.aslinearoperator(D) for D in Ds] Ds = [D * M.T for D in Ds] D = lo.concatenate(Ds) # handle tau which needs to be an ndarray keywords["dli"]["tau"] *= np.ones(D.shape[0]) algo = lo.DoubleLoopAlgorithm(H, y, D, noise_covariance=N, fmin_args=keywords["fmin_args"], lanczos=keywords["lanczos"], **keywords["dli"]) # optimize xe = algo() # reshape xe = (M.T * xe).reshape(projection.shapein) # recast as tamasis map xe = tm.Map(xe) # save xe.save(output_file)
def pipeline_photproject(filenames, output_file, keywords): """ Perform regularized least-square inversion of state of the art PACS model. The PACS model includes tod mask, compression, response, projection and noise covariance (invntt). Processing steps are as follows : - define PacsObservation instance - mask first scanlines to avoid drift - get Time Ordered Data (get_tod) - 2nd level deglitching (with standard projector and tod median filtering with a narrow window) - median filtering - define projection - perform inversion on model - save file Arguments --------- filenames: list of strings List of data filenames. output_file : string Name of the output fits file. keywords: dict Dictionary containing options for all steps as dictionary. Returns ------- Returns nothing. Save result as a fits file. """ from scipy.sparse.linalg import cgs import tamasis as tm # define observation obs = tm.PacsObservation(filenames, **keywords["PacsObservation"]) # extra masking tm.step_scanline_masking(obs, **keywords["scanline_masking"]) # get data tod = obs.get_tod(**keywords["get_tod"]) # deglitching tm.step_deglitching(obs, tod, **keywords["deglitching"]) # median filtering tod = tm.filter_median(tod, **keywords["filter_median"]) # define projector projection = tm.Projection(obs, oversampling=False, **keywords["Projection"]) # build instrument model masking = tm.Masking(tod.mask) model = masking * projection # set tod masked values to zero tod = masking(tod) # perform map-making map_naive = tm.mapper_naive(tod, model, **keywords["mapper_naive"]) # save map_naive.save(output_file)
def pipeline_rls(filenames, output_file, keywords, verbose=False): """ Perform regularized least-square inversion of a simple model (tod mask and projection). Processing steps are as follows : - define PacsObservation instance - get Time Ordered Data (get_tod) - define projection - perform inversion on model - save file Arguments --------- filenames: list of strings List of data filenames. output_file : string Name of the output fits file. keywords: dict Dictionary containing options for all steps as dictionary. verbose: boolean (default False) Set verbosity. Returns ------- Returns nothing. Save result as a fits file. """ from scipy.sparse.linalg import cgs import tamasis as tm # verbosity keywords["mapper_rls"]["verbose"] = verbose # define observation obs_keys = keywords.get("PacsObservation", {}) obs = tm.PacsObservation(filenames, **obs_keys) # get data tod_keys = keywords.get("get_tod", {}) tod = obs.get_tod(**tod_keys) # define projector proj_keys = keywords.get("Projection", {}) projection = tm.Projection(obs, **proj_keys) # define mask masking_tod = tm.Masking(tod.mask) # define full model model = masking_tod * projection # perform map-making inversion mapper_keys = keywords.get("mapper_rls", {}) map_rls = tm.mapper_rls(tod, model, solver=cgs, **mapper_keys) # save map_rls.save(output_file)
def load_data(filename, header=None, resolution=3.): #mask = np.zeros((32, 64), dtype=np.int8) obs = tm.PacsObservation(filename=filename, fine_sampling_factor=1, #detector_mask=mask ) tod = obs.get_tod() if header is None: header = obs.get_map_header() header.update('CDELT1', resolution / 3600) header.update('CDELT2', resolution / 3600) npix = 5 good_npix = False while good_npix is False: try: projection = tm.Projection(obs, header=header, resolution=resolution, oversampling=False, npixels_per_sample=npix) good_npix = True except(RuntimeError): npix +=1 return tod, projection, header, obs
def pipeline_huber(filenames, output_file, keywords, verbose=False): """ Perform huber regularized inversion of state of the art PACS model. The PACS model includes tod mask, compression, response, projection and noise covariance (invntt). Processing steps are as follows : - define PacsObservation instance - mask first scanlines to avoid drift - get Time Ordered Data (get_tod) - 2nd level deglitching (with standard projector and tod median filtering with a narrow window) - median filtering - define projection - perform inversion on model - save file Arguments --------- filenames: list of strings List of data filenames. output_file : string Name of the output fits file. keywords: dict Dictionary containing options for all steps as dictionary. verbose: boolean (default False) Set verbosity. Returns ------- Returns nothing. Save result as a fits file. """ from scipy.sparse.linalg import cgs import tamasis as tm import linear_operators as lo # verbosity # define observation obs = tm.PacsObservation(filenames, **keywords["PacsObservation"]) # extra masking tm.step_scanline_masking(obs, **keywords["scanline_masking"]) # get data tod = obs.get_tod(**keywords["get_tod"]) # deglitching # need to adapt degltiching to any compression model tm.step_deglitching(obs, tod, **keywords["deglitching"]) # median filtering tod = tm.filter_median(tod, **keywords["filter_median"]) # define projector projection = tm.Projection(obs, **keywords["Projection"]) P = lo.aslinearoperator(projection) # build instrument model masking = tm.Masking(tod.mask) Mt = lo.aslinearoperator(masking) # compression mode = compressions[keywords["compression"].pop("mode")] factor = keywords["compression"].pop("factor") C = mode(data, factor, **keywords["compression"]) # define map mask M = map_mask(tod, model, **keywords["map_mask"]) # recast model as a new-style LinearOperator from linear_operators package H = Mt * C * P * M # vectorize data so it is accepted by LinearOperators y = tod.ravel() Ds = [ tm.DiscreteDifference(axis=i, shapein=projection.shapein) for i in (0, 1) ] Ds = [lo.aslinearoperator(D) for D in Ds] Ds = [D * M.T for D in Ds] # set tod masked values to zero tod = masking(tod) # perform map-making inversion hypers = (keywords["hacg"].pop("hyper"), ) * len(Ds) deltas = (keywords["hacg"].pop("delta"), ) * (len(Ds) + 1) map_huber = lo.hacg(H, tod.ravel(), priors=Ds, hypers=hypers, deltas=deltas, **keywords["hacg"]) # save map_huber = (M.T * map_huber).reshape(projection.shapein) map_huber = tm.Map(map_huber) map_huber.save(output_file)
#!/usr/bin/env python import numpy as np import tamasis as tm import lo import csh.filter as filt from time import time import scipy.sparse.linalg as spl # data pacs = tm.PacsObservation(filename=tm.tamasis_dir+'tests/frames_blue.fits') tod = pacs.get_tod() # projector model = tm.Projection(pacs, resolution=3.2, oversampling=False, npixels_per_sample=6) # naive map backmap = model.transpose(tod) # transform to lo P = lo.aslinearoperator(model.aslinearoperator()) # derive filter kernel = filt.kernel_from_tod(tod, length=10) #kern = np.mean(kernel, axis=0) N = filt.kernels_convolve(tod.shape, 1 / np.sqrt(kernel)) # apply to data yn = N * tod.flatten() # apply to model M = N * P # priors Ds = [lo.diff(backmap.shape, axis=axis) for axis in xrange(backmap.ndim)] #Ds.append(lo.pywt_lo.wavelet2(backmap.shape, "haar")) # inversion #y = tod.flatten() x, conv = lo.rls(M, Ds, (1e1, 1e1, 1e-1), yn, spl.bicgstab)
pointing_params = { 'cam_angle': 0., "scan_angle": 0., "scan_nlegs": 1, "scan_length": 400., "scan_speed": 60.0, "compression_factor": 1 # we will compress later } pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params) # create obs object obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep") # we dont need 200 first and last frames obs.pointing.removed[200:] = True obs.pointing.removed[-201:] = True # create projector projection = tm.Projection(obs, npixels_per_sample=4) P = lo.aslinearoperator(projection) # simulate data x0 = tm.gaussian(projection.shapein, 3) # map with gaussian source tod0 = projection(x0) # data n = np.random.randn(*tod0.shape) # noise nsr = 1e-2 tod = tod0 + nsr * n # noisy data y = tod.ravel() # as 1d array # load compression matrix filename = os.path.join(os.getenv("HOME"), "data", "pacs", "mmc_cam_angle_0_scan_angle_0_speed60.fits") c = fa.FitsArray(file=filename).astype(np.float64) cmm = csh.compression.AnyOperator(c) C = cmm((projection.shapeout[0], projection.shapeout[1][0]), )
y = tod.flatten() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # compress data factor = 8 compression = tm.CompressionAverage(factor) C = lo.aslinearoperator( compression.aslinearoperator(shape=(tod.size / factor, tod.size))) ctod = compression.direct(tod) # uncompress for deglitching uctod = tod.copy(tod.shape) y0, t = lo.spl.cgs(C.T * C, C.T * ctod.flatten()) uctod[:] = y0.reshape(tod.shape) # deglitching projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=5) uctod.mask = tm.deglitch_l2mad(uctod, projection) ctod = compression.direct(uctod) # model masking = tm.Masking(uctod.mask) model = compression * masking * projection # remove drift #ctod = tm.filter_median(ctod, length=3000 / 8.) # first map M = lo.aslinearoperator(model.aslinearoperator()) #P = lo.aslinearoperator(projection.aslinearoperator()) #C = csh.averaging(tod.shape, factor=8) #I = lo.mask(uctod.mask) #M = C * I.T * I * P #M = C * P
factor = 4 ext = ".fits" pre = "ngc6946_madmap1_" # find a map for each compression and save it obs = tm.PacsObservation(filename=filename, fine_sampling_factor=1, detector_policy='remove') tod = obs.get_tod() header = obs.get_map_header() header.update('CDELT1', resolution / 3600) header.update('CDELT2', resolution / 3600) npix = 5 good_npix = False projection = tm.Projection(obs, header=header, resolution=resolution, oversampling=False, npixels_per_sample=npix) model = projection #C = csh.averaging(tod.shape, factor=factor) compression_shape = [ tod.size / factor, tod.size, ] C = tm.CompressionAverage( compression_factor=factor).aslinearoperator(compression_shape) C = lo.aslinearoperator(C) # compress data ctod = compress(tod, C, factor) # uncompress for preprocessing uctod = uncompress(ctod, C, factor)
pacs = tm.PacsObservation(filename=filenames, fine_sampling_factor=1, keep_bad_detectors=False) # reset pacs header to have a shape multiple of 4 #header = pacs.get_map_header() #header['NAXIS1'] = 192 #header['NAXIS2'] = 192 #header['CRPIX1'] = 96 #header['CRPIX2'] = 96 # data tod = pacs.get_tod() # remove bad pixels (by updating mask !) #tod = remove_bad_pixels(tod) # deglitching #projection = tm.Projection(pacs, header=header, resolution=3., npixels_per_sample=6) projection = tm.Projection(pacs, resolution=3., npixels_per_sample=6) tm.deglitch_l2mad(tod, projection) # model masking = tm.Masking(tod.mask) model = masking * projection # remove drift tod = tm.filter_median(tod, length=999) model = masking * projection # naive map backmap = model.transpose(tod) # coverage map weights = model.transpose(tod.ones(tod.shape)) # mask on map mask = weights == 0 M = lo.mask(mask)
def pipeline_wrls(filenames, output_file, keywords, verbose=False): """ Perform regularized least-square inversion of state of the art PACS model. The PACS model includes tod mask, compression, response, projection and noise covariance (invntt). Processing steps are as follows : - define PacsObservation instance - mask first scanlines to avoid drift - get Time Ordered Data (get_tod) - 2nd level deglitching (with standard projector and tod median filtering with a narrow window) - median filtering - define projection - perform inversion on model - save file Arguments --------- filenames: list of strings List of data filenames. output_file : string Name of the output fits file. keywords: dict Dictionary containing options for all steps as dictionary. verbose: boolean (default False) Set verbosity. Returns ------- Returns nothing. Save result as a fits file. """ from scipy.sparse.linalg import cgs import tamasis as tm # verbosity keywords["mapper_rls"]["verbose"] = verbose # define observation obs = tm.PacsObservation(filenames, **keywords["PacsObservation"]) # extra masking tm.step_scanline_masking(obs, **keywords["scanline_masking"]) # get data tod = obs.get_tod(**keywords["get_tod"]) # deglitching tm.step_deglitching(obs, tod, **keywords["deglitching"]) # median filtering tod = tm.filter_median(tod, **keywords["filter_median"]) # define projector projection = tm.Projection(obs, **keywords["Projection"]) # build instrument model response = tm.ResponseTruncatedExponential( obs.pack(obs.instrument.detector.time_constant) / obs.SAMPLING_PERIOD) compression = tm.CompressionAverage(obs.slice.compression_factor) masking = tm.Masking(tod.mask) model = masking * compression * response * projection # set tod masked values to zero tod = masking(tod) # N^-1 operator invntt = tm.InvNtt(obs) # perform map-making inversion map_rls = tm.mapper_rls(tod, model, invntt=invntt, solver=cgs, **keywords["mapper_rls"]) # save map_rls.save(output_file)
'cam_angle': 0., "scan_angle": 0., "scan_nlegs": 1, "scan_length": 4., "compression_factor": 1, "scan_speed": 60.0 } pointing = tm.pacs_create_scan(ra0, dec0, **pointing_params) # create obs object obs = tm.PacsSimulation(pointing, "red", policy_bad_detector="keep") # to define my compression matrix I need only 8 frames # so I mask the others : obs.pointing.removed[:] = True obs.pointing.removed[201:209] = False # create projector projection = tm.Projection(obs) P = lo.aslinearoperator(projection) # define masking of coefficients # ------------------------------ # number of coefficients to keep factor = 8. nk = P.shape[0] / factor # coverage map w = P.T * np.ones(P.shape[0]) # sort coverage map coefficients ws = np.sort(w) # find the nk largest coverage elements threshold = ws[-nk] mask = w < threshold # define decimation Linear Operator according to coverage map coef