def test_table_fits_io_astropy(table): """Test `astropy.table.Table` FITS I/O in Astropy. Having these tests in Gammapy is to check / ensure that the features we rely on work properly for all Astropy versions we support in CI (currently Astropy 1.3 and up) This is useful, because Table FITS I/O was pretty shaky for a while and incrementally improved over time. These are the same examples that we have in the docstring at the top of `gammapy/utils/fits.py`. """ # Check Table -> BinTableHDU hdu = fits.BinTableHDU(table) assert hdu.header["TTYPE2"] == "b" assert hdu.header["TFORM2"] == "K" assert hdu.header["TUNIT2"] == "m" # Check BinTableHDU -> Table table2 = Table.read(hdu) assert isinstance(table2.meta, dict) assert table2.meta == {"VERSION": 42} assert table2["b"].unit == "m"
def to_hdulist(self, format="ogip", hdu_bands=None, hdu_region=None): """Convert geom to hdulist Parameters ---------- format : {"gadf", "ogip", "ogip-sherpa"} HDU format hdu : str Name of the HDU with the map data. Returns ------- hdulist : `~astropy.io.fits.HDUList` HDU list """ if hdu_bands is None: hdu_bands = "HDU_BANDS" if hdu_region is None: hdu_region = "HDU_REGION" if format != "gadf": hdu_region = "REGION" hdulist = fits.HDUList() hdulist.append( self.axes.to_table_hdu(hdu_bands=hdu_bands, format=format)) # region HDU if self.region: region_table = self._to_region_table() region_hdu = fits.BinTableHDU(region_table, name=hdu_region) hdulist.append(region_hdu) return hdulist
def to_hdulist(self, format="ogip"): """Convert to `~astropy.io.fits.HDUList`. Parameters ---------- format : {"ogip", "ogip-sherpa", "ogip-arf", "ogip-arf-sherpa"} Format specification Returns ------- hdulist : `~astropy.fits.HDUList` HDU list """ hdulist = fits.HDUList() # add data hdu table = self.to_table(format=format) hdulist.append(fits.BinTableHDU(table)) if format in ["ogip", "ogip-sherpa"]: hdulist_geom = self.geom.to_hdulist(format=format)[1:] hdulist.extend(hdulist_geom) return hdulist
hpb_type = np.dtype([('grouped', '=i4'), ('ungrouped', '=i4'), ('potential', '=f4')]) bPotential = True NSIDE = 64 N = 12 * NSIDE**2 hdr = fits.Header() hdr['EXTEND'] = 'T' primary_hdu = fits.PrimaryHDU(header=hdr) primary_hdu.writeto('test.fits') # Use BinTableHDU as a template hdr = fits.BinTableHDU(Table(names=['SIGNAL'], dtype=['=f4' if bPotential else '=i4']), name='BINTABLE').header hdr['ORDERING'] = ("RING", "Pixel ordering scheme, either RING or NESTED") hdr['INDXSCHM'] = ("IMPLICIT", "Pixel indexing scheme (IMPLICIT or EXPLICIT)") hdr['NSIDE'] = (NSIDE, "Resolution parameter for HEALPIX") hdr['COORDSYS'] = ("C", "Pixelisation coordinate system") hdr['PIXTYPE'] = ("HEALPIX", "HEALPIX Pixelisation") hdr['NAXIS'] = 2 hdr['NAXIS2'] = N hdr['NAXIS1'] = 1 hdr['BITPIX'] = -32 if bPotential else 32 hdu = fits.StreamingHDU('test.fits', hdr) for file in argv[1:]: with open(file, 'rb') as hpb: data = np.fromfile(hpb, dtype=hpb_type)
def to_fits(self, name="EFFECTIVE AREA"): """Convert to `~astropy.io.fits.BinTable`.""" return fits.BinTableHDU(self.to_table(), name=name)
def make_spw_cube(spw='spw{0}', spwnum=0, fntemplate='OrionSourceI', overwrite_existing=False, bmaj_limits=None, fnsuffix="", filesuffix='image.pbcor.fits', first_endchannel='*', cropends=False, minimize=True, debug_mode=False, add_beam_info=True): """ Parameters ---------- spw : str String template for the input/output name spwnum : int The spectral window number fntemplate : str Filename template (goes into the glob) overwrite_existing : bool Overwrite data in the output cube? cropends: bool or int Number of pixels to crop off the ends of an image minimize: bool Compute the spatial minimal subcube before building the cube? Slices for all subsequent cubes will be computed from the first cube. """ if debug_mode: lvl = log.getEffectiveLevel() log.setLevel('DEBUG') spw = spw.format(spwnum) big_filename = '{1}_{0}{2}_lines.fits'.format(spw, fntemplate, fnsuffix) header_fn = glob.glob( 'OrionSourceI.B3.{0}.lines0-{4}.clarkclean1000.{3}'.format( spw, fntemplate, fnsuffix, filesuffix, first_endchannel)) if len(header_fn) != 1: raise ValueError( "Found too many or too few matches: {0}".format(header_fn)) else: header_fn = header_fn[0] # First set up an empty file if not os.path.exists(big_filename): log.info("Creating large cube based on header {0}".format(header_fn)) if minimize: cube0 = SpectralCube.read(header_fn) slices = cube0.subcube_slices_from_mask(cube0.mask, spatial_only=True) # use the calculated 3rd dimension, plus the difference of the # x and y slices #header['NAXIS2'] = slices[1].stop-slices[1].start #header['NAXIS1'] = slices[2].stop-slices[2].start header = cube0[slices].header else: header = fits.getheader(header_fn) # Make an arbitrary, small data before prepping the header data = np.zeros((100, 100), dtype=np.float32) hdu = fits.PrimaryHDU(data=data, header=header) cdelt_sign = np.sign(hdu.header['CDELT3']) # Set the appropriate output size (this can be extracted from the LISTOBS) naxis3_in = header['NAXIS3'] header['NAXIS3'] = nchans_total[spwnum] header_wcs = wcs.WCS(fits.getheader(header_fn)) header_specwcs = header_wcs.sub([wcs.WCSSUB_SPECTRAL]) if cdelt_sign == -1: ind0, ind1 = getinds(header_fn) #5/20/2017: redoing some of this, and the text below is frightening but no longer relevant # a +1 was on the next line before an edit on 4/10/2017 # it may have been rendered irrelevant when I included +1 # channel in each cube? Not clear - the arithmetic no longer # makes sense but is empirically necessary. assert ind0 == 0 # these reindex the cube so that it has an increasing cdelt. header['CRPIX3'] = 1 #nchans_total[spwnum] header['CRVAL3'] = header_specwcs.wcs_pix2world( [nchans_total[spwnum]], 1)[0][0] header['CDELT3'] = np.abs(header_specwcs.wcs.cdelt[0]) # ensure that the new CRVAL evaluated at its own position matches # the CRVAL3. This should be impossible to fail unless WCS itself # fails newheaderspecwcs = wcs.WCS(header).sub([wcs.WCSSUB_SPECTRAL]) crval3 = newheaderspecwcs.wcs_pix2world([header['CRPIX3']], 1)[0][0] np.testing.assert_array_almost_equal_nulp(crval3, header['CRVAL3']) shape = (header['NAXIS3'], header['NAXIS2'], header['NAXIS1']) # Write to disk header.tofile(big_filename) # Using the 'append' io method, update the *header* with open(big_filename, 'rb+') as fobj: # Seek past the length of the header, plus the length of the # data we want to write. # The -1 is to account for the final byte that we are about to # write: # 'seek' works on bytes, so divide #bits / (bytes/bit) fobj.seek( len(header.tostring()) + (shape[0] * shape[1] * shape[2] * int(np.abs(header['BITPIX']) / 8)) - 1) fobj.write(b'\0') big_cube = SpectralCube.read(big_filename) header_cube = SpectralCube.read(header_fn) # in both cases, SpectralCube sorts the extrema if cdelt_sign == 1: np.testing.assert_array_almost_equal_nulp( big_cube.spectral_extrema[0].value, header_cube.spectral_extrema[0].value) np.testing.assert_array_almost_equal_nulp( big_cube.wcs.wcs.cdelt, header_cube.wcs.wcs.cdelt) elif cdelt_sign == -1: np.testing.assert_array_almost_equal_nulp( big_cube.spectral_extrema[1].value, header_cube.spectral_extrema[1].value) np.testing.assert_array_almost_equal_nulp( big_cube.wcs.wcs.cdelt[-1] * -1, header_cube.wcs.wcs.cdelt[-1]) log.info("Cube creation completed. Now moving on to populating it.") # Find the appropriate files (this is NOT a good way to do this! Better to # provide a list. But wildcards are quick & easy... fileglob = "OrionSourceI.B3.{0}.lines*{3}".format(spw, fntemplate, fnsuffix, filesuffix) files = glob.glob(fileglob) log.info("Files to be merged with glob {0}: ".format(fileglob)) log.info(str(files)) # open the file in update mode (it should have the right dims now) hdul = fits.open(big_filename, mode='update') main_wcs = wcs.WCS(hdul[0].header).sub([wcs.WCSSUB_SPECTRAL]) if add_beam_info: shape = hdul[0].data.shape[0] if len(hdul) > 1 and isinstance(hdul[1], fits.BinTableHDU): pass else: hdul.append( fits.BinTableHDU( np.recarray(shape, names=['BMAJ', 'BMIN', 'BPA', 'CHAN', 'POL'], formats=['f4', 'f4', 'f4', 'i4', 'i4']))) # sorted so that we deal with zero first, since it has potential to be a problem. for fn in ProgressBar(sorted(files)): log.info("inds={0} fn={1}".format(getinds(fn), fn)) ind0, ind1 = getinds(fn) # this is not correct...? # or maybe it only applies if cropends is set.... # if ind0 == 0: # ind1 = ind1 + 1 cdelt = fits.getheader(fn)['CDELT3'] if 'cdelt_sign' not in locals(): cdelt_sign = np.sign(cdelt) log.warn("cdelt_sign was not defined: overwriting a" " previously-existing file. " "This may not be what you want; the data could be going " "opposite the parent cube. Check that the original " "header is OK. sign(CDELT) is now {0}, " "while for the big header it is {1}".format( cdelt_sign, np.sign(fits.getheader(big_filename)['CDELT3']))) if cropends: # don't crop 1st or last pixel in full cube if ind0 > 0: log.debug("ind0 going from {0} to {1}".format( ind0, ind0 + cropends)) ind0 = ind0 + cropends if cdelt_sign == 1: dataind0 = cropends log.debug("dataind0 going to {0}".format(cropends)) else: dataind1 = -cropends log.debug("dataind1 going to {0}".format(-cropends)) else: if cdelt_sign == 1: dataind0 = 0 log.debug("dataind0 going to {0}".format(0)) elif cdelt_sign == -1: log.debug("dataind1 going to {0}".format(None)) dataind1 = None if (ind1 < nchans_total[spwnum] - 1): log.debug("ind1 going from {0} to {1}".format( ind1, ind1 - cropends)) ind1 = ind1 - cropends if cdelt_sign == 1: dataind1 = -cropends log.debug("dataind1 going to {0}".format(-cropends)) elif cdelt_sign == -1: dataind0 = cropends log.debug("dataind0 going to {0}".format(cropends)) else: if cdelt_sign == 1: dataind1 = None else: log.debug("dataind0 going to {0}".format(0)) dataind0 = 0 else: dataind0 = 0 dataind1 = None if cdelt_sign == -1: log.debug("Reversing indices from {0} {1} to ".format(ind0, ind1)) ind1, ind0 = (nchans_total[spwnum] - ind0, nchans_total[spwnum] - ind1) log.debug("{0} {1}".format(ind0, ind1)) if ind0 < 0: ind0 = 0 log.info("inds have been remapped to {0}, {1}".format(ind0, ind1)) plane = hdul[0].data[ind0] if np.all(plane == 0) or overwrite_existing: log.info("Replacing indices {0}->{2} {1}".format( getinds(fn), fn, (ind0, ind1))) data = fits.getdata(fn) dwcs = wcs.WCS(fits.getheader(fn)).sub([wcs.WCSSUB_SPECTRAL]) dataind1 = data.shape[0] + (dataind1 or 0) # handle the case where I made the indices NOT match the cube... # this is really stupid and should be removed because I should have # made the input cubes correct. Oh well. if np.abs(ind1 - ind0) < np.abs(dataind1 - dataind0): dataind1 = dataind0 + np.abs(ind1 - ind0) if cdelt_sign == -1: dataind0, dataind1 = dataind1, dataind0 dwcs0 = dwcs.wcs_pix2world([dataind0 - 1], 0)[0][0] dwcs1 = dwcs.wcs_pix2world([dataind1], 0)[0][0] else: dwcs0 = dwcs.wcs_pix2world([dataind0], 0)[0][0] dwcs1 = dwcs.wcs_pix2world([dataind1 - 1], 0)[0][0] hwcs0 = main_wcs.wcs_pix2world([ind0], 0)[0][0] hwcs1 = main_wcs.wcs_pix2world([ind1 - 1], 0)[0][0] if not np.isclose(hwcs0, dwcs0, atol=0.5 * np.abs(cdelt), rtol=0): log.error( "current data, big cube indices: {0},{1} and {2},{3}". format(dataind0, dataind1, ind0, ind1)) raise ValueError( "World coordinates of first pixels do not match: {0} - {1} = {2} ({3} cdelt)" .format(dwcs0, hwcs0, dwcs0 - hwcs0, (dwcs0 - hwcs0) / cdelt)) if not np.isclose(hwcs1, dwcs1, atol=0.5 * np.abs(cdelt), rtol=0): log.error( "current data, big cube indices: {0},{1} and {2},{3}". format(dataind0, dataind1, ind0, ind1)) raise ValueError( "World coordinates of last pixels do not match: {0} - {1} = {2} ({3} cdelt)" .format(dwcs1, hwcs1, dwcs1 - hwcs1, (dwcs1 - hwcs1) / cdelt)) if 'slices' not in locals(): if minimize: log.info("Determining slices") cube0 = SpectralCube.read(header_fn) slices = cube0.subcube_slices_from_mask(cube0.mask, spatial_only=True) log.info("Slices are {0}".format(slices)) else: slices = (slice(None), ) * 3 if bmaj_limits is not None: log.info("Identifying acceptable beams") beamtable = fits.open(fn)[1] ok_beam = ((beamtable.data['BMAJ'] > bmaj_limits[0]) & (beamtable.data['BMAJ'] < bmaj_limits[1])) data[~ok_beam] = np.nan log.info("Found {0} bad beams of {1}".format((~ok_beam).sum(), ok_beam.size)) if cdelt_sign == -1: if dataind1 == 0: dataslice = slice(dataind0 - 1, None, -1) elif dataind1 >= 1: dataslice = slice(dataind0 - 1, dataind1 - 1, -1) else: raise ValueError("Something is wrong with dataind0") else: dataslice = slice(dataind0, dataind1, 1) log.info("Dataslice is {0}".format(dataslice)) assert hdul[0].data[ind0:ind1].shape == data[dataslice, slices[1], slices[2]].shape if not debug_mode: if add_beam_info: log.info("Adding beam information") beamtable = fits.open(fn)[1] hdul[1].data[ind0:ind1] = beamtable.data[dataslice] log.info("Inserting data") hdul[0].data[ind0:ind1, :, :] = data[dataslice, slices[1], slices[2]] log.info("Flushing") hdul.flush() log.info("Done with iteration for {0}".format(fn)) if debug_mode: log.setLevel(lvl)
hdu_table.meta.pop("BASE_DIR") # Add the rows for the background HDUs hdu_table = vstack([hdu_table, hdu_table_bkg]) hdu_table.sort("OBS_ID") # In[ ]: hdu_table[:7] # In[ ]: # Put index tables and background models in a FITS file hdu_list = fits.HDUList() hdu = fits.BinTableHDU(hdu_table) hdu.name = "HDU_INDEX" hdu_list.append(hdu) hdu = fits.BinTableHDU(data_store.obs_table) hdu_list.append(hdu) for idx, model in enumerate(models): hdu = model.to_fits() hdu.name = "BKG{}".format(idx) hdu_list.append(hdu) print([_.name for _ in hdu_list]) import os
def to_fits(self, name="BACKGROUND"): """Convert to `~astropy.io.fits.BinTableHDU`.""" return fits.BinTableHDU(self.to_table(), name=name)
def _write_filename(filename, c_hdr, s_hdr, cov_index_map, sparse_map, compress=False, compress_tilesize=None): """ Write to a filename, using fitsio or astropy.io.fits. This assumes that you want to overwrite any existing file (as should be checked in calling function). Parameters ---------- filename : `str` Name of file to write to. c_hdr : `fitsio.FITSHDR` or `astropy.io.fits.Header` Coverage index map header s_hdr : `fitsio.FITSHDR` or `astropy.io.fits.Header` Sparse map header cov_index_map : `np.ndarray` Coverage index map sparse_map : `np.ndarray` Sparse map compress : `bool`, optional Write with FITS compression? """ # Currently, all writing is done with astropy.io.fits because it supports # lossless compression of floating point data. Unfortunately, the header # is wrong so we have a header patch below. c_hdr['EXTNAME'] = 'COV' s_hdr['EXTNAME'] = 'SPARSE' hdu_list = fits.HDUList() hdu = fits.PrimaryHDU(data=cov_index_map, header=fits.Header()) for n in c_hdr: if n not in FITS_RESERVED: hdu.header[n] = c_hdr[n] hdu_list.append(hdu) if compress: hdu = fits.CompImageHDU(data=sparse_map, header=fits.Header(), compression_type='GZIP_2', tile_size=(compress_tilesize, ), quantize_level=0.0) else: if sparse_map.dtype.fields is not None: hdu = fits.BinTableHDU(data=sparse_map, header=fits.Header()) else: hdu = fits.ImageHDU(data=sparse_map, header=fits.Header()) for n in s_hdr: if n not in FITS_RESERVED: hdu.header[n] = s_hdr[n] hdu_list.append(hdu) hdu_list.writeto(filename, overwrite=True) # When writing a gzip unquantized (lossless) floating point image, # current versions of astropy (4.0.1 and earlier, at least) write # the ZQUANTIZ header value as NO_DITHER, while cfitsio expects # this to be NONE for unquantized data. The only way to overwrite # this reserved header keyword is to manually overwrite the bytes # in the file. The following code uses mmap to overwrite the # necessary header keyword without loading the full image into # memory. Note that healsparse files only have one compressed # extension, so there will only be one use of ZQUANTIZ in the file. if compress and not is_integer_value(sparse_map[0]): with open(filename, "r+b") as f: try: mm = mmap.mmap(f.fileno(), 0) loc = mm.find(b"ZQUANTIZ= 'NO_DITHER'") if loc >= 0: mm.seek(loc) mm.write(b"ZQUANTIZ= 'NONE '") except OSError: # Some systems do not have the mmap available, # we need to read in the full file. data = f.read() loc = data.find(b"ZQUANTIZ= 'NO_DITHER'") if loc >= 0: f.seek(loc) f.write(b"ZQUANTIZ= 'NONE '")
def save_results(wd, mode, header, neon_header, superbias, flat_coeff, WL_map, Y_correction, reduct_star, star_wl, sensitivity, obj, err, neon, dots, obj_bin, err_bin, sep, medium_results, sky): fits.PrimaryHDU(sky.astype('float32')).writeto(wd + 'sky.fits', overwrite=True) fits.PrimaryHDU(medium_results['initial'].astype('float32')).writeto( wd + 'm_initial.fits', overwrite=True) if mode['bias'] == 'Yes': fits.PrimaryHDU(superbias.astype('float32'), header=header).writeto(wd + 'sbias.fits', overwrite=True) if mode['bias'] != 'No': fits.PrimaryHDU(medium_results['bias'].astype('float32')).writeto( wd + 'm_bias.fits', overwrite=True) if mode['flat'] == 'Yes': hdu = fits.PrimaryHDU(flat_coeff.astype('float32')) hdu.writeto(wd + 'flatcoef.fits', overwrite=True) if mode['flat'] != 'No': fits.PrimaryHDU(medium_results['flat'].astype('float32')).writeto( wd + 'm_flat.fits', overwrite=True) if mode['x_lin'] == 'Yes': fits.PrimaryHDU(WL_map.astype('float32')).writeto(wd + 'WL_map.fits', overwrite=True) fits.PrimaryHDU(neon.astype('float32'), header=neon_header).writeto(wd + 'neon.fits', overwrite=True) if mode['y_lin'] == 'Yes': fits.PrimaryHDU(Y_correction.astype('float32')).writeto( wd + 'Y_correction.fits', overwrite=True) fits.PrimaryHDU(dots.astype('float32')).writeto(wd + 'dots.fits', overwrite=True) if mode['x_lin'] != 'No' or mode['y_lin'] != 'No' or mode['WL'] != 'No': fits.PrimaryHDU( medium_results['linerization'].astype('float32')).writeto( wd + 'm_linerization.fits', overwrite=True) if mode['star'] == 'Yes': star_image = fits.PrimaryHDU(reduct_star.astype('float32')) star_k = fits.BinTableHDU.from_columns([ fits.Column(name='wavelenght', format='E', array=star_wl), fits.Column(name='sensitivity', format='E', array=sensitivity) ]) fits.HDUList(hdus=[star_image, star_k]).writeto(wd + 'star.fits', overwrite=True) if mode['star'] != 'No': fits.PrimaryHDU(medium_results['star'].astype('float32')).writeto( wd + 'm_star.fits', overwrite=True) if mode['cosmics'] != 'No': fits.PrimaryHDU(medium_results['cosmics'].astype('float32')).writeto( wd + 'm_cosmics.fits', overwrite=True) fits.PrimaryHDU(obj.astype('float32'), header=header).writeto(wd + 'result.fits', overwrite=True) fits.PrimaryHDU(err.astype('float32'), header=header).writeto(wd + 'errors.fits', overwrite=True) SNR = obj / err fits.PrimaryHDU(SNR.astype('float32')).writeto(wd + 'SNR.fits', overwrite=True) if mode['bin'] == 'Yes': image = fits.PrimaryHDU(obj_bin.astype('float32'), header=header) table = fits.BinTableHDU(data=Table([sep])) fits.HDUList(hdus=[image, table]).writeto(wd + 'H_bins.fits', overwrite=True) image = fits.PrimaryHDU(err_bin.astype('float32'), header=header) fits.HDUList(hdus=[image, table]).writeto(wd + 'H_err_bins.fits', overwrite=True) image = fits.PrimaryHDU((obj_bin / err_bin).astype('float32'), header=header) fits.HDUList(hdus=[image, table]).writeto(wd + 'H_SNR_bins.fits', overwrite=True)
return d, h2 if __name__ == "__main__": np.random.seed(42) beams = np.recarray(4, dtype=[('BMAJ', '>f4'), ('BMIN', '>f4'), ('BPA', '>f4'), ('CHAN', '>i4'), ('POL', '>i4')]) beams['BMAJ'] = [0.4,0.3,0.3,0.4] # arcseconds beams['BMIN'] = [0.1,0.2,0.2,0.1] beams['BPA'] = [0,45,60,30] # degrees beams['CHAN'] = [0,1,2,3] beams['POL'] = [0,0,0,0] beams = fits.BinTableHDU(beams) # Single Stokes h = fits.header.Header.fromtextfile(HEADER_FILENAME) h['BUNIT'] = 'K' # Kelvins are a valid unit, JY/BEAM are not: they should be tested separately h['NAXIS1'] = 2 h['NAXIS2'] = 3 h['NAXIS3'] = 4 h['NAXIS4'] = 1 d = np.random.random((1, 2, 3, 4)) fits.writeto('advs.fits', d, h, overwrite=True) d, h = transpose(d, h, [1, 2, 3, 0]) fits.writeto('dvsa.fits', d, h, overwrite=True)
def tpf2lc(fname, radii, aper_shape='round', outlier_sigma=5, flat_window=301, corr_window=51, cutoff_limit=1.0, polyorder=4, break_tolerance=5,save_as_tpf=False, verbose=False, outdir='reduced'): ''' Do aperture photometry with multiple apertures and a mask. The light curve with a given aperture is appended with the original data and saved as separate fits. The best light curve/aperture (determined with cdpp) is saved in index=1. ''' fname_new = os.path.join(outdir,epic+'_'+aper_shape+'.fits') if print('\nAperture photometry with r={} and {} mask...\n'.format(radii,aper_shape)) if verbose: print("""sigma cut for outliers: {}\nwindow length (flatten): {}\nwindow length (sff): {}\ncutoff limit (if mask=irregular): {}\n """.format(outlier_sigma,flat_window,corr_window,cutoff_limit)) hdr = fits.getheader(fname) hdulist = fits.open(fname) tpf = KeplerTargetPixelFile(fname, quality_bitmask='hardest') epic=str(tpf.keplerid) if epic not in hdulist.filename(): raise ValueError('Kepler ID in header doesn\'t match the filename') flux_per_r = {} cdpps = {} for r in radii: mask = make_mask(tpf.flux, rad=r, shape=aper_shape, epic=epic) lc = tpf.to_lightcurve(aperture_mask=mask); lc2 = lc.remove_nans().remove_outliers(sigma=outlier_sigma) flat_lc2, trend = lc2.flatten(window_length=flat_window, polyorder=polyorder, break_tolerance=break_tolerance, return_trend=True) corr_lc = flat_lc2.correct(method='sff',windows=corr_window) flux_per_r[r]=(corr_lc.time,corr_lc.flux,corr_lc.flux_err) cdpps[r] = corr_lc.cdpp() ###TO DO: add lc generated with irregular mask mask = make_mask(tpf.flux, rad=r, shape='irregular', epic=epic) lc = tpf.to_lightcurve(aperture_mask=mask); lc2 = lc.remove_nans().remove_outliers(sigma=outlier_sigma) flat_lc2, trend = lc2.flatten(window_length=flat_window, polyorder=polyorder, break_tolerance=break_tolerance, return_trend=True) corr_lc = flat_lc2.correct(method='sff',windows=corr_window) flux_per_r['irreg']=(corr_lc.time,corr_lc.flux,corr_lc.flux_err) cdpps['irreg'] = corr_lc.cdpp() if save_as_tpf: cdpp_list=[] #append to hdulist photometry of each aperture and save for num,r in enumerate(flux_per_r): comment_num = 'COMMENT{}'.format(num) aper_name = '{}_APER{}'.format(aper_shape,num) hdr['ap_rad'] = r hdr['ap_shape'] = aper_shape hdr['cdpp'] = cdpps[r] cdpp_list.append(cdpps[r]) tab = table.Table(flux_per_r[r], names=['time','flux','flux_err']) bintab=fits.BinTableHDU(tab,name=aper_name,header=hdr) #append bin table to original hdulist hdulist.append(bintab) #find smallest cdpp best_r=min(cdpps.items(), key=operator.itemgetter(1))[0] #re-create bin table #tab = table.Table(flux_per_r[best_r], names=['time','flux','flux_err']) #bintab=fits.BinTableHDU(tab,name=aper_name,header=hdr) #move to index 3 (0: primary; 1: table; 2: aperture) #hdulist.insert(3, bintab) #alternatively, move hdu into last index of hdulist hdulist += [hdulist.pop(3+np.argmin(cdpp_list))] #make hdu for best mask if best_r == 'irreg': mask = make_mask(tpf.flux, rad=r, shape='irreg', epic=epic) else: mask = make_mask(tpf.flux, rad=best_r, shape=aper_shape, epic=epic) hdu=fits.hdu.ImageHDU(np.array(mask,dtype=float), name='APERTURE', header=hdr) #problem with bool #replace aperture hdulist[2] = hdu #save fits #fname_new = os.path.join(outdir,fname.split('/')[-1].split('-')[0][4:]+'_'+aper_shape+'.fits') if not os.path.exists(outdir): os.makedirs(outdir) hdulist.writeto(fname_new) print('Saved: {}\n'.format(fname_new)) return flux_per_r, mask def read_tpf(fname,index,return_hdr=True): ''' fname: str, filename index: int, hdulist index [0,1,2] = primary, target table, aperture mask [4,...] = photometry using specified aperture return_hdr: bool ''' hdulist = fits.open(fname) if index == 0: #primary data = hdulist[index].data hdr = hdulist[index].header if return_hdr: return data, hdr else: return data elif index == 1: #target tables data = hdulist[index].data hdr = hdulist[index].header if return_hdr: return data, hdr else: return data elif index == 2: #aperture mask data = hdulist[index].data hdr = hdulist[index].header if return_hdr: return data, hdr else: return data else: df=table.Table(hdulist[index].data).to_pandas() hdr = hdulist[index].header if return_hdr: return df, hdr else: return df def plot_lc(fname,index,verbose=True,show_all_lc=False,show_mask=True,sigma=None): hdulist = fits.open(fname) hdulen = len(hdulist) tpf = KeplerTargetPixelFile(fname, quality_bitmask='hardest') if verbose: print(hdulist.info()) if index == 0: #primary data, hdr = read_tpf(fname,index,return_hdr=True) print('Plot of primary hdu currently unsupported. Try index=2\n') sys.exit() elif index == 1: #target tables data, hdr = read_tpf(fname,index,return_hdr=True) print('Plot of target table currently unsupported. Try index=2\n') sys.exit() nrows=hdulen-3 #remove first 3 indices fig, ax = pl.subplots(nrows=nrows,ncols=1,figsize=(10,10),sharex=True,squeeze=True) if index<hdulen and index>2: #read flux per r epic = str(tpf.keplerid) if show_all_lc: n=0 for idx in np.arange(3,hdulen,1): df, hdr = read_tpf(fname,idx,return_hdr=True) t = df['time'] f = df['flux'] ferr = df['flux_err'] rad = hdr['ap_rad'] shape = hdr['ap_shape'] if sigma is not None: f_c=sigma_clip(f, sigma=sigma) t = t[~f_c.mask] f = f[~f_c.mask] ferr = ferr[~f_c.mask] print('removed {} outliers.\n'.format(np.sum(f_c.mask))) ax[n].errorbar(t,f,yerr=ferr,marker='o',label='r={}'.format(rad)) ax[n].legend() n+=1 pl.title('EPIC-'+epic) else: df, hdr = read_tpf(fname,index,return_hdr=True) t = df['time'] f = df['flux'] ferr = df['ferr'] rad = hdr['ap_radius'] shape = hdr['ap_shape'] ax.errorbar(t,f,yerr=ferr,marker='o',label='r={}'.format(rad)) ax.set_title(epic) pl.legend() if show_mask: mask = hdulist[2].data #fluxes = tpf.flux #plot_aper_mask(fluxes,rad,aper_shape=shape,contrast=0.1,epic=tpf.keplerid) ax = tpf.plot(aperture_mask=mask, mask_color='w', #frame=300, scale='linear', cmap='viridis', show_colorbar=True) pl.show() elif index>hdulen: print('hdulist has index until {} only. Exiting!\n'.format(hdulen)) sys.exit() else: print('Incorrect index. Set verbose=True.\n') return fig #---------------------------STATS---------------------------# def noise_statistic(t, f, timescale=0.25, verbose=False): ''' c.f. lightkurve.cdpp() ''' nchunks = int((t[-1]-t[0])/timescale)+1 idx = [(t > t[0] + n * timescale) & (t < t[0] + (n + 1) * timescale) for n in range(nchunks)] chunks = [f[ix] for ix in idx if ix.sum() > 1] cdpp = np.std([np.nanmedian(ch) for ch in chunks]) if verbose: print('cdpp = {:.4f}'.format(cdpp)) return cdpp
def precompute_synthetic_grid(output_dirname, ranges, wavelengths, to_resolution, modeled_layers_pack, atomic_linelist, isotopes, solar_abundances, segments=None, number_of_processes=1, code="spectrum", use_molecules=False, steps=False, tmp_dir=None): """ Pre-compute a synthetic grid with some reference ranges (Teff, log(g) and MH combinations) and all the steps that iSpec will perform in the astrophysical parameter determination process. All the non-convolved spectra will be saved in a subdir and a complete grid file with the reference points already convolved will be saved in a FITS file for fast comparison. The output directory can be used by the routines 'model_spectrum' and 'estimate_initial_ap'. """ code = code.lower() if code not in ['spectrum', 'turbospectrum', 'moog', 'synthe', 'sme']: raise Exception("Unknown radiative transfer code: %s" % (code)) reference_list_filename = output_dirname + "/parameters.tsv" if to_resolution is not None: reference_grid_filename = output_dirname + "/convolved_grid_%i.fits.gz" % to_resolution fits_dir = os.path.join(output_dirname, "grid/") mkdir_p(fits_dir) if steps: steps_fits_dir = os.path.join(output_dirname, "steps/") mkdir_p(steps_fits_dir) import dill # To allow pickle of lambda functions (e.g., one element in modeled_layers_pack) import pickle pickled_modeled_layers_pack = pickle.dumps(modeled_layers_pack) # For code != "grid", ranges are always in position 7 (for grid it would be in position 8) valid_ranges = modeled_layers_pack[7] teff_range = valid_ranges['teff'] logg_range = valid_ranges['logg'] MH_range = valid_ranges['MH'] alpha_range = valid_ranges.get( 'alpha', (-1.5, 1.5) ) # Default (0.,) if 'alpha' is not a free parameter for atmosphere interpolation vmic_range = valid_ranges.get( 'vmic', (0.0, 50.) ) # Default (0.,) if 'vmic' is not a free parameter for atmosphere interpolation # Parallelization pool if number_of_processes == 1: pool = None else: pool = Pool(number_of_processes) # Create grid binary file elapsed = 0 # seconds num_ref_spec = len(ranges) num_spec = num_ref_spec * 9 # Reference + 8 variations in Teff, logg, MH, alpha, vmic, vmac, vsini, limb darkening coeff i = 0 for teff, logg, MH, alpha, vmic in ranges: if vmic is None: vmic = estimate_vmic(teff, logg, MH) vmac = 0.0 # This can be modified after synthesis if needed vsini = 0.0 # This can be modified after synthesis if needed limb_darkening_coeff = 0.00 # This can be modified after synthesis if needed resolution = 0 # This can be modified after synthesis if needed is_step = False if not valid_atmosphere_target(modeled_layers_pack, { 'teff': teff, 'logg': logg, 'MH': MH, 'alpha': alpha }): raise Exception( "Target parameters out of the valid ranges: teff={} logg={} MH={} alpha={}" .format(teff, logg, MH, alpha)) points = [ (teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), ] if steps: is_step = True new_teff = teff + Constants.SYNTH_STEP_TEFF if teff + Constants.SYNTH_STEP_TEFF <= teff_range[ -1] else teff - Constants.SYNTH_STEP_TEFF new_logg = logg + Constants.SYNTH_STEP_LOGG if logg + Constants.SYNTH_STEP_LOGG <= logg_range[ -1] else logg - Constants.SYNTH_STEP_LOGG new_MH = MH + Constants.SYNTH_STEP_MH if MH + Constants.SYNTH_STEP_MH <= MH_range[ -1] else MH - Constants.SYNTH_STEP_MH new_alpha = alpha + Constants.SYNTH_STEP_ALPHA if alpha + Constants.SYNTH_STEP_ALPHA <= alpha_range[ -1] else alpha - Constants.SYNTH_STEP_ALPHA new_vmic = vmic + Constants.SYNTH_STEP_VMIC if vmic + Constants.SYNTH_STEP_VMIC <= vmic_range[ -1] else vmic - Constants.SYNTH_STEP_VMIC # For each reference point, calculate also the variations that iSpec will perform in the first iteration points += [ # Final unconvolved spectra where vmic/vmac are free and do not follow vmic/vmac empirical relations (new_teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, new_logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, new_MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, MH, new_alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, MH, alpha, new_vmic, vmac, vsini, limb_darkening_coeff, is_step), ] points += [ # Final unconvolved spectra where vmic is not free and does follow vmic empirical relations (new_teff, logg, MH, alpha, estimate_vmic(new_teff, logg, MH), vmac, vsini, limb_darkening_coeff, is_step), (teff, new_logg, MH, alpha, estimate_vmic(teff, new_logg, MH), vmac, vsini, limb_darkening_coeff, is_step), (teff, logg, new_MH, alpha, estimate_vmic(teff, logg, new_MH), vmac, vsini, limb_darkening_coeff, is_step), ] for j, (teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, is_step) in enumerate(points): if is_step: filename_out = steps_fits_dir + "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff) + ".fits.gz" else: filename_out = fits_dir + "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff) + ".fits.gz" if os.path.exists(filename_out): print("Skipping", teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, "already computed") continue if pool is None: if sys.platform == "win32": # On Windows, the best timer is time.clock() default_timer = time.clock else: # On most other platforms the best timer is time.time() default_timer = time.time lock = FileLock(filename_out + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is computing this spectrum, do not continue print("Skipping", teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, "already locked") continue try: tcheck = default_timer() # Validate parameters __generate_synthetic_fits(filename_out, wavelengths, segments, teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, resolution, pickled_modeled_layers_pack, atomic_linelist, isotopes, solar_abundances, code=code, use_molecules=use_molecules, tmp_dir=tmp_dir, locked=True) elapsed = default_timer() - tcheck print( "-----------------------------------------------------" ) print("Remaining time:") print("\t", (num_spec - i) * elapsed, "seconds") print("\t", (num_spec - i) * (elapsed / 60), "minutes") print("\t", (num_spec - i) * (elapsed / (60 * 60)), "hours") print("\t", (num_spec - i) * (elapsed / (60 * 60 * 24)), "days") print( "-----------------------------------------------------" ) finally: lock.release() else: pool.apply_async(__generate_synthetic_fits, [ filename_out, wavelengths, segments, teff, logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff, resolution, pickled_modeled_layers_pack, atomic_linelist, isotopes, solar_abundances ], kwds={ 'code': code, 'use_molecules': use_molecules, 'tmp_dir': tmp_dir, 'locked': False }) i += 1 if pool is not None: pool.close() pool.join() # Create parameters.tsv reference_list = Table() if len(np.unique(ranges[['logg', 'MH', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_teff', dtype=int)) else: reference_list.add_column(Column(name='teff', dtype=int)) if len(np.unique(ranges[['teff', 'MH', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_logg', dtype=float)) else: reference_list.add_column(Column(name='logg', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'alpha', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_MH', dtype=float)) else: reference_list.add_column(Column(name='MH', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'MH', 'vmic']])) == len(ranges): reference_list.add_column(Column(name='fixed_alpha', dtype=float)) else: reference_list.add_column(Column(name='alpha', dtype=float)) if len(np.unique(ranges[['teff', 'logg', 'MH', 'alpha']])) == len(ranges): reference_list.add_column(Column(name='fixed_vmic', dtype=float)) else: reference_list.add_column(Column(name='vmic', dtype=float)) reference_list.add_column(Column(name='filename', dtype='|S100')) for teff, logg, MH, alpha, vmic in ranges: # Only use the first spectra generated for each combination zero_vmac = 0.0 zero_vsini = 0.0 zero_limb_darkening_coeff = 0.00 reference_filename_out = "./grid/{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, zero_vmac, zero_vsini, zero_limb_darkening_coeff) + ".fits.gz" reference_list.add_row( (int(teff), logg, MH, alpha, vmic, reference_filename_out)) if not os.path.exists(reference_list_filename): lock = FileLock(reference_list_filename + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is writing this file, do not continue print("Skipping", reference_list_filename, "already locked") else: try: ascii.write(reference_list, reference_list_filename, delimiter='\t', overwrite=True) print("Written", reference_list_filename) finally: lock.release() if to_resolution is not None: if not os.path.exists(reference_grid_filename): lock = FileLock(reference_grid_filename + ".lock") try: lock.acquire(timeout=-1) # Don't wait except (LockTimeout, AlreadyLocked) as e: # Some other process is computing this spectrum, do not continue print("Skipping", reference_grid_filename, "already locked") else: try: reference_grid = None complete_reference_list = Table() complete_reference_list.add_column( Column(name='teff', dtype=int)) complete_reference_list.add_column( Column(name='logg', dtype=float)) complete_reference_list.add_column( Column(name='MH', dtype=float)) complete_reference_list.add_column( Column(name='alpha', dtype=float)) complete_reference_list.add_column( Column(name='vmic', dtype=float)) complete_reference_list.add_column( Column(name='vmac', dtype=float)) complete_reference_list.add_column( Column(name='vsini', dtype=float)) complete_reference_list.add_column( Column(name='limb_darkening_coeff', dtype=float)) for teff, logg, MH, alpha, vmic in ranges: # Only use the first spectra generated for each combination zero_vmac = 0.0 zero_vsini = 0.0 zero_limb_darkening_coeff = 0.00 vmac = estimate_vmac(teff, logg, MH) vsini = 1.6 # Sun limb_darkening_coeff = 0.6 reference_filename_out = "{0}_{1:.2f}_{2:.2f}_{3:.2f}_{4:.2f}_{5:.2f}_{6:.2f}_{7:.2f}".format( int(teff), logg, MH, alpha, vmic, zero_vmac, zero_vsini, zero_limb_darkening_coeff) + ".fits.gz" if not os.path.exists(fits_dir + reference_filename_out): continue complete_reference_list.add_row( (int(teff), logg, MH, alpha, vmic, vmac, vsini, limb_darkening_coeff)) # Spectra in the grid is convolved to the specified resolution for fast comparison print("Quick grid:", reference_filename_out) spectrum = read_spectrum(fits_dir + reference_filename_out) segments = None vrad = (0, ) spectrum['flux'] = apply_post_fundamental_effects(spectrum['waveobs'], spectrum['flux'], segments, \ macroturbulence=vmac, vsini=vsini, \ limb_darkening_coeff=limb_darkening_coeff, R=to_resolution, vrad=vrad) if reference_grid is None: reference_grid = spectrum['flux'] else: reference_grid = np.vstack( (reference_grid, spectrum['flux'])) if len(ranges) == len(complete_reference_list): # Generate FITS file with grid for fast comparison primary_hdu = fits.PrimaryHDU(reference_grid) wavelengths_hdu = fits.ImageHDU(wavelengths, name="WAVELENGTHS") params_bintable_hdu = fits.BinTableHDU( complete_reference_list.as_array(), name="PARAMS") fits_format = fits.HDUList([ primary_hdu, wavelengths_hdu, params_bintable_hdu ]) fits_format.writeto(reference_grid_filename, overwrite=True) print("Written", reference_grid_filename) finally: lock.release()
def to_ogip_files(self, outdir=None, use_sherpa=False, overwrite=False): """Write OGIP files. If you want to use the written files with Sherpa you have to set the ``use_sherpa`` flag. Then all files will be written in units 'keV' and 'cm2'. The naming scheme is fixed, with {name} the dataset name: * PHA file is named pha_obs{name}.fits * BKG file is named bkg_obs{name}.fits * ARF file is named arf_obs{name}.fits * RMF file is named rmf_obs{name}.fits Parameters ---------- outdir : `pathlib.Path` output directory, default: pwd use_sherpa : bool, optional Write Sherpa compliant files, default: False overwrite : bool Overwrite existing files? """ # TODO: refactor and reduce amount of code duplication outdir = Path.cwd() if outdir is None else make_path(outdir) outdir.mkdir(exist_ok=True, parents=True) phafile = f"pha_obs{self.name}.fits" bkgfile = phafile.replace("pha", "bkg") arffile = phafile.replace("pha", "arf") rmffile = phafile.replace("pha", "rmf") counts_table = self.counts.to_table() counts_table["QUALITY"] = np.logical_not(self.mask_safe) counts_table["BACKSCAL"] = self.acceptance counts_table["AREASCAL"] = np.ones(self.acceptance.size) meta = self._ogip_meta() meta["respfile"] = rmffile meta["backfile"] = bkgfile meta["ancrfile"] = arffile meta["hduclas2"] = "TOTAL" counts_table.meta = meta name = counts_table.meta["name"] hdu = fits.BinTableHDU(counts_table, name=name) hdulist = fits.HDUList( [fits.PrimaryHDU(), hdu, self._ebounds_hdu(use_sherpa)]) if self.gti is not None: hdu = fits.BinTableHDU(self.gti.table, name="GTI") hdulist.append(hdu) hdulist.writeto(outdir / phafile, overwrite=overwrite) self.aeff.write(outdir / arffile, overwrite=overwrite, use_sherpa=use_sherpa) if self.counts_off is not None: counts_off_table = self.counts_off.to_table() counts_off_table["QUALITY"] = np.logical_not(self.mask_safe) counts_off_table["BACKSCAL"] = self.acceptance_off counts_off_table["AREASCAL"] = np.ones(self.acceptance.size) meta = self._ogip_meta() meta["hduclas2"] = "BKG" counts_off_table.meta = meta name = counts_off_table.meta["name"] hdu = fits.BinTableHDU(counts_off_table, name=name) hdulist = fits.HDUList( [fits.PrimaryHDU(), hdu, self._ebounds_hdu(use_sherpa)]) hdulist.writeto(outdir / bkgfile, overwrite=overwrite) if self.edisp is not None: self.edisp.write(outdir / rmffile, overwrite=overwrite, use_sherpa=use_sherpa)
def main(name,version,HSTband,scalepc): '''match nebulae and association catalogue ''' print(f'parameters: {name} {version} {HSTband} {scalepc}') # ===================================================================== # Read in the data # ===================================================================== #p = {x:sample_table.loc[name][x] for x in sample_table.columns} # DAP linemaps (Halpha and OIII) filename = data_ext / 'MUSE' / 'DR2.1' / 'copt' / 'MUSEDAP' filename = [x for x in filename.iterdir() if x.stem.startswith(name)][0] with fits.open(filename) as hdul: Halpha = NDData(data=hdul['HA6562_FLUX'].data, uncertainty=StdDevUncertainty(hdul['HA6562_FLUX_ERR'].data), mask=np.isnan(hdul['HA6562_FLUX'].data), meta=hdul['HA6562_FLUX'].header, wcs=WCS(hdul['HA6562_FLUX'].header)) OIII = NDData(data=hdul['OIII5006_FLUX'].data, uncertainty=StdDevUncertainty(hdul['OIII5006_FLUX_ERR'].data), mask=np.isnan(hdul['OIII5006_FLUX'].data), meta=hdul['OIII5006_FLUX'].header, wcs=WCS(hdul['OIII5006_FLUX'].header)) # the original catalogue from Francesco with fits.open(nebulae_file) as hdul: nebulae = Table(hdul[1].data) nebulae['SkyCoord'] = SkyCoord(nebulae['cen_ra']*u.deg,nebulae['cen_dec']*u.deg,frame='icrs') ''' with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_dig.fits') as hdul: dig = Table(hdul[1].data) with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_fuv.fits') as hdul: fuv = Table(hdul[1].data) with fits.open(basedir/'data'/'interim'/f'Nebulae_Catalogue_v2p1_eq.fits') as hdul: eq_width = Table(hdul[1].data) nebulae = join(nebulae,fuv,keys=['gal_name','region_ID']) nebulae = join(nebulae,eq_width,keys=['gal_name','region_ID']) nebulae = join(nebulae,dig,keys=['gal_name','region_ID'])' ''' nebulae.rename_columns(['cen_x','cen_y'],['x','y']) with np.errstate(divide='ignore',invalid='ignore'): nebulae['[SIII]/[SII]'] = np.nan SII = nebulae['SII6716_FLUX_CORR']+nebulae['SII6730_FLUX_CORR'] SIII = nebulae['SIII6312_FLUX_CORR']+nebulae['SIII9068_FLUX_CORR'] nebulae[SII>0]['[SIII]/[SII]'] = SIII[SII>0]/SII[SII>0] #nebulae['HA/FUV'] = nebulae['HA6562_FLUX_CORR']/nebulae['FUV_FLUX_CORR'] #nebulae['HA/FUV_err'] = nebulae['HA/FUV']*np.sqrt((nebulae['HA6562_FLUX_CORR_ERR']/nebulae['HA6562_FLUX_CORR'])**2+(nebulae['FUV_FLUX_CORR_ERR']/nebulae['FUV_FLUX_CORR'])**2) nebulae = nebulae[nebulae['gal_name']==name] nebulae.add_index('region_ID') filename = data_ext / 'Products' / 'Nebulae_catalogs'/'Nebulae_catalogue_v2' /'spatial_masks'/f'{name}_nebulae_mask_V2.fits' with fits.open(filename) as hdul: nebulae_mask = NDData(hdul[0].data.astype(float),mask=Halpha.mask,meta=hdul[0].header,wcs=WCS(hdul[0].header)) nebulae_mask.data[nebulae_mask.data==-1] = np.nan #print(f'{name}: {len(nebulae)} HII-regions in final catalogue') # the association catalogue and mask target = name.lower() associations, associations_mask = read_associations(folder=association_folder, target=target,scalepc=scalepc, HSTband=HSTband,version=version,data='all') if not associations: return 0 # enviornmental masks with fits.open(env_masks_folder / f'{name}_simple.fits') as hdul: mask = reproject_interp(hdul[0],Halpha.meta,order='nearest-neighbor',return_footprint=False) env_masks_neb = NDData(data=mask, meta=hdul[0].header, wcs=Halpha.wcs) #print(f'{name}: {len(associations)} associations in catalogue') # ===================================================================== # reproject and match catalogues # ===================================================================== nebulae_hst, _ = reproject_interp(nebulae_mask, output_projection=associations_mask.wcs, shape_out=associations_mask.data.shape, order='nearest-neighbor') # we scale the associations such that the the id is in the decimal scale = 10**np.ceil(np.log10(max(associations_mask.data[~np.isnan(associations_mask.data)]))) s_arr = associations_mask.data/scale+nebulae_hst #print(f'masks reprojected') # ids of associations, nebulae and combination (sum) of both a_id = np.unique(associations_mask.data[~np.isnan(associations_mask.data)]).astype(int) n_id = np.unique(nebulae_mask.data[~np.isnan(nebulae_mask.data)]).astype(int) s_id = np.unique(s_arr[~np.isnan(s_arr)]) # this splits the sum into two parts (nebulae and associations) a_modf,n_modf = np.modf(s_id) n_modf = n_modf.astype(int) a_modf = np.round(a_modf*scale).astype(int) unique_a, count_a = np.unique(a_modf,return_counts=True) unique_n, count_n = np.unique(n_modf,return_counts=True) nebulae_dict = {int(n) : a_modf[n_modf==n].tolist() for n in n_id} associations_dict = {int(a) : n_modf[a_modf==a].tolist() for a in a_id} # so far we ensured that the nebulae in unique_n have only one association, # but it is possible that this association goes beyond the nebulae and into # a second nebulae. Those objects are excluded here isolated_nebulae = set() isolated_assoc = set() for n,v in nebulae_dict.items(): if len(v)==1: if len(associations_dict[v[0]])==1: isolated_nebulae.add(n) isolated_assoc.add(v[0]) #print(f'n_associations = {len(associations_dict)}') #print(f'n_nebulae = {len(nebulae_dict)}') #print(f'1to1 match = {len(isolated_nebulae)}') # we save those two dicts so we do not have to redo this everytime with open(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_nebulae.yml','w+') as f: yaml.dump(nebulae_dict,f) with open(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations.yml','w+') as f: yaml.dump(associations_dict,f) # find all assoc that have at least one pixel outside of the nebulae masks mask = associations_mask.data.copy() mask[~np.isnan(nebulae_hst)] = np.nan outside = np.unique(mask[~np.isnan(mask)].astype(int)) # find all assoc that have at least one pixel inside of the nebulea masks mask = associations_mask.data.copy() mask[np.isnan(nebulae_hst)] = np.nan inside = np.unique(mask[~np.isnan(mask)].astype(int)) contained = np.setdiff1d(inside,outside) partial = np.intersect1d(inside,outside) isolated = np.setdiff1d(outside,inside) #print(f'contained: {len(contained)}\npartial: {len(partial)}\nisolated: {len(isolated)}') assoc_tmp = associations[['assoc_ID']].copy() assoc_tmp.add_index('assoc_ID') x_asc,y_asc = associations['SkyCoord'].to_pixel(env_masks_neb.wcs) outside = (x_asc > env_masks_neb.data.shape[1]) | (y_asc > env_masks_neb.data.shape[0]) x_asc[outside] = 0 y_asc[outside] = 0 assoc_tmp['env_asc'] = [environment_dict[env_masks_neb.data[y,x]] for x,y in zip(x_asc.astype(int),y_asc.astype(int))] assoc_tmp[outside]['env_asc'] = '' assoc_tmp['overlap'] = np.empty(len(associations),dtype='U9') assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],contained)] = 'contained' assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],partial)] = 'partial' assoc_tmp['overlap'][np.isin(assoc_tmp['assoc_ID'],isolated)] = 'isolated' assoc_tmp['1to1'] = False assoc_tmp['1to1'][np.isin(assoc_tmp['assoc_ID'],list(isolated_assoc))] = True assoc_tmp['Nnebulae'] = [len(associations_dict[k]) for k in assoc_tmp['assoc_ID']] assoc_tmp['region_ID'] = np.nan assoc_tmp['region_ID'][assoc_tmp['1to1']] = [associations_dict[k][0] for k in assoc_tmp[assoc_tmp['1to1']]['assoc_ID']] overlap = join( Table(np.unique(associations_mask.data[~np.isnan(associations_mask.data)],return_counts=True),names=['assoc_ID','size']), Table(np.unique(associations_mask.data[~np.isnan(nebulae_hst) & ~np.isnan(associations_mask.data)],return_counts=True),names=['assoc_ID','overlap_size']), keys=['assoc_ID'],join_type='outer') overlap = overlap.filled(0) overlap['overlap_asc'] = overlap['overlap_size']/overlap['size'] overlap['overlap_asc'].info.format = '%.2f' assoc_tmp = join(assoc_tmp,overlap[['assoc_ID','overlap_asc']],keys='assoc_ID') #print('write to file') hdu = fits.BinTableHDU(assoc_tmp,name='joined catalogue') hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations.fits',overwrite=True) nebulae_tmp = nebulae[['region_ID','x','y']].copy() nebulae_tmp.add_index('region_ID') nebulae_tmp['env_neb'] = [environment_dict[env_masks_neb.data[y,x]] for x,y in zip(nebulae_tmp['x'].astype(int),nebulae_tmp['y'].astype(int))] nebulae_tmp['neighbors'] = np.nan for row in nebulae_tmp: row['neighbors'] = len(find_neighbors(nebulae_mask.data,tuple(row[['x','y']]),row['region_ID'],plot=False)) del nebulae_tmp[['x','y']] nebulae_tmp['1to1'] = False nebulae_tmp['1to1'][np.isin(nebulae_tmp['region_ID'],list(isolated_nebulae))] = True nebulae_tmp['Nassoc'] = [len(nebulae_dict[k]) for k in nebulae_tmp['region_ID']] nebulae_tmp['assoc_ID'] = np.nan nebulae_tmp['assoc_ID'][nebulae_tmp['1to1']] = [nebulae_dict[k][0] for k in nebulae_tmp[nebulae_tmp['1to1']]['region_ID']] overlap = join( Table(np.unique(nebulae_hst[~np.isnan(nebulae_hst)],return_counts=True),names=['region_ID','size']), Table(np.unique(nebulae_hst[~np.isnan(nebulae_hst) & ~np.isnan(associations_mask.data)],return_counts=True),names=['region_ID','overlap_size']), keys=['region_ID'],join_type='outer') overlap = overlap.filled(0) overlap['overlap_neb'] = overlap['overlap_size']/overlap['size'] overlap['overlap_neb'].info.format = '%.2f' nebulae_tmp = join(nebulae_tmp,overlap[['region_ID','overlap_neb']],keys='region_ID') hdu = fits.BinTableHDU(nebulae_tmp,name='joined catalogue') hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_nebulae.fits',overwrite=True) #del nebulae_tmp['1to1'] #print(f'{np.sum(nebulae_tmp["neighbors"]==0)} nebulae have no neighbors') catalogue = join(assoc_tmp,nebulae_tmp,keys=['assoc_ID','region_ID']) catalogue = join(catalogue,nebulae,keys='region_ID') catalogue = join(catalogue,associations,keys='assoc_ID') # pay attention to the order of assoc, neb catalogue.rename_columns(['X','Y','x','y','RA','DEC','cen_ra','cen_dec', 'reg_area','region_area', 'EBV_1','EBV_2','EBV_err','EBV_ERR', 'SkyCoord_1','SkyCoord_2'], ['x_asc','y_asc','x_neb','y_neb','ra_asc','dec_asc','ra_neb','dec_neb', 'area_asc','area_neb', 'EBV_balmer','EBV_stars','EBV_balmer_err','EBV_stars_err', 'SkyCoord_asc','SkyCoord_neb']) # separation to other associations and nebulae idx,sep_asc,_= match_coordinates_sky(catalogue['SkyCoord_asc'],associations['SkyCoord'],nthneighbor=2) idx,sep_neb,_= match_coordinates_sky(catalogue['SkyCoord_neb'],nebulae['SkyCoord'],nthneighbor=2) catalogue['sep_asc'] = sep_asc.to(u.arcsec) catalogue['sep_neb'] = sep_neb.to(u.arcsec) # select the columns of the joined catalogue columns = ['assoc_ID','region_ID','x_asc','y_asc','x_neb','y_neb', 'ra_asc','dec_asc','ra_neb','dec_neb','SkyCoord_asc','SkyCoord_neb', 'env_asc','env_neb','area_asc','area_neb', 'sep_asc','sep_neb','neighbors','Nassoc','overlap','overlap_asc','overlap_neb', 'age','age_err','mass','mass_err','EBV_stars','EBV_stars_err','EBV_balmer','EBV_balmer_err', 'met_scal','met_scal_err','logq_D91','logq_D91_err',] + \ [x for x in nebulae.columns if x.endswith('_FLUX_CORR')] + \ [x for x in nebulae.columns if x.endswith('_FLUX_CORR_ERR')] + \ ['NUV_FLUX','NUV_FLUX_ERR','U_FLUX','U_FLUX_ERR','B_FLUX','B_FLUX_ERR', 'V_FLUX','V_FLUX_ERR','I_FLUX','I_FLUX_ERR'] catalogue = catalogue[columns] catalogue.rename_columns([col for col in catalogue.columns if col.endswith('FLUX_CORR')], [col.replace('FLUX_CORR','flux') for col in catalogue.columns if col.endswith('FLUX_CORR')]) catalogue.rename_columns([col for col in catalogue.columns if col.endswith('FLUX_CORR_ERR')], [col.replace('FLUX_CORR_ERR','flux_err') for col in catalogue.columns if col.endswith('FLUX_CORR_ERR')]) catalogue['assoc_ID'] = catalogue['assoc_ID'].astype('int') catalogue['region_ID'] = catalogue['region_ID'].astype('int') catalogue.info.description = 'Joined catalogue between associations and nebulae' mean_sep = np.mean(catalogue['SkyCoord_asc'].separation(catalogue['SkyCoord_neb'])) #print(f'{len(catalogue)} objects in catalogue') #print(f'the mean separation between cluster and association center is {mean_sep.to(u.arcsecond):.2f}') export = catalogue.copy() #[catalogue['contained']] #export.add_column(export['SkyCoord_asc'].to_string(style='hmsdms',precision=2),index=6,name='RaDec_asc') #export.add_column(export['SkyCoord_neb'].to_string(style='hmsdms',precision=2),index=8,name='RaDec_neb') RA_asc ,DEC_asc = zip(*[x.split(' ') for x in export['SkyCoord_asc'].to_string(style='hmsdms',precision=2)]) RA_neb ,DEC_neb = zip(*[x.split(' ') for x in export['SkyCoord_neb'].to_string(style='hmsdms',precision=2)]) export.add_column(RA_asc,index=6,name='Ra_asc') export.add_column(DEC_asc,index=8,name='Dec_asc') export.add_column(RA_neb,index=10,name='Ra_neb') export.add_column(DEC_neb,index=12,name='Dec_neb') for col in export.columns: if col not in ['Ra_asc','Dec_asc','Ra_neb','Dec_neb','region_ID','cluster_ID','overlap','env_asc','env_neb']: export[col].info.format = '%.2f' del export[['ra_asc','dec_asc','ra_neb','dec_neb','SkyCoord_neb','SkyCoord_asc']] hdu = fits.BinTableHDU(export,name='joined catalogue') hdu.writeto(basedir/version/HSTband/f'{scalepc}pc'/f'{name}_{HSTband}_{scalepc}pc_associations_and_nebulae_joined.fits',overwrite=True)
def calibrate_all(self, use_calon=False): """Calibrate the scan in all available ways. The basic calibration is `(on - off)/off`, where `on` and `off` are on-source and off-source spectra respectively. New HDU lists are produced and added to the existing, uncalibrated ones. If the calibration mark has been used in some scan, an additional calibration is applied and further HDU lists are produced. Other Parameters ---------------- use_calon : bool, default False If False, only the OFF + CAL is used for the calibration. If True, Also the ON + CAL is used and the calibration constant is averaged with that obtained through OFF + CAL. """ new_tables = {} for caltype in ["cal", "onoff"]: for (filekey, hdul) in self.tables.items(): new_filekey = filekey.replace("_all", "_" + caltype) new_hdul = copy.deepcopy(hdul) table = Table(new_hdul[1].data) table.sort(['MJD', 'TELESCOP', 'LINE']) out_grouped = table.group_by(['TELESCOP', 'LINE']) new_rows = 0 astropy_table_from_results = None apply_cal = caltype == "cal" for _, out_group in zip(out_grouped.groups.keys, out_grouped.groups): out_group = find_cycles(out_group, ['SIGNAL', 'CAL_IS_ON']) grouped = out_group.group_by(['CYCLE']) for _, group in zip(grouped.groups.keys, grouped.groups): # group = vstack([group, group]) results, _ = normalize_on_off_cal(group, smooth=False, apply_cal=apply_cal, use_calon=use_calon) if results is None: break if astropy_table_from_results is None: astropy_table_from_results = results else: astropy_table_from_results = \ vstack((astropy_table_from_results, results)) new_rows += 1 if astropy_table_from_results is None: continue astropy_table_from_results.remove_column('CYCLE') astropy_table_from_results.remove_column('SIGNAL') astropy_table_from_results.remove_column('CAL_IS_ON') astropy_table_from_results.remove_column('BINARY_COL') astropy_table_from_results.remove_column('MJD') dummy_hdu = \ fits.BinTableHDU(data=astropy_table_from_results) new_hdul[1].data = dummy_hdu.data new_tables[new_filekey] = new_hdul self.tables.update(new_tables)
def to_hdulist(self): """Convert to `~astropy.io.fits.HDUList`""" hdu = fits.BinTableHDU(self.to_table()) ebounds = energy_axis_to_ebounds(self[0].energy.bins) return fits.HDUList([fits.PrimaryHDU(), hdu, ebounds])
def write_image(outfile, image, meta=None): """Writes image object to outfile Args: outfile : output file string image : desispec.image.Image object (or any object with 2D array attributes image, ivar, mask) Optional: meta : dict-like object with metadata key/values (e.g. FITS header) """ log = get_logger() if meta is not None: hdr = fitsheader(meta) else: hdr = fitsheader(image.meta) add_dependencies(hdr) #- Work around fitsio>1.0 writing blank keywords, e.g. on 20191212 for key in hdr.keys(): if type(hdr[key]) == fits.card.Undefined: log.warning('Setting blank keyword {} to None'.format(key)) hdr[key] = None outdir = os.path.dirname(os.path.abspath(outfile)) if not os.path.isdir(outdir): os.makedirs(outdir) hx = fits.HDUList() hdu = fits.ImageHDU(image.pix.astype(np.float32), name='IMAGE', header=hdr) if 'CAMERA' not in hdu.header: hdu.header.append( ('CAMERA', image.camera.lower(), 'Spectrograph Camera')) if 'RDNOISE' not in hdu.header and np.isscalar(image.readnoise): hdu.header.append( ('RDNOISE', image.readnoise, 'Read noise [RMS electrons/pixel]')) hx.append(hdu) hx.append(fits.ImageHDU(image.ivar.astype(np.float32), name='IVAR')) hx.append(fits.CompImageHDU(image.mask.astype(np.int16), name='MASK')) if not np.isscalar(image.readnoise): hx.append( fits.ImageHDU(image.readnoise.astype(np.float32), name='READNOISE')) if hasattr(image, 'fibermap'): if isinstance(image.fibermap, Table): fmhdu = fits.convenience.table_to_hdu(image.fibermap) fmhdu.name = 'FIBERMAP' else: fmhdu = fits.BinTableHDU(image.fibermap, name='FIBERMAP') hx.append(fmhdu) hx.writeto(outfile + '.tmp', overwrite=True, checksum=True) os.rename(outfile + '.tmp', outfile) return outfile
#------header----------------- hdr = fits.Header() hdr['lambda'] = 'Vacuum Wavelength in Ang' hdr['minwav'] = 'Lower complex fitting wavelength range' hdr['maxwav'] = 'Upper complex fitting wavelength range' hdr['ngauss'] = 'Number of Gaussians for the line' hdr['inisig'] = 'Initial guess of linesigma [in lnlambda]' hdr['minsig'] = 'Lower range of line sigma [lnlambda]' hdr['maxsig'] = 'Upper range of line sigma [lnlambda]' hdr['voff '] = 'Limits on velocity offset from the central wavelength [lnlambda]' hdr['vindex'] = 'Entries w/ same NONZERO vindex constrained to have same velocity' hdr['windex'] = 'Entries w/ same NONZERO windex constrained to have same width' hdr['findex'] = 'Entries w/ same NONZERO findex have constrained flux ratios' hdr['fvalue'] = 'Relative scale factor for entries w/ same findex' #------save line info----------- hdu = fits.BinTableHDU(data=newdata, header=hdr, name='data') hdu.writeto(path + 'qsopar.fits', overwrite=True) # ============================================================================= # ============================================================================= # 4. Begin the fitting process and append the result to DataFrame # ============================================================================= # whether you want to reuse the computed output or not read_from_file = False z = 6.345 # here we do teh Monte-Carlo simulation to propagate the flux error for j in range(1): if read_from_file == True: result = pd.read_csv(path + 'result/result_temp.csv')
def make_cubes(indir, outdir, redo=False, bands=None, bscale=1e-19): """ Get results from cutouts and join them in a cube. """ filenames = os.listdir(indir) galaxy = os.path.split(indir)[1] fields = set([_.split("_")[-4] for _ in filenames]) sizes = set([_.split("_")[-2] for _ in filenames]) bands = context.bands if bands is None else bands wave = np.array([context.wave_eff[band] for band in bands]) * u.Angstrom flam_unit = u.erg / u.cm / u.cm / u.s / u.AA fnu_unit = u.erg / u.s / u.cm / u.cm / u.Hz ext = {"swp": "DATA", "swpweight": "WEIGHTS"} hfields = ["GAIN", "PSFFWHM", "DATE-OBS"] for field, size in itertools.product(fields, sizes): cubename = os.path.join(outdir, "{}_{}_{}.fits".format(galaxy, field, size)) if os.path.exists(cubename) and not redo: continue # Loading and checking images imgs = [os.path.join(indir, "{}_{}_{}_{}_swp.fits".format(galaxy, field, band, size)) for band in bands] if not all([os.path.exists(_) for _ in imgs]): continue # Checking if images have calibration available headers = [fits.getheader(img, ext=1) for img in imgs] if not all(["MAGZP" in h for h in headers]): continue # Checking if weight images are available wimgs = [os.path.join(indir, "{}_{}_{}_{}_swpweight.fits".format( galaxy, field, band, size)) for band in bands] has_errs = all([os.path.exists(_) for _ in wimgs]) # Making new header with WCS h = headers[0].copy() del h["FILTER"] del h["MAGZP"] w = WCS(h) nw = WCS(naxis=3) nw.wcs.cdelt[:2] = w.wcs.cdelt nw.wcs.crval[:2] = w.wcs.crval nw.wcs.crpix[:2] = w.wcs.crpix nw.wcs.ctype[0] = w.wcs.ctype[0] nw.wcs.ctype[1] = w.wcs.ctype[1] try: nw.wcs.pc[:2, :2] = w.wcs.pc except: pass h.update(nw.to_header()) # Performin calibration m0 = np.array([h["MAGZP"] for h in headers]) gain = np.array([h["GAIN"] for h in headers]) f0 = np.power(10, -0.4 * (48.6 + m0)) data = np.array([fits.getdata(img, 1) for img in imgs]) fnu = data * f0[:, None, None] * fnu_unit flam = fnu * const.c / wave[:, None, None]**2 flam = flam.to(flam_unit).value / bscale if has_errs: weights = np.array([fits.getdata(img, 1) for img in wimgs]) dataerr = 1 / weights + np.clip(data, 0, np.infty) / gain[:, None, None] fnuerr= dataerr * f0[:, None, None] * fnu_unit flamerr = fnuerr * const.c / wave[:, None, None] ** 2 flamerr = flamerr.to(flam_unit).value / bscale # Making table with metadata tab = [] tab.append(bands) tab.append([context.wave_eff[band] for band in bands]) tab.append([context.exptimes[band] for band in bands]) names = ["FILTER", "WAVE_EFF", "EXPTIME"] for f in hfields: if not all([f in h for h in headers]): continue tab.append([h[f] for h in headers]) names.append(f) tab = Table(tab, names=names) # Producing data cubes HDUs. hdus = [fits.PrimaryHDU()] hdu1 = fits.ImageHDU(flam, h) hdu1.header["EXTNAME"] = ("DATA", "Name of the extension") hdus.append(hdu1) if has_errs: hdu2 = fits.ImageHDU(flamerr, h) hdu2.header["EXTNAME"] = ("ERRORS", "Name of the extension") hdus.append(hdu2) for hdu in hdus: hdu.header["BSCALE"] = (bscale, "Linear factor in scaling equation") hdu.header["BZERO"] = (0, "Zero point in scaling equation") hdu.header["BUNIT"] = ("{}".format(flam_unit), "Physical units of the array values") thdu = fits.BinTableHDU(tab) hdus.append(thdu) thdu.header["EXTNAME"] = "METADATA" hdulist = fits.HDUList(hdus) hdulist.writeto(cubename, overwrite=True)
def writexipbias(samples,rhosfilenames,xim=False, plots=False,nameterms='terms_dxi.png',namedxip='dxi.png',namecovmat='covmat_pars.png',filename='dxip.fits'): from readjson import read_rhos from maxlikelihood import bestparameters from plot_stats import pretty_rho from readfits import read_corr from astropy.io import fits import numpy as np #plot covariance matrix of parameters alpha, beta and eta. if plots: par_matcov = np.cov(samples) corr=corrmatrix(par_matcov) print(par_matcov) print(corr) cov_vmin=np.min(corr) plt.imshow(corr,cmap='viridis'+'_r', interpolation='nearest', aspect='auto', origin='lower', vmin=cov_vmin, vmax=1.) plt.colorbar() plt.title(r'$\alpha \mid \beta \mid \eta $') plt.savefig(namecovmat, dpi=500) print(namecovmat, 'Printed!') a = b = n = 0; vara = varb = varn = 0; covab = covan = covbn = 0 bestpar = bestparameters(samples) par_matcov = np.cov(samples) if (par_matcov.size==1 ): variances = par_matcov else: variances = np.diagonal(par_matcov) covariances = sum( (par_matcov[i,i+1: ].tolist() for i in range(len(samples) - 1)) , [] ) if(len(samples)==3): a, b, n = bestpar vara, varb, varn = variances covab, covan, covbn = covariances elif(len(samples)==2): a, b = bestpar vara, varb = variances covab = covariances[0] elif(len(samples)==1): a = bestpar[0] vara = variances else: print("Warning, test type not defined") rhonames = args.rhos meanr, rho0, cov_rho0 = read_corr(rhonames[0]) meanr, rho1, cov_rho1 = read_corr(rhonames[1]) meanr, rho2, cov_rho2 = read_corr(rhonames[2]) meanr, rho3, cov_rho3 = read_corr(rhonames[3]) meanr, rho4, cov_rho4 = read_corr(rhonames[4]) meanr, rho5, cov_rho5 = read_corr(rhonames[5]) sig_rho0 = np.sqrt(np.diag(cov_rho0)) sig_rho1 = np.sqrt(np.diag(cov_rho1)) sig_rho2 = np.sqrt(np.diag(cov_rho2)) sig_rho3 = np.sqrt(np.diag(cov_rho3)) sig_rho4 = np.sqrt(np.diag(cov_rho4)) sig_rho5 = np.sqrt(np.diag(cov_rho5)) #Ploting each term of the bias if(plots): xlim = [2., 300.] #supposing that a,b and n are idependent of rhos(scale independent) var0 = ((2*a*rho0p)**2)*vara + (a**2)*(sig_rho0**2) var1 = ((2*b*rho1p)**2)*varb + (b**2)*(sig_rho1**2) var2 = ((2*n*rho3p)**2)*varn + (n**2)*(sig_rho3**2) varab = vara*(b**2) + varb*(a**2) + 2*covab*(a*b) #varab = ((a*b)**2)*( (vara/((a)**2)) + (varb/((b)**2)) + 2*covab/(a*b) ) var3 = 4*( (rho2p**2)*varab + (sig_rho2**2)*((a*b)**2) ) #var3 = 4*((a*b*rho2p)**2)*( varab/((a*b)**2) + (sig_rho2/rho2p)**2 ) varbn = varn*(b**2) + varb*(n**2) + 2*covbn*(b*n) #varbn = ((n*b)**2)*( (varn/((n)**2)) + (varb/((b)**2)) + 2*covbn/(b*n) ) var4 = 4*( (rho4p**2)*varbn + (sig_rho4**2)*((n*b)**2) ) #var4 = 4*((n*b*rho4p)**2)*(varbn/((b*n)**2) + (sig_rho4/rho4p)**2) varan = varn*(a**2) + vara*(n**2) + 2*covan*(a*n) #varan = ((n*a)**2)*( (varn/((n)**2)) + (vara/((a)**2)) + 2*covan/(a*n) ) var5 = 4*( (rho5p**2)*varan + (sig_rho5**2)*((n*a)**2) ) #var5 = 4*((n*a*rho5p)**2)*(varan/((a*n)**2) + (sig_rho5/rho5p)**2) plt.clf() lfontsize = 7 if (len(samples)==3): pretty_rho(meanr, (a**2)*rho0p, np.sqrt(np.diag(cov_rho0)), legend=r'$\alpha^{2} \rho_{0}$',lfontsize=lfontsize, color='red', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (b**2)*rho1p, np.sqrt(var1), legend=r'$\beta^{2}\rho_{1}$',lfontsize=lfontsize, color='green', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (n**2)*rho3p, np.sqrt(var2), legend=r'$\eta^{2}\rho_{3}$', lfontsize=lfontsize, color='black', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (2*a*b)*rho2p, np.sqrt(var3), legend=r'$2\alpha\beta \rho_{2}$',lfontsize=lfontsize, color='yellow', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (2*b*n)*rho4p, np.sqrt(var4), legend=r'$2\beta\eta\rho_{4}$',lfontsize=lfontsize, color='blue', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (2*n*a)*rho5p, np.sqrt(var5), legend=r'$2\eta\alpha\rho_{5}$', lfontsize=lfontsize, color='gray', ylabel='Correlations', xlim=xlim) print('Printing', nameterms) plt.savefig(nameterms, dpi=200) if (len(samples)==2): pretty_rho(meanr, (a**2)*rho0p, np.sqrt(var0), legend=r'$\alpha^{2} \rho_{0}$',lfontsize=lfontsize, color='red', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (b**2)*rho1p, np.sqrt(var1), legend=r'$\beta^{2}\rho_{1}$',lfontsize=lfontsize, color='green', ylabel='Correlations', xlim=xlim) pretty_rho(meanr, (2*a*b)*rho2p, np.sqrt(var3), legend=r'$2\alpha\beta \rho_{2}$',lfontsize=lfontsize, color='yellow', ylabel='Correlations', xlim=xlim) print('Printing', nameterms) plt.savefig(nameterms, dpi=200) if (len(samples)==1): pretty_rho(meanr, (a**2)*rho0p, np.sqrt(var0), legend=r'$\alpha^{2} \rho_{0}$',lfontsize=lfontsize, color='red', ylabel='Correlations', xlim=xlim) print('Printing', nameterms) plt.savefig(nameterms, dpi=200) #supposing that a,b and n are idependent of rhos(scale independent) dxip = (a**2)*rho0p + (b**2)*rho1p + (n**2)*rho3p + (2*a*b)*rho2p + (2*b*n)*rho4p + (2*n*a)*rho5p f1 = 2*(a*rho0p + b*rho2p + n*rho5p) f2 = 2*(b*rho1p + a*rho2p + n*rho4p) f3 = 2*(n*rho3p + b*rho4p + a*rho5p) f4 = a**2 ; f5 = b**2; f6 = 2*a*b f7 = n**2 ; f8 = 2*b*n; f9 = 2*n*a covmat_dxip = np.diag( (f1**2)*vara + (f2**2)*varb + (f3**2)*varn + + 2*(f1*f2*covab + f1*f3*covan + f2*f3*covbn) ) \ + (f4**2)*(cov_rho0) + (f5**2)*(cov_rho1) + (f6**2)*(cov_rho2) + (f7**2)*(cov_rho3) +(f8**2)*(cov_rho4) + (f9**2)*(cov_rho5) if(plots): plt.clf() pretty_rho(meanr, dxip, np.sqrt(np.diag(covmat_dxip)) , legend=r"$\delta \xi_{+}$", ylabel=r"$\delta \xi_{+}$", xlim=xlim) print('Printing', dxipname) plt.savefig(dxipname, dpi=150) nrows = len(dxip) hdu = fits.PrimaryHDU() hdul = fits.HDUList([hdu]) covmathdu = fits.ImageHDU(covmat_dxip, name='COVMAT') hdul.insert(1, covmathdu) angarray = meanr valuearray = np.array(dxip) bin1array = np.array([ -999]*nrows) bin2array = np.array([ -999]*nrows) angbinarray = np.arange(nrows) array_list = [bin1array, bin2array, angbinarray, valuearray, angarray ] for array, name in zip(array_list, names): outdata[name] = array corrhdu = fits.BinTableHDU(outdata, name=nam) hdul.insert(2, corrhdu) if xim: hdul.writeto(filename + 'm.fits', clobber=True) else: hdul.writeto(filename + 'p.fits', clobber=True)
def filter_evt_file(fits, outfile='./cleaned_file.fits', obsid = ''): """ fiter the events in the fits file by dtf and status input: fits --- fits file output: fits --- filtered fits file """ # #--- get obsid # if obsid == '': atemp = re.split('hrcf', fits) btemp = re.split('_', atemp[1]) obsid = btemp[0] # #--- copy the fits file and make sure that it is not zipped # mc = re.search('gz', fits) if mc is not None: cmd = 'cp ' + fits + ' ztemp1.fits.gz' os.system(cmd) cmd = 'gzip -d ztemp1.fits.gz' os.system(cmd) else: cmd = 'cp ' + fits + ' ztemp1.fits' os.system(cmd) # #--- filter out dtf <= 0.98 # [pstart, pstop] = get_dead_period(obsid) t = pyfits.open('ztemp1.fits') tdata = t[1].data t_list = tdata['time'] t.close() # #--- if there are clipped parts based of dtf, create mask for that # clen = len(pstart) if clen > 0: m = 0 mask = [] cend = 0 for t in t_list: if cend == 0: for k in range(m, clen): if t < pstart[k]: mask.append(True) break elif (t >= pstart[k]) and (t <= pstop[k]): mask.append(False) break elif t > pstop[k]: if k == clen -1: cend = 1 break m = k continue else: mask.append(True) mask.append(False) mask = numpy.array(mask) newtdata = tdata[mask] # #--- remove the masked part and create a new fits file # hdu = pyfits.BinTableHDU(data=newtdata) hdu.writeto('ztemp2.fits') cmd = 'mv -f ztemp2.fits ztemp1.fits' os.system(cmd) # #--- filter by status # filter_by_status('ztemp1.fits') cmd = 'mv -f ztemp1.fits ' + outfile os.system(cmd)
1., 1.82574186, 3.16227766, 5.77350269, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ])), ('Theta_inc', 0, 18.200001, 5.0, 0.0, 18.200001, 87.099998, 90.0, 3, numpy.array([ 0, 60, 90, 0.0, 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ])), ], dtype=dtype) assert numpy.product(parameters['NUMBVALS']) == len(table), ( 'parameter definition does not match spectra table', parameters['NUMBVALS'], numpy.product(parameters['NUMBVALS']), len(table)) hdu = pyfits.BinTableHDU(data=parameters) hdu.header['DATE'] = nowstr hdu.header['EXTNAME'] = 'PARAMETERS' hdu.header['HDUCLAS1'] = 'XSPEC TABLE MODEL' hdu.header['HDUVERS1'] = '1.0.0' hdu.header['NINTPARM'] = len(parameters) hdu.header['NADDPARM'] = 0 hdu.header['HDUCLAS2'] = 'PARAMETERS' hdus.append(hdu) # ENERG_LO, ENERG_HI dtype = [('ENERG_LO', '>f4'), ('ENERG_HI', '>f4')] energies = numpy.array(list(zip(energy_lo, energy_hi)), dtype=dtype) hdu = pyfits.BinTableHDU(data=energies) hdu.header['DATE'] = nowstr hdu.header['EXTNAME'] = 'ENERGIES'
"RA_SCZ"), f['SC_DATA'].data.field("DEC_SCZ") ang_dis = getAngularDistance(args.ra, args.dec, ra_scz, dec_scz) # Find the time where the distance was at its minimum (keeping some margin to later select the data) min_idx = ang_dis[:-n_entries].argmin() time_of_min_distance = f['SC_DATA'].data.field("START")[min_idx] log.info("At MET = %.3f the distance between the pointing and the " "input position was %.3f deg" % (time_of_min_distance, ang_dis[min_idx])) # Now create a new FT2 file with the pointing fixed at the desired position new_sc_table = pyfits.BinTableHDU(f['SC_DATA'].data[min_idx:min_idx + n_entries]) new_sc_table.data.RA_SCZ[:] = new_sc_table.data.RA_SCZ[0] new_sc_table.data.DEC_SCZ[:] = new_sc_table.data.DEC_SCZ[0] new_sc_table.data.RA_SCX[:] = new_sc_table.data.RA_SCX[0] new_sc_table.data.DEC_SCX[:] = new_sc_table.data.DEC_SCX[0] new_sc_table.data.RA_ZENITH[:] = new_sc_table.data.RA_ZENITH[0] new_sc_table.data.DEC_ZENITH[:] = new_sc_table.data.DEC_ZENITH[0] new_sc_table.data.RA_NPOLE[:] = new_sc_table.data.RA_NPOLE[0] new_sc_table.data.DEC_NPOLE[:] = new_sc_table.data.DEC_NPOLE[0] new_sc_table.data.ROCK_ANGLE[:] = new_sc_table.data.ROCK_ANGLE[0] new_sc_table.data.START = np.arange(args.met_start, stop_time, dt)
def to_hdulist(self, name=None): """Convert to `~astropy.io.fits.HDUList`.""" return fits.HDUList( [fits.PrimaryHDU(), fits.BinTableHDU(self.to_table(), name=name)] )
def _process(self, prod, sci_catalog, astrometry=True): if prod.calibed_files is None: raise ValueError("Product failed: No calibrated images. raw files:" " {}".format(prod.files.files)) fm = FileManager(self.image_ext) if check_iterable(prod.calibed_files): if len(prod.calibed_files) > 1: fg = fm.create_filegroup(files=prod.calibed_files, ext=self.image_ext) stacked = None for i in fg.hdus(): if stacked is None: stacked = i else: s = hdu_shift_images([stacked, i], method=self._align_method)[1] stacked = imarith(stacked, s, '+') elif len(prod.calibed_files) == 1: stacked = prod.calibed_files[0] fg = fm.create_filegroup(files=prod.calibed_files, ext=self.image_ext) stacked = check_hdu(stacked, ext=self.image_ext) else: raise ValueError("Product failed: No calibrated images." "raw files: {}".format(prod.files.files)) return else: stacked = prod.calibed_files fg = fm.create_filegroup(files=prod.calibed_files, ext=self.image_ext) stacked = check_hdu(stacked, ext=self.image_ext) filt = self.get_filter(fg) cat = self.select_catalog(filt) wcs = WCS(stacked.header, relax=True) if '' in wcs.wcs.ctype or astrometry: wcs = None plate_scale = self.get_platescale(stacked) phot, wcs = process_calib_photometry(stacked, science_catalog=sci_catalog, identify_catalog=cat, filter=filt, wcs=wcs, return_wcs=True, plate_scale=plate_scale, **self.photometry_parameters, **self.astrometry_parameters) apertures = np.unique(phot['aperture']) if len(apertures) > 1: selected_aperture = 0 snr = 0 for g in phot.group_by('aperture').groups: g_snr = np.sum(g['flux'] / g['flux_error']) if g_snr > snr: selected_aperture = g['aperture'][0] snr = g_snr phot = phot[phot['aperture'] == selected_aperture] phot = phot.as_array() phot.meta['astropop n_images'] = len(prod.calibed_files) phot.meta['astropop night'] = prod.files.values('night', unique=True)[0] phot.meta['astropop filter'] = filt stacked.header['astropop n_images'] = len(prod.calibed_files) stacked.header['astropop night'] = prod.files.values('night', unique=True)[0] stacked.header['astropop filter'] = filt header = stacked.header if wcs is not None: header.update(wcs.to_header(relax=True)) imhdu = fits.PrimaryHDU(stacked.data, header=header) tbhdu = fits.BinTableHDU(phot, name='photometry') if prod.sci_result is None: filename = self.get_filename(fg) else: filename = prod.sci_result mkdir_p(os.path.dirname(filename)) fits.HDUList([imhdu, tbhdu]).writeto(filename)
def to_fits(self): header = self._prepare_fits_header(self.header) table = Table.from_pandas(self.data) hdu = fits.BinTableHDU(table, header) return hdu
def grab_sdss_spectra(radec, radius=0.1 * u.deg, outfil=None, debug=False, maxsep=None, timeout=600., zmin=None): """ Grab SDSS spectra Parameters ---------- radec : tuple RA, DEC in deg radius : float, optional (0.1*u.deg) Search radius -- Astroquery actually makes a box, not a circle timeout : float, optional Timeout limit for connection with SDSS outfil : str ('tmp.fits') Name of output file for FITS table maxsep : float (None) :: Mpc Maximum separation to include zmin : float (None) Minimum redshift to include Returns ------- tbl : Table """ cC = coords.SkyCoord(ra=radec[0], dec=radec[1]) # Query photoobj_fs = ['ra', 'dec', 'objid', 'run', 'rerun', 'camcol', 'field'] mags = [ 'petroMag_u', 'petroMag_g', 'petroMag_r', 'petroMag_i', 'petroMag_z' ] magsErr = [ 'petroMagErr_u', 'petroMagErr_g', 'petroMagErr_r', 'petroMagErr_i', 'petroMagErr_z' ] phot_catalog = SDSS.query_region(cC, spectro=True, radius=radius, timeout=timeout, photoobj_fields=photoobj_fs + mags + magsErr) # Unique spec_catalog = SDSS.query_region(cC, spectro=True, radius=radius, timeout=timeout) # Duplicates exist nobj = len(phot_catalog) # print('grab_sdss_spectra: Found {:d} sources in the search box.'.format( nobj)) # Coordinates cgal = SkyCoord(ra=phot_catalog['ra'] * u.degree, dec=phot_catalog['dec'] * u.degree) sgal = SkyCoord(ra=spec_catalog['ra'] * u.degree, dec=spec_catalog['dec'] * u.degree) sepgal = cgal.separation(cC) #in degrees # Check for problems and parse z zobj = np.zeros(nobj) idx, d2d, d3d = coords.match_coordinates_sky(cgal, sgal, nthneighbor=1) if np.max(d2d) > 1. * u.arcsec: print('No spectral match!') xdb.set_trace() else: zobj = spec_catalog['z'][idx] idx, d2d, d3d = coords.match_coordinates_sky(cgal, cgal, nthneighbor=2) if np.min(d2d.to('arcsec')) < 1. * u.arcsec: print('Two photometric sources with same RA/DEC') xdb.set_trace() #xdb.set_trace() # Cut on Separation if not maxsep is None: print('grab_sdss_spectra: Restricting to {:g} Mpc separation.'.format( maxsep)) sepgal_kpc = cosmo.kpc_comoving_per_arcmin(zobj) * sepgal.to('arcmin') sepgal_mpc = sepgal_kpc.to('Mpc') gdg = np.where(sepgal_mpc < (maxsep * u.Unit('Mpc')))[0] phot_catalog = phot_catalog[gdg] #xdb.set_trace() nobj = len(phot_catalog) print('grab_sdss_spectra: Grabbing data for {:d} sources.'.format(nobj)) # Grab Spectra from SDSS # Generate output table attribs = galaxy_attrib() npix = 5000 #len( spec_hdus[0][1].data.flux ) spec_attrib = [(str('FLUX'), np.float32, (npix, )), (str('SIG'), np.float32, (npix, )), (str('WAVE'), np.float64, (npix, ))] tbl = np.recarray((nobj, ), dtype=attribs + spec_attrib) tbl['RA'] = phot_catalog['ra'] tbl['DEC'] = phot_catalog['dec'] tbl['TELESCOPE'] = str('SDSS 2.5-M') # Deal with spectra separately (for now) npix = 5000 #len( spec_hdus[0][1].data.flux ) for idx, obj in enumerate(phot_catalog): #print('idx = {:d}'.format(idx)) # Grab spectra (there may be duplicates) mt = np.where( sgal.separation(cgal[idx]).to('arcsec') < 1. * u.Unit('arcsec'))[0] if len(mt) > 1: # Use BOSS if you have it mmt = np.where(spec_catalog[mt]['instrument'] == 'BOSS')[0] if len(mmt) > 0: mt = mt[mmt[0]] else: mt = mt[0] elif len(mt) == 0: xdb.set_trace() else: mt = mt[0] # Grab spectra spec_hdus = SDSS.get_spectra(matches=Table(spec_catalog[mt])) tbl[idx]['INSTRUMENT'] = spec_catalog[mt]['instrument'] spec = spec_hdus[0][1].data npp = len(spec.flux) tbl[idx]['FLUX'][0:npp] = spec.flux sig = np.zeros(npp) gdi = np.where(spec.ivar > 0.)[0] if len(gdi) > 0: sig[gdi] = np.sqrt(1. / spec.ivar[gdi]) tbl[idx]['SIG'][0:npp] = sig tbl[idx]['WAVE'][0:npp] = 10.**spec.loglam # Redshifts meta = spec_hdus[0][2].data for attrib in ['Z', 'Z_ERR']: tbl[idx][attrib] = meta[attrib] if debug: sep_to_qso = cgal[idx].separation(cC).to('arcmin') print('z = {:g}, Separation = {:g}'.format(tbl[idx].Z, sep_to_qso)) xdb.set_trace() # Fill in rest tbl[idx].SDSS_MAG = np.array([obj[phot] for phot in mags]) tbl[idx].SDSS_MAGERR = np.array([obj[phot] for phot in magsErr]) # Clip on redshift to excise stars/quasars if zmin is not None: gd = np.where(tbl['Z'] > zmin)[0] tbl = tbl[gd] # Write to FITS file if outfil is not None: prihdr = fits.Header() prihdr['COMMENT'] = 'SDSS Spectra' prihdu = fits.PrimaryHDU(header=prihdr) tbhdu = fits.BinTableHDU(tbl) thdulist = fits.HDUList([prihdu, tbhdu]) thdulist.writeto(outfil, clobber=True) print('Wrote SDSS table to {:s}'.format(outfil)) return tbl
def mk_ccf_mask(template, doplot=False): # CSV table to contain systemic velocities. Will replace the entry of the same object if it is already present # in the table, will create the table if it does not exist systemic_velocity_table = 'systemic_velo.csv' print(template) # Path where models are saved path_to_models = 'HiResFITS' # some parameters, don't worry dv = 0.00 # km/s -- width of the CCF box c = (constants.c / 1000) # create directory if needed if not os.path.isdir(path_to_models): os.system('mkdir {0}'.format(path_to_models)) # read wavelength and flux. The wavelength is expressed in Ang, we convert to µm ftp_link = 'ftp://phoenix.astro.physik.uni-goettingen.de/HiResFITS/' wave_file = 'WAVE_PHOENIX-ACES-AGSS-COND-2011.fits' if not os.path.isfile(path_to_models + '/' + wave_file): os.system('wget {0}{1}'.format(ftp_link, wave_file)) os.system('mv {0} {1}'.format(wave_file, path_to_models)) wave_phoenix = fits.getdata(path_to_models + '/' + wave_file) / 10 if 'FP' not in template: # get goettigen models if you don't have them. for temperature in np.arange(3000, 6100, 100): temperature = str(np.int(np.round(temperature, -2))) outname = '{0}/lte0{1}-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'.format( path_to_models, temperature) if not os.path.isfile(outname): os.system( 'wget {0}PHOENIX-ACES-AGSS-COND-2011/Z-0.0/lte0{1}-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits' .format(ftp_link, temperature)) os.system( 'mv lte0{1}-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits {0}' .format( path_to_models, temperature, )) # read template and header tbl, hdr = fits.getdata(template, ext=1, header=True) if 'FP' not in template: hdr2 = fits.getheader(template, ext=2) nsp_input = hdr2['NAXIS2'] else: nsp_input = 0 hdr['OBJECT'] = 'FP' out_pos_name = hdr['OBJECT'].upper() + '_pos.fits' if os.path.isfile(out_pos_name): print('File {} exists, we skip'.format(out_pos_name)) return if 'FP' not in template: # round temperature in header to nearest 100 and get the right model if 'OBJTEMP' in hdr: temperature = hdr['OBJTEMP'] if temperature < 3000: temperature = 3000 if temperature > 6000: temperature = 6000 temperature = str(np.int(np.round(temperature, -2))) else: temperature = '3600' print('Temperature = ', temperature) outname = '{0}/lte0{1}-4.50-0.0.PHOENIX-ACES-AGSS-COND-2011-HiRes.fits'.format( path_to_models, temperature) print('Model file = ', outname) flux_phoenix = fits.getdata(outname) # get wave and flux vectors w = np.array(tbl['wavelength']) f = np.array(tbl['flux']) f2 = np.array(f) mask = np.isfinite(f) f2[~mask] = 0 mask = mask * 1.0 f = np.convolve(f2, np.ones(5), mode='same') / np.convolve( mask, np.ones(5), mode='same') # find the first and second derivative of the flux df = np.gradient(f) ddf = np.gradient(np.gradient(f)) # lines are regions there is a sign change in the derivative of the flux # we also have some checks for NaNs line = np.where((np.sign(df[1:]) != np.sign(df[:-1])) & np.isfinite(ddf[1:]) & np.isfinite(df[1:]) & np.isfinite(df[:-1]))[0] # create the output table tbl = dict() tbl['ll_mask_s'] = np.zeros_like(line, dtype=float) tbl['ll_mask_e'] = np.zeros_like(line, dtype=float) # the weight is the second derivative of the flux. The sharper the line, # the more weight we give it tbl['w_mask'] = ddf[line] tbl['value'] = f[line] tbl['depth'] = np.zeros_like(tbl['value']) tbl['depth'][1:-1] = 1 - tbl['value'][1:-1] / ( (tbl['value'][0:-2] + tbl['value'][2:]) / 2) for i in tqdm(range(len(line))): # we perform a linear interpolation to find the exact wavelength # where the derivatives goes to zero wave_cen = (np.polyfit(df[line[i]:line[i] + 2], w[line[i]:line[i] + 2], 1))[1] # we offset that wavelength by the systemic velocity and subtract # half of the line width corrv = np.sqrt((1 + (-dv / 2) / c) / (1 - (-dv / 2) / c)) tbl['ll_mask_s'][i] = wave_cen * corrv # same but for the upper bound to the line position corrv = np.sqrt((1 + (dv / 2) / c) / (1 - (dv / 2) / c)) tbl['ll_mask_e'][i] = wave_cen * corrv weight = tbl['w_mask'] systemic_velocity = 0 if 'FP' not in template: # create a spline of the model model = InterpolatedUnivariateSpline(wave_phoenix, flux_phoenix) # assume a 0 velocity and search dv0 = 0 scale = 1.0 tbl0 = Table(tbl) low_contrast = False for ite in range(3): corrv = np.sqrt( (1 + systemic_velocity / c) / (1 - systemic_velocity / c)) tbl['ll_mask_s'] = tbl0['ll_mask_s'] / corrv tbl['ll_mask_e'] = tbl0['ll_mask_e'] / corrv wavelines = (tbl['ll_mask_s'] + tbl['ll_mask_e']) / 2.0 dvs = np.arange(400, dtype=float) dvs -= np.mean(dvs) dvs *= scale #dvs += systemic_velocity neg_mask = weight > 0 weight_tmp = weight[neg_mask] wave_tmp = wavelines[neg_mask] cc = np.zeros_like(dvs) for i in range(len(dvs)): corrv = np.sqrt((1 + dvs[i] / c) / (1 - dvs[i] / c)) cc[i] = np.sum(weight_tmp * model(wave_tmp / corrv)) # just centering the cc around one and removing low-f trends #cc = (cc / medfilt(cc, 21)) minpos = np.argmin(cc) fit = np.polyfit(dvs[minpos - 1:minpos + 2], cc[minpos - 1:minpos + 2], 2) if doplot: plt.plot(dvs + systemic_velocity, cc, alpha=0.5) systemic_velocity += (-.5 * fit[1] / fit[0]) print(systemic_velocity) scale /= 5.0 if np.min(cc) / np.max(cc) > 0.95: low_contrast = True print('not enough ccf contrast, will end after the plot') if doplot: plt.title('CCF of model SP with target' 's line list\nThis gets you the systemic velocity') plt.xlabel('Velocity') plt.ylabel('Abritrary flux') plt.show() if low_contrast: return hdr['SYSVELO'] = systemic_velocity, 'meas. systemic velocity (km/s)' hdr['VELOFILE'] = outname, 'model used for SYSVEL cc' print( '\n\tsystemic velocity : {0:.2f}km/s\n'.format(systemic_velocity)) if os.path.isfile(systemic_velocity_table) == False: tbl_sysvelo = Table() tbl_sysvelo['OBJECT'] = [hdr['OBJECT']] tbl_sysvelo['SYSTEMIC_VELOCITY'] = [systemic_velocity] tbl_sysvelo['MODEL_FILE'] = [hdr['VELOFILE']] print('We create {0}'.format(systemic_velocity_table)) tbl_sysvelo.write(systemic_velocity_table) else: tbl_old = Table.read(systemic_velocity_table) tbl_sysvelo = Table() tbl_sysvelo['OBJECT'] = np.append(hdr['OBJECT'], tbl_old['OBJECT']) tbl_sysvelo['SYSTEMIC_VELOCITY'] = np.append( systemic_velocity, tbl_old['SYSTEMIC_VELOCITY']) tbl_sysvelo['MODEL_FILE'] = np.append(hdr['VELOFILE'], tbl_old['MODEL_FILE']) print('We append {0}'.format(systemic_velocity_table)) tbl_sysvelo.write(systemic_velocity_table, overwrite=True) # convert back to table for manipulation tbl = et.td_convert(tbl) if 'FP' not in template: valid = np.isfinite(f) spline = InterpolatedUnivariateSpline(w[valid], f[valid], k=1, ext=0) # DETERMINATION OF H-band FWHM # cen, ew, amp, zp, slope dvs = np.arange(-50000, 50000, 500) + systemic_velocity * 1000 cc = np.zeros_like(dvs, dtype=float) H = (tbl['ll_mask_s'] > 1500) * (tbl['ll_mask_s'] > 1800) * (tbl['w_mask'] > 0) wave_H = np.array(tbl['ll_mask_s'][H]) weights_H = np.array(tbl['w_mask'][H]) for i in range(len(dvs)): cc[i] = np.sum(weights_H * spline(et.doppler(wave_H, -dvs[i]))) imin = np.nanargmin(cc) p0 = [ dvs[imin], 4000, np.nanmin(cc) - np.nanmedian(cc), np.nanmedian(cc), 0 ] fit_gau = et.fit_gauss(dvs, cc, p0) gfit = et.gauss(dvs, *fit_gau) cc /= np.polyval(fit_gau[[4, 3]], dvs) gfit /= np.polyval(fit_gau[[4, 3]], dvs) print(fit_gau) if doplot: plt.plot(dvs / 1000, cc, color='black', alpha=0.5, label='normalized CCF') plt.plot(dvs / 1000, gfit, alpha=0.5, label='normalized gaussian fit') plt.ylabel('flux') plt.xlabel('velocity [km/s]') plt.legend() plt.show() hdr['CCF_FWHM'] = np.sqrt( 2 * np.log(2)) * 2 * fit_gau[1] / 1000, 'H-band CCF FWHM in km/s' hdr['CCF_CONT'] = 1 - np.min(cc), 'Fractionnal CCF contrast' if doplot: plt.plot(w, f, 'g-', label='input spectrum') plt.vlines(tbl[tbl['w_mask'] < 0]['ll_mask_s'], np.nanmin(f), np.nanmax(f), 'k', alpha=0.2, label='positive feature') plt.vlines(tbl[tbl['w_mask'] > 0]['ll_mask_s'], np.nanmin(f), np.nanmax(f), 'r', alpha=0.2, label='negative feature') plt.legend() plt.xlabel('Wavelength [nm]') plt.ylabel('Arbitrary flux') plt.show() # write the output table fits.writeto(hdr['OBJECT'] + '.fits', tbl, hdr, overwrite=True) pos_mask = tbl['w_mask'] < 0 neg_mask = tbl['w_mask'] > 0 tbl['w_mask'] /= np.nanmean(np.abs(tbl['w_mask'])) tbl[pos_mask].write(hdr['OBJECT'] + '_pos.csv', format='ascii', overwrite=True) tbl[neg_mask].write(hdr['OBJECT'] + '_neg.csv', format='ascii', overwrite=True) tbl.write(hdr['OBJECT'] + '_full.mas', format='ascii', overwrite=True) tbl2 = tbl[tbl['w_mask'] > 0] tbl2['w_mask'] /= np.nanmedian(tbl2['w_mask']) tbl2['depth'] /= np.nanmedian(tbl2['depth']) tbl2['depth'] = np.abs(tbl2['depth']) f = open(hdr['OBJECT'] + '_neg.mas', 'w') for i in range(len(tbl2)): f.write(' ' + ' '.join([ str(tbl2['ll_mask_s'][i])[0:14], str(tbl2['ll_mask_e'][i])[0:14], str(tbl2['w_mask'][i])[0:12] ]) + '\n') f.close() f = open(hdr['OBJECT'] + '_neg_depth.mas', 'w') for i in range(len(tbl2)): f.write(' ' + ' '.join([ str(tbl2['ll_mask_s'][i])[0:14], str(tbl2['ll_mask_e'][i])[0:14], str(tbl2['depth'][i])[0:12] ]) + '\n') f.close() tbl2 = tbl[tbl['w_mask'] < 0] tbl2['w_mask'] /= np.nanmedian(tbl2['w_mask']) hdu1 = fits.PrimaryHDU() hdu1.header['SYSTVEL'] = systemic_velocity, 'Systemic velocity' hdu1.header['NSPTEMPL'] = nsp_input, 'Number of spectra used for tempalte' keys_transfer = ['OBJTEMP', 'PI_NAME', 'CCF_FWHM', 'CCF_CONT'] for key in keys_transfer: if key in hdr.keys(): hdu1.header[key] = hdr[key] hdu2 = fits.BinTableHDU(tbl2) # convert back from dictionnary to table and save new_hdul = fits.HDUList([hdu1, hdu2]) new_hdul.writeto(out_pos_name, overwrite=True)
meta=hdul['AGE_LW'].header, wcs=WCS(hdul['AGE_LW'].header)) tmp = nebulae[nebulae['gal_name'] == gal_name] positions = np.transpose((tmp['cen_x'], tmp['cen_y'])) apertures = CircularAperture(positions, 2) ages_mw = aperture_photometry(age_mw, apertures)['aperture_sum'] / apertures.area ages_lw = aperture_photometry(age_lw, apertures)['aperture_sum'] / apertures.area nebulae['age_mw'][nebulae['gal_name'] == gal_name] = ages_mw nebulae['age_lw'][nebulae['gal_name'] == gal_name] = ages_lw # write to file columns = ['gal_name', 'region_ID', 'age_mw', 'age_lw'] doc = f'''this catalogue contains the ages from the stellar populations fit measured at the position of the nebulae. last update: {date.today().strftime("%b %d, %Y")} ''' primary_hdu = fits.PrimaryHDU() for i, comment in enumerate(doc.split('\n')): if i == 0: primary_hdu.header['COMMENT'] = comment else: primary_hdu.header[''] = comment table_hdu = fits.BinTableHDU(nebulae[columns]) hdul = fits.HDUList([primary_hdu, table_hdu]) hdul.writeto('Nebulae_Catalogue_v2p1_age.fits', overwrite=True)