def subtract_stackedbias(self, stackedbias, stackedbias_err): sb = fits.open(stackedbias) sb_err = fits.open(stackedbias_err) self.data -= sb[1].data.T #self.data_err = np.sqrt(self.data_err**2.0 + sb_err[1].data.T**2.0) # TODO: use real RMS error self.data_err = np.sqrt(self.data_err**2.0 + 7.0**2.0)
def read_radio_fitsfile(self,mpgrp_fitsfile,ipgrp_fitsfile,radiogti_fitsfile): self.mpgrp_fitsfile = mpgrp_fitsfile self.ipgrp_fitsfile = ipgrp_fitsfile self.radiogti_fitsfile = radiogti_fitsfile if not os.path.exists(self.mpgrp_fitsfile): raise FileNotFoundError("{} not found.".format(self.mpgrp_fitsfile)) try: self.mpgrp_hdu = fits.open(self.mpgrp_fitsfile) except OSError as e: raise print("{} is successfully loaded.".format(self.mpgrp_fitsfile)) if not os.path.exists(self.ipgrp_fitsfile): raise FileNotFoundError("{} not found.".format(self.ipgrp_fitsfile)) try: self.ipgrp_hdu = fits.open(self.ipgrp_fitsfile) except OSError as e: raise print("{} is successfully loaded.".format(self.ipgrp_fitsfile)) if not os.path.exists(self.radiogti_fitsfile): raise FileNotFoundError("{} not found.".format(self.radiogti_fitsfile)) try: self.radiogti_hdu = fits.open(self.radiogti_fitsfile) except OSError as e: raise print("{} is successfully loaded.".format(self.radiogti_fitsfile))
def __init__(self, filename, ext=0): """Return a FITS image and the associated header. The image is returned as a numpy array. By default, the first HDU is read. A different HDU can be specified with the argument 'ext'. The header of the image is modified to remove unncessary keywords such as HISTORY and COMMENT, as well as keywords associated with a 3rd and 4th dimension (e.g. NAXIS3, NAXIS4). """ if not isinstance(filename, list): img_hdu = fits.open(filename) self.data = img_hdu[ext].data self.hdr = clean_header(img_hdu[ext].header) else: img_hdr = [] img_data = [] if ext == 0: ext = [ext] * len(filename) elif len(ext) != len(filename): raise IndexError('Length of the extension array must match \ number of images.') for i in range(len(filename)): img_hdu = fits.open(filename[i]) img_data.append(img_hdu[ext[i]].data) img_hdr.append(clean_header(img_hdu[ext[i]].header)) self.data = img_data self.hdr = img_hdr
def combine(fitsfiles, outputfile, method="average"): """Combine a set of FITS files, taking care of header keywords :argument fitsfiles: FITS filenames to combine :type fitsfiles: tuple :argument outputfile: output FITS filename :type outputfile: str :keyword method: average or sum the images :type method: str :returns: None """ if method is None: return N = len(fitsfiles) if N == 1: shutil.copyfile(fitsfiles[0], outputfile) return hdulist0 = pyfits.open(fitsfiles[0]) header0 = hdulist0[0].header data = hdulist0[0].data freqs = [header0['reffreq']] header0.update('orig0', os.path.basename(fitsfiles[0]), 'original fitsfile') for i, filename in enumerate(fitsfiles[1:]): with pyfits.open(filename) as hdulist: header = hdulist[0].header data += hdulist[0].data freqs.append(header['reffreq']) header0.update( 'orig%d' % (i + 1), os.path.basename(filename), 'original fitsfile') if method == "average": data /= float(N) minfreq, maxfreq = min(freqs), max(freqs) hdu = pyfits.PrimaryHDU(data) reffreq = (minfreq + maxfreq) / 2 bandwidth = maxfreq - minfreq header0.update('reffreq', reffreq, 'reference frequency') header0.update('bandwidt', bandwidth, 'estimated bandwidth') header0.update('FREQ_MIN', minfreq, 'minimum frequency') header0.update('FREQ_MAX', maxfreq, 'maximum frequency') # frequencies are stored in WCS coords (dimension 4), # but since we've copied those from the first image, # they need to be updated header0.update('crval4', reffreq) header0.update('cdelt4', bandwidth) hdu.header = header0 hdulist = pyfits.HDUList([hdu]) hdulist.writeto(outputfile) hdulist.close() hdulist0.close()
def create_master_frame(calibration_dir, extension, master_path, nframes): """ Take the median of a series of images to produce a master calibration frame (master dark, bias, ...) :param calibration_dir: path to calibration images (fits files) :param extension: file extension. Can be 'FTS', 'fits', 'FTS.gz', 'fits.gz' :return: Master image. Files are printed in dark_dir. """ file_list = glob.glob(os.path.join(calibration_dir, '*.' + extension)) hdu = fits.open(file_list[0], ignore_missing_end=True) header = hdu[0].header naxis1 = header['NAXIS1'] naxis2 = header['NAXIS2'] # If the number of files in the directory is smaller than the user input number of frames, # need to take the latter. nframes = np.min([len(file_list), nframes]) frames = np.zeros([naxis1, naxis2, nframes]) print('Creating master frame from %d frames...' %nframes) for i in range(0, nframes): hdu = fits.open(file_list[i], ignore_missing_end=True) frames[:,:,i] = hdu[0].data master_frame = np.median(frames, 2) # Write to disk if path is set for the master frame if master_path: write_uset_fits(master_frame, header, master_path) print('Wrote master into %s ' %master_path) return master_frame
def test_different_keyword_values(file1, file2): hdulist1 = fits.open(file1) header1 = hdulist1[0].header hdulist2 = fits.open(file2) header2 = hdulist2[0].header diff = HeaderDiff(header1, header2) assert not diff.identical
def set_grism_flat(grism='G141', verbose=True): import threedhst.grism_sky as bg if bg.flat_grism == grism: return True if verbose: print 'Set flat for grism: %s' %(grism) if grism == 'G141': flat_f140 = pyfits.open(IREF+'/uc721143i_pfl.fits') #flat_f140 = pyfits.open(IREF+'cosmos_f140w_flat.fits') #flat_f140 = pyfits.open(IREF+'/flat_3DHST_F140W_t1_v0.1.fits') flat_g141 = pyfits.open(IREF+'/u4m1335mi_pfl.fits') flat = flat_g141[1].data[5:1019,5:1019] / flat_f140[1].data[5:1019, 5:1019] flat[flat <= 0] = 5 flat[flat > 5] = 5 bg.flat = flat bg.flat_grism = 'G141' bg.flat_direct = flat_f140.filename() else: flat_f105 = pyfits.open(IREF+'/uc72113oi_pfl.fits') flat_g102 = pyfits.open(IREF+'/u4m1335li_pfl.fits') flat = flat_g102[1].data[5:1019,5:1019] / flat_f105[1].data[5:1019, 5:1019] flat[flat <= 0] = 5 flat[flat > 5] = 5 bg.flat = flat bg.flat_grism = 'G102' bg.flat_direct = flat_f105.filename() return True
def makeImgArray(obs_properties, flist, directory, raw_fname, exp_values): img_array=pandas.DataFrame.from_items([('expnum',[]),('refccd',[]),('refpng',[]), ('refdate',[]),('reftef',[]),('compimages',[]),('compexp',[]),('compccd',[]),('compdate',[]),('comptef',[])]) def getkey(item): return item[0] for i in range(0,len(obs_properties)): refnum=obs_properties['expnum'][i] #refccd=obs_properties['ccd'][i] refccd=obs_properties['ccd'][i] refband=obs_properties['band'][i] refdate=obs_properties['date'][i] ref_ra=obs_properties['ra'][i] ref_dec=obs_properties['dec'][i] CompImgs=[] expnums=[] refpng='' reftef=round(float(exp_values[exp_values['expnum']==refnum]['t_eff']),3) for thumb in flist: os.chdir(directory+'/'+raw_fname+'/'+thumb) fit_list = glob.glob('*fits') for f in fit_list: h=fits.open(f) if h[0].header["EXPNUM"]==refnum and h[0].header['CCDNUM']==refccd: reffit=f refpng=thumb+'/'+reffit[:-5]+'Circ.png' drawObsCircle(reffit,reffit[:-5]+'Circ.png','temp.reg',ref_ra,ref_dec) for g in fit_list: j=fits.open(g) jnum=j[0].header['EXPNUM'] if (j[0].header['BAND']==refband and jnum != refnum and not (jnum in expnums)): try: teff=float(exp_values[exp_values['expnum']==jnum]['t_eff']) except: teff=0 print jnum pnglabel=g[:-5]+'Circ.png' drawObsCircle(g, pnglabel, 'temp.reg',ref_ra, ref_dec) date=str(j[0].header['DATE-OBS']) nicedate=date[0:4]+'/'+date[5:7]+'/'+date[8:10]+' '+date[11:19] CompImgs.append([teff,thumb+'/'+pnglabel, jnum, j[0].header['CCDNUM'], nicedate]) expnums+=[jnum] j.close() flist.remove(thumb) break h.close() sortedcomps=sorted(CompImgs, key=getkey, reverse=True) tempframe=pandas.DataFrame.from_items([('expnum', refnum),('refccd',refccd), ('refpng',[ refpng]), ('refdate', [refdate]),('reftef',[reftef]),('compimages', [[x[1] for x in sortedcomps]]),('compexp',[[x[2] for x in sortedcomps]]), ('compccd',[[x[3] for x in sortedcomps]]),('compdate',[[x[4] for x in sortedcomps]]),('comptef',[[round(x[0],3) for x in sortedcomps]])]) img_array=img_array.append(tempframe, ignore_index=True) return img_array
def test_identical_number_keywords_headers(file1, file2): hdulist1 = fits.open(file1) header1 = hdulist1[0].header hdulist2 = fits.open(file2) header2 = hdulist2[0].header diff = HeaderDiff(header1, header2) assert diff.diff_keyword_count[0] == diff.diff_keyword_count[1]
def getQsoCatalogueAllObjects(): ''' ''' path = '/home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/' listPAth = [path+'QSO_DR7_DR12_EBOSS_2016_01_08.fits', path+'DLA_all.fits', path+'all_Britt.fits', path+'VIPERS.fits', path+'QSO_3DHST.fits', path+'LOWZ_all.fits', path+'CMASS_all.fits'] name = ['QSO','DLA','Britt','VIPERS','3DHST','LOWZ','CMASS'] ## Distribution redshift for i in numpy.arange(len(listPAth)): cat = pyfits.open(listPAth[i], memmap=True )[1].data cat = cat[ (cat['Z']>0.1) ] cat = cat[ (cat['Z']<7.) ] if (cat.size==0): continue plt.hist(cat['Z'], bins=100,histtype='step',label=name[i]) plt.xlabel("Z") plt.ylabel("#") myTools.deal_with_plot(False,False,True) plt.show() ### Merge everyThing cat = pyfits.open(listPAth[0], memmap=True )[1].data ra = cat['RA'] de = cat['DEC'] zz = cat['Z'] for i in numpy.arange(1,len(listPAth)): cat = pyfits.open(listPAth[i], memmap=True )[1].data ra = numpy.append(ra, cat['RA']) de = numpy.append(de, cat['DEC']) zz = numpy.append(zz, cat['Z']) ## Map plt.ticklabel_format(style='sci', axis='z', scilimits=(0,0)) plt.grid() plt.plot(ra, de, linestyle="", marker="o") plt.xlabel("Right Ascension (degree)") plt.ylabel("Declination (degree)") plt.show() ## Distribution redshift plt.ticklabel_format(style='sci', axis='z', scilimits=(0,0)) plt.grid() plt.hist(zz, bins=200) plt.xlabel("Z") plt.ylabel("#") plt.show() ### Save col_ra = pyfits.Column(name='RA', format='D', array=ra, unit='deg') col_de = pyfits.Column(name='DEC', format='D', array=de, unit='deg') col_zz = pyfits.Column(name='Z', format='D', array=zz) tbhdu = pyfits.BinTableHDU.from_columns([col_ra, col_de, col_zz]) tbhdu.writeto('/home/gpfs/manip/mnt0607/bao/hdumasde/Data/Catalogue/ALL_EVERY_OBJECTS_2016_01_08.fits', clobber=True)
def set_total_costs_matrix(self, i, j, def_param = None): def_param = total_costs_matrix_base curve_name_i = all_curves[i][0] curve_type_i = all_curves[i][1] curve_file_i = fits.open(os.getcwd()+'/memoria/'+ 'inputs/'+curve_type_i+'/'+curve_name_i+'.fits', memmap=False) curve_data_i = Extractor.get_values(curve_file_i) curve_file_i.close() curve_name_j = all_curves[j][0] curve_type_j = all_curves[j][1] curve_file_j = fits.open(os.getcwd()+'/memoria/'+ 'inputs/'+curve_type_j+'/'+curve_name_j+'.fits', memmap=False) curve_data_j = Extractor.get_values(curve_file_j) curve_file_j.close() x,y = curve_data_i, curve_data_j dtw = DTW(x,y) cost_matrix = dtw.compute_cost_matrix(DTW.euclidean_distance) acc_cost_matrix, cost = dtw.compute_acc_cost_matrix(cost_matrix) self.total_costs_matrix[i,j] = cost
def test_aa(self): cmd = """{executable} {specter_dir}/bin/specter \ -i {sky} \ -p {monospot_file} \ -t {throughput_file} \ -o {imgfile} \ -w 7500,7620 \ -n --specmin 0 --nspec 2 --exptime 1500""".format( executable=self.executable, specter_dir=self.specter_dir, sky=self.sky_file, monospot_file=self.monospot_file, throughput_file=self.throughput_file, imgfile = imgfile1) print(cmd) err = os.system(cmd) self.assertEqual(err, 0, 'Error code {} != 0'.format(err)) self.assertTrue(os.path.exists(imgfile1)) with fits.open(imgfile1) as fx: self.assertIn('CCDIMAGE', fx) self.assertIn('IVAR', fx) #- Test the I/O routines while we have the file handy image, ivar, hdr = read_image(imgfile1) self.assertEqual(image.shape, ivar.shape) os.remove(imgfile1) cmd = cmd + ' --extra' err = os.system(cmd) self.assertEqual(err, 0, 'Error code {} != 0'.format(err)) self.assertTrue(os.path.exists(imgfile1)) with fits.open(imgfile1) as fx: self.assertIn('PHOTONS', fx) self.assertIn('XYWAVE', fx)
def mask_satellite_trail(flc='jcdu36e5q_flc.backup.fits', mask_files=['jcdu36e5q_mask_1.reg', 'jcdu36e5q_mask_2.reg']): """ Use Pyregion to mask satellite trails with a region file. Note: save the region file in "fk5" sky coordinates """ import astropy.io.fits as pyfits import pyregion ### open the FITS image for modifying im = pyfits.open(flc) ### Make the mask for both "DQ" extensions for both ACS/WFC chips for extension in [1,2]: ### Get a mask returning 1 *within* the region and 0 elsewhere im['SCI',extension].header.remove('CPDIS1') im['SCI',extension].header.remove('CPDIS2') reg = pyregion.open(mask_files[extension-1]).as_imagecoord(im['SCI', extension].header) mask = reg.get_mask(im['SCI', extension]) ### Apply the mask to the DQ extension im['DQ', extension].data |= 4096*mask imf = pyfits.open(flc, mode='update') for extension in [1,2]: imf['DQ', extension].data = im['DQ',extension].data ### Write the file imf.flush()
def validate_vs_jwpsf_nircam(): models = [ ('NIRCam','F200W', 'f200w_perfect_offset', '/Users/mperrin/software/jwpsf_v3.0/data/NIRCam/OPD/perfect_opd.fits', 0.034,True), ('NIRCam','F200W', 'f200w_perfect', '/Users/mperrin/software/jwpsf_v3.0/data/NIRCam/OPD/perfect_opd.fits', 0.034,False), ('NIRCam','F200W', 'f200w', '/Users/mperrin/software/jwpsf_v3.0/data/NIRCam/OPD/nircam_obs_w_rsrv1.fits', 0.034,True), ('MIRI','F1000W', 'f1000w', '/Users/mperrin/software/jwpsf_v3.0/data/MIRI/OPD/MIRI_OPDisim1.fits', 0.11,True)] fig = P.figure(1, figsize=(13,8.5), dpi=80) oversamp=4 for params in models: nc = webbpsf_core.Instrument(params[0]) nc.filter = params[1] nc.pupilopd = params[3] #'/Users/mperrin/software/jwpsf_v3.0/data/NIRCam/OPD/nircam_obs_w_rsrv1.fits' nc.pixelscale = params[4] #0.034 # this is wrong, but compute this way to match JWPSF exactly if params[5]: # offset by half a pixel to match the JWPSF convention nc.options['source_offset_r'] = params[4]/2 * N.sqrt(2)/oversamp # offset half a pixel each in X and Y nc.options['source_offset_theta'] = -45 jw_fn = 'jwpsf_%s_%s.fits' % (params[0].lower(), params[2].lower()) my_fn = 'test_vs_' + jw_fn if not os.path.exists( my_fn): my_psf = nc.calcPSF(my_fn, oversample=oversamp, fov_pixels=512./oversamp) else: my_psf = fits.open(my_fn) jw_psf = fits.open(jw_fn) jw_psf[0].header.update('PIXELSCL', jw_psf[0].header['CDELT1']*3600) P.clf() #P.subplots_adjust(top=0.95, bottom=0.05, left=0.01, right=0.99) P.subplot(231) titlestr = "%s %s, \n"% (params[0], params[2]) poppy.display_PSF(my_psf, title=titlestr+"computed with WebbPSF" , colorbar=False) P.subplot(232) poppy.display_PSF(jw_psf, title=titlestr+"computed with JWPSF" , colorbar=False) P.subplot(233) poppy.display_PSF_difference(my_psf,jw_psf, title=titlestr+'Difference Image', colorbar=False) imagecrop = 30*params[4] P.subplot(234) poppy.display_PSF(my_psf, title=titlestr+"computed with WebbPSF", colorbar=False, imagecrop=imagecrop) centroid = poppy.measure_centroid(my_psf) P.gca().set_xlabel("centroid = (%.3f,%.3f)" % centroid) P.subplot(235) poppy.display_PSF(jw_psf, title=titlestr+"computed with JWPSF", colorbar=False, imagecrop=imagecrop) centroid = poppy.measure_centroid(jw_psf) P.gca().set_xlabel("centroid = (%.3f,%.3f)" % centroid) P.subplot(236) poppy.display_PSF_difference(my_psf,jw_psf, title='Difference Image', colorbar=False, imagecrop=imagecrop) P.savefig("results_vs_jwpsf_%s_%s.pdf" % (params[0], params[2]))
def deep_obj_mask(img, ota, apply=False): from astropy.io import fits from astropy.stats import sigma_clipped_stats image = odi.scaledpath+'scaled_'+ota+'.'+str(img[16:]) ota_mask = 'objmask_'+ota+'.'+str(img[16:17])+'.fits' hdulist = fits.open(image) hdu_ota = hdulist[0] # maskhdu = fits.open(bppath+ota_mask) gapshdu = fits.open(odi.bppath+'reproj_mask_'+ota+'.'+str(img[16:])) total_mask = gapshdu[0].data #maskhdu[0].data + nx, ny = hdu_ota.data.shape print nx, ny mean1, median1, std1 = sigma_clipped_stats(hdu_ota.data[0:ny/2,0:nx/2], mask=total_mask[0:ny/2,0:nx/2], sigma=3.0, iters=3) mean2, median2, std2 = sigma_clipped_stats(hdu_ota.data[0:ny/2,nx/2:nx], mask=total_mask[0:ny/2,nx/2:nx], sigma=3.0, iters=3) mean3, median3, std3 = sigma_clipped_stats(hdu_ota.data[ny/2:ny,0:nx/2], mask=total_mask[ny/2:ny,0:nx/2], sigma=3.0, iters=3) mean4, median4, std4 = sigma_clipped_stats(hdu_ota.data[ny/2:ny,nx/2:nx], mask=total_mask[ny/2:ny,nx/2:nx], sigma=3.0, iters=3) mean = [mean1, mean2, mean3, mean4] median = [median1, median2, median3, median4] std = [std1, std2, std3, std4] # plt.imshow(hdu_ota.data, origin='lower', cmap='Greys_r', vmin=-10., vmax=500.) # plt.imshow(total_mask, cmap=random_cmap(), alpha=0.5) # plt.show() return mean, median, std
def Downsample(filein,keys=[]): """ Selects keys from a fits file and produces a new file. -Input: filein (str): file to read. keys (list): a list containing the columnames to keep. """ data = fits.open(filein)[1].data columnnames = fits.open(filein)[1].columns.names columnformats = fits.open(filein)[1].columns.formats for key_ in keys: if not key_ in columnames: raise ValueError('The name '+key_+' is not present.') newdata = [] newcol = [] newtype = [] for key_ in keys: newdata.append( data[key_] ) newcol.append( key_ ) newtype.append( columnformats[columnnames.index(key_)] ) columnlist = map(lambda name_,format_,array_: fits.Column( name=name_,format=format_,array=array_ ),newcol,newtype,newdata) cols = fits.ColDefs(columnlist) tbhdu = fits.BinTableHDU.from_columns(cols) tbhdu.writeto(filein+'_downsample')
def read(cls, filename, er_filename=None, **kwargs): """ Reads in a given FITS file and returns a dictionary of new EISSpectralCubes. Additional parameters are given to fits.open. Parameters ---------- filename: string Complete location of the FITS file er_filename: string Location of the error FITS file """ hdulist = fits.open(name=filename, **kwargs) errlist = fits.open(er_filename) if er_filename is not None else None header = _clean(hdulist[0].header) # TODO: Make sure each cube has a correct wcs. wavelengths = [c.name for c in hdulist[1].columns if c.dim is not None] data = [hdulist[1].data[wav] for wav in wavelengths] errs = [errlist[1].data[wav] if errlist is not None else None for wav in wavelengths] cubes = [] for i in range(len(data)): window = i + 1 header = _dictionarize_header(hdulist[1].header, hdulist[0].header, window) wcs = WCS(header=header, naxis=3) uncertainty = sdu(errs[i]) if errlist is not None else None cubes += [EISCube(data[i], wcs, header, errors=uncertainty)] return dict(zip(wavelengths, cubes))
def FITS_to_sinos(fitsdir,slice_frac,sample_freq): """ Takes a path to the directory of FITS files, assumed only having files (no directories) and an 2-length array (read as 1d) of the first and last vertical portions to store. These are read as fractions of the whole (for example, [.1,.9] will read all data between 10% and 90%--only includ- ing the file at 10% and not the one at 90%--of the vertical). We sample the files with frequency sample_freq, so downsampling along the After reading this data from the FITS files, sinograms are formatted. SUudirectories are ignored. We assume all files have the same shaped images and that fitsdir DOES end with a '/'. Also, assume that slicefrac[1]>=slicefrac[0] """ from astropy.io import fits import glob #Get list of files files = glob.glob(fitsdir+'*.fits') #If in Mac, need to remove hidden files, not sure how to do it smart hdulist = fits.open(files[0],mode = 'readonly') data = hdulist[0].data #Determine slices which we store ind1 = np.round(slice_frac[0]*data.shape[1]).astype(int) ind2 = np.round(slice_frac[1]*data.shape[1]).astype(int) if ind1==ind2: indices =ind1 else: indices = np.arange(ind1,ind2,1) file_ind = range(0,len(files),sample_freq) sinos = np.empty([indices.size,data.shape[0],len(file_ind)]) for i in range(len(file_ind)): hdulist = fits.open(files[file_ind[i]]) sinos[:,:,i]= hdulist[0].data[indices,:] sinos = np.transpose(sinos,(1,2,0)) return sinos
def Mask(filecat=None,filemask=[],maskname=[]): """ Given a Fits file, appends the value of the mask to the table fits on a new file -Input: filecat (str): The name of the file with the catalog. filemask (list): List of the names of the file with the mask. maskname (list): List of the names of the new field """ if len(filemask) == 0: raise ValueError('No mask given.') if not len(filemask) == len(maskname): raise ValueError('The number of files and headers does not match.') catalog = fits.open(filecat)[1].data columnnames = fits.open(filecat)[1].columns.names columnformats = fits.open(filecat)[1].columns.formats masklist = [] for file_ in filemask: masklist.append( GetMaskArray(file_,catalog['ra'],catalog['dec']) ) columns = [ catalog[col_] for col_ in columnnames ] columns += masklist columnnames += maskname columnformats += [ 'E' for name_ in maskname ] columnlist = map(lambda name_,format_,array_: fits.Column( name=name_,format=format_,array=array_ ),columnnames,columnformats,columns) cols = fits.ColDefs(columnlist) tbhdu = fits.BinTableHDU.from_columns(cols) tbhdu.writeto(filecat+'_'.join(maskname))
def gaia(filename,estimator='max',axis=0): f = pyfits.open(filename) hdr = f[0].header cube = f[0].data dv,v0,p3 = hdr.get('CD3_3'),hdr.get('CRVAL3'),hdr.get('CRPIX3') dr,r0,p1 = hdr.get('CD1_1'),hdr.get('CRVAL1'),hdr.get('CRPIX1') dd,d0,p2 = hdr.get('CD2_2'),hdr.get('CRVAL2'),hdr.get('CRPIX2') if dv is None: dv = hdr.get('CDELT3') if dr is None: dr = hdr.get('CDELT1') if dd is None: dd = hdr.get('CDELT2') xtora = lambda x: (x-p1+1)*dr+r0 # convert pixel coordinates to RA/Dec/Velocity ytodec = lambda y: (y-p2+1)*dd+d0 vconv = lambda v: (v-p3+1)*dv+v0 if axis > 0: cube = cube.swapaxes(0,axis) if estimator == 'max': p = where(isnan(cube),0,cube).max(axis=0) elif estimator == 'int': p = where(isnan(cube),0,cube).sum(axis=0) * dv elif estimator == 'intdivmax': cut = MAD(cube.ravel()) + nanmedian(cube.ravel()) if cut < 0: cut = 0 m = where(isnan(cube),0,cube).max(axis=0) i = where(isnan(cube),0,cube).sum(axis=0) * dv p = where(i<0,0,i)/where(m<=cut,numpy.inf,m) elif estimator[-5:] == ".fits": p = pyfits.open(estimator)[0].data mapplot(p,cube,vconv,xtora,ytodec)
def cross_correlation_HST_diff_NDR(): # Cross correlated the differential non-destructive reads from HST Scanning mode with WFC3 and G141 import image_registration as ir from glob import glob from pylab import *;ion() from astropy.io import fits fitsfiles = glob("*ima*fits") ylow = 50 yhigh = 90 nExts = 36 extGap = 5 shifts_ndr = zeros((len(fitsfiles), (nExts-1)//extGap, 2)) fitsfile0 = fits.open(fitsfiles[0]) for kf, fitsfilenow in enumerate(fitsfiles): fitsnow = fits.open(fitsfilenow) for kndr in range(extGap+1,nExts+1)[::extGap][::-1]: fits0_dndrnow = fitsfile0[kndr-extGap].data[ylow:yhigh]- fitsfile0[kndr].data[ylow:yhigh] fitsN_dndrnow = fitsnow[kndr-extGap].data[ylow:yhigh] - fitsnow[kndr].data[ylow:yhigh] shifts_ndr[kf][(kndr-1)//extGap-1] = ir.chi2_shift(fits0_dndrnow, fitsN_dndrnow)[:2] #ax.clear() #plt.imshow(fitsN_dndrnow) #ax.set_aspect('auto') #plt.pause(1e-3) plot(shifts_ndr[:,:-1,0],'o') # x-shifts plot(shifts_ndr[:,:-1,1],'o') # y-shifts
def test_iteration_of_lazy_loaded_hdulist(self): """ Regression test for https://github.com/astropy/astropy/issues/5585 """ hdulist = fits.HDUList(fits.PrimaryHDU()) hdulist.append(fits.ImageHDU(name='SCI')) hdulist.append(fits.ImageHDU(name='SCI')) hdulist.append(fits.ImageHDU(name='nada')) hdulist.append(fits.ImageHDU(name='SCI')) filename = self.temp('many_extension.fits') hdulist.writeto(filename) f = fits.open(filename) # Check that all extensions are read if f is not sliced all_exts = [ext for ext in f] assert len(all_exts) == 5 # Reload the file to ensure we are still lazy loading f.close() f = fits.open(filename) # Try a simple slice with no conditional on the ext. This is essentially # the reported failure. all_exts_but_zero = [ext for ext in f[1:]] assert len(all_exts_but_zero) == 4 # Reload the file to ensure we are still lazy loading f.close() f = fits.open(filename) # Check whether behavior is proper if the upper end of the slice is not # omitted. read_exts = [ext for ext in f[1:4] if ext.header['EXTNAME'] == 'SCI'] assert len(read_exts) == 2
def test_update_resized_header2(self): """ Regression test for https://aeon.stsci.edu/ssb/trac/pyfits/ticket/150 This is similar to test_update_resized_header, but specifically tests a case of multiple consecutive flush() calls on the same HDUList object, where each flush() requires a resize. """ data1 = np.arange(100) data2 = np.arange(100) + 100 phdu = fits.PrimaryHDU(data=data1) hdu = fits.ImageHDU(data=data2) phdu.writeto(self.temp('temp.fits')) with fits.open(self.temp('temp.fits'), mode='append') as hdul: hdul.append(hdu) with fits.open(self.temp('temp.fits'), mode='update') as hdul: idx = 1 while len(str(hdul[0].header)) <= 2880 * 2: hdul[0].header['TEST{}'.format(idx)] = idx idx += 1 hdul.flush() hdul.append(hdu) with fits.open(self.temp('temp.fits')) as hdul: assert (hdul[0].data == data1).all() assert hdul[1].header == hdu.header assert (hdul[1].data == data2).all() assert (hdul[2].data == data2).all()
def read(cls, phafile, rmffile=None): """Read PHA fits file The energy binning is not contained in the PHA standard. Therefore is is inferred from the corresponding RMF EBOUNDS extension. Parameters ---------- phafile : str PHA file with ``SPECTRUM`` extension rmffile : str RMF file with ``EBOUNDS`` extennsion, optional """ phafile = make_path(phafile) spectrum = fits.open(str(phafile))["SPECTRUM"] counts = [val[1] for val in spectrum.data] if rmffile is None: val = spectrum.header["RESPFILE"] if val == "": raise ValueError("RMF file not set in PHA header. " "Please provide RMF file for energy binning") parts = phafile.parts[:-1] rmffile = Path.cwd() for part in parts: rmffile = rmffile.joinpath(part) rmffile = rmffile.joinpath(val) rmffile = make_path(rmffile) ebounds = fits.open(str(rmffile))["EBOUNDS"] bins = EnergyBounds.from_ebounds(ebounds) livetime = Quantity(0, "s") return cls(counts, bins, livetime=livetime)
def fits_osiris_allclose(a, b): """Assert that two OSIRIS fits files are close.""" a = fits.open(a) b = fits.open(b) try: del a[0].header['COMMENT'] del b[0].header['COMMENT'] report = StringIO() diff = FITSDiff( a, b, ignore_keywords=["COMMENT"], ignore_comments=["SIMPLE"], ignore_fields=[], ignore_blanks=True, ignore_blank_cards=True, tolerance=1e-5) diff.report(fileobj=report) assert diff.identical, report.getvalue() finally: a.close() b.close()
def test_parnames_round_trip(self): """ Regression test for https://aeon.stsci.edu/ssb/trac/pyfits/ticket/130 Ensures that opening a random groups file in update mode or writing it to a new file does not cause any change to the parameter names. """ # Because this test tries to update the random_groups.fits file, let's # make a copy of it first (so that the file doesn't actually get # modified in the off chance that the test fails self.copy_file('random_groups.fits') parameters = ['UU', 'VV', 'WW', 'BASELINE', 'DATE'] with fits.open(self.temp('random_groups.fits'), mode='update') as h: assert h[0].parnames == parameters h.flush() # Open again just in read-only mode to ensure the parnames didn't # change with fits.open(self.temp('random_groups.fits')) as h: assert h[0].parnames == parameters h.writeto(self.temp('test.fits')) with fits.open(self.temp('test.fits')) as h: assert h[0].parnames == parameters
def test_translated_map(): """ Map a pixel array to at translated array. """ first_file = os.path.join(DATA_DIR, 'input1.fits') first_hdu = fits.open(first_file) first_header = first_hdu[1].header first_wcs = wcs.WCS(first_header) naxis1, naxis2 = first_wcs.pixel_shape first_hdu.close() second_file = os.path.join(DATA_DIR, 'input3.fits') second_hdu = fits.open(second_file) second_header = second_hdu[1].header second_wcs = wcs.WCS(second_header) second_hdu.close() ok_pixmap = np.indices((naxis1, naxis2), dtype='float32') - 2.0 ok_pixmap = ok_pixmap.transpose() pixmap = calc_pixmap.calc_pixmap(first_wcs, second_wcs) npt.assert_equal(pixmap.shape, ok_pixmap.shape) # Got x-y transpose right npt.assert_almost_equal(pixmap, ok_pixmap, decimal=5) # Mapping an array to a translated array
def test_save_backup(self): """Test for https://aeon.stsci.edu/ssb/trac/pyfits/ticket/121 Save backup of file before flushing changes. """ self.copy_file('scale.fits') with ignore_warnings(): with fits.open(self.temp('scale.fits'), mode='update', save_backup=True) as hdul: # Make some changes to the original file to force its header # and data to be rewritten hdul[0].header['TEST'] = 'TEST' hdul[0].data[0] = 0 assert os.path.exists(self.temp('scale.fits.bak')) with fits.open(self.data('scale.fits'), do_not_scale_image_data=True) as hdul1: with fits.open(self.temp('scale.fits.bak'), do_not_scale_image_data=True) as hdul2: assert hdul1[0].header == hdul2[0].header assert (hdul1[0].data == hdul2[0].data).all() with ignore_warnings(): with fits.open(self.temp('scale.fits'), mode='update', save_backup=True) as hdul: # One more time to see if multiple backups are made hdul[0].header['TEST2'] = 'TEST' hdul[0].data[0] = 1 assert os.path.exists(self.temp('scale.fits.bak')) assert os.path.exists(self.temp('scale.fits.bak.1'))
def genFitsMeans(cubeName,procDir='/data/VIMS/covims_0004/procdata'): log={} cwd=os.getcwd() cube=cubeName.split('/')[1].split('.')[0] opDir=procDir+'/'+cubeName.split('/')[0] os.chdir(opDir) try: with pyf.open('c'+cube+'_ir.fits') as irHan: ir=irHan[0].data mean=num.mean(num.concatenate([ir[:34,:,:],ir[82:105]]),axis=0) print mean.shape HDU=pyf.PrimaryHDU(mean) list=pyf.HDUList([HDU]) if os.path.isfile('c'+cube+'_ir_mean.fits'): os.remove('c'+cube+'_ir_mean.fits') list.writeto('c'+cube+'_ir_mean.fits') except: pass try: with pyf.open('c'+cube+'_vis.fits') as visHan: vis=visHan[0].data mean=num.mean(vis[:100,:,:],axis=0) HDU=pyf.PrimaryHDU(mean) list=pyf.HDUList([HDU]) if os.path.isfile('c'+cube+'_vis_mean.fits'): os.remove('c'+cube+'_vis_mean.fits') list.writeto('c'+cube+'_vis_mean.fits') except: pass os.chdir(cwd)
def main(): sv = fits.open(home_dir + "DES/data/sva1_gold_detmodel_gals.fits")[1].data hi = fits.open(home_dir + "DES/magnification/lbgselect/hi-z_mag_probability.fits")[1].data hi_test = fits.open(home_dir + "DES/magnification/lbgselect/hi-z_mag_probability_test.fits")[1].data lo = fits.open(home_dir + "DES/magnification/lbgselect/lo-z_mag_probability_test.fits")[1].data hi.sort(), sv.sort(), lo.sort(), hi_test.sort() svid, rid = match(sv["coadd_objects_id"], hi["coadd_objects_id"]) ri = sv["mag_detmodel_r"][svid] - sv["mag_detmodel_i"][svid] gr = sv["mag_detmodel_g"][svid] - sv["mag_detmodel_r"][svid] Phi = hi["hi-z_prob"][rid] / (hi["hi-z_prob"][rid] + lo["lo-z_prob"][rid]) Plo = lo["lo-z_prob"][rid] / (hi["hi-z_prob"][rid] + lo["lo-z_prob"][rid]) print "P(hi-z)>=0.8: {}/{}".format(len(Phi[Phi >= 0.8]), len(Phi)) print "P(hi-z)==P(hi-z)_test? {}".format(np.all(hi["hi-z_prob"] == hi_test["hi-z_prob"])) # plt.scatter(Plo, Phi, edgecolor='none') # plt.ylim(0.8, 1.0) # plt.xlabel('P(lo-z)') # plt.ylabel('P(hi-z)') # plt.savefig(home_dir+'DES/magnification/lbgselect/Phi-vs-Plo') # plt.close() plt.scatter(np.arange(len(hi["hi-z_prob"])), hi["hi-z_prob"] - hi_test["hi-z_prob"], edgecolor="none") plt.ylabel("old code - new code") plt.savefig(home_dir + "DES/magnification/lbgselect/oldhi-vs_newhi") plt.close()
# Two command-line arguments: Name of a file containing a list of the single-sensor # FITS images and the name of the output file # Merge a set of single CCD images into a (large) FITS file # assumed to be in S00, SO1, ... order import astropy.io.fits as fits import sys # Read the list of files io = open(sys.argv[1], "r") infiles = [] for line in io: infiles.append(line.rstrip()) io.close() hdulist = fits.HDUList() # Take primary header from the first file in the list hdu = fits.open(infiles[0]) hdulist.append(hdu[0]) for infile in infiles: hdu = fits.open(infile) for i in range(1, 17): hdulist.append(hdu[i]) hdulist.writeto(sys.argv[2])
def subtract_sky_background(in_files, out_files=None, method='scalar', source='sky', mask_sources=False, file_path=None, tmp_dir=None, show=False, debug=False): """Estimate and subtract the sky background via different methods and sources. TODO: Implement sky subtraction from image Args: in_files (specklepy.FileArchive): File archive storing the information of all the files in the reduction. out_files (list): List of files to apply the sky subtraction to. If left empty, the list stored in the `in_files` FileArchive is used. method (str, optional): Switch between a scalar (`scalar`) background value or a 2D image (`images`). source (str, optional): Source for estimating the background from. Can be either `sky` to measure from dedicated sky frames or `science` to use the science frames themselves. Typically, these frames have a high number of sources, so `mask_sources` should be switched on. mask_sources (bool, optional): In empty reference fields, this masking option should stay at `False`, since source masking is not well tested. However, masking sources yields a more precise result. file_path (str, optional): Path to the files, listed in `in_files`. tmp_dir (str, optional): Directory to which temporary results and QA data is stored. show (bool, optional): Show plots of sky estimates for each sequence. They will be created and stored regardless of this choice. debug (bool, optional): Show debugging information. """ # Set logging level if debug: logger.setLevel('DEBUG') # Apply fall back values if method is None: method = 'scalar' logger.info(f"Sky background subtraction method: {method}") if source is None: source = 'sky' logger.info(f"Sky background subtraction source: {source}") if out_files is None: out_files = in_files.product_files if out_files is None: logger.warning( f"Output files are not declared in subtract_sky_background!") # Identify the observing sequences sequences = in_files.identify_sequences(source=source) # Start the background estimates if method == 'scalar': # Iterate through observing sequences for s, sequence in enumerate(sequences): logger.info( f"Starting observing sequence {s} :: Object {sequence.object} :: Setup {sequence.setup}" ) # Compute weights based on the time offset to the individual sky observations weights = sequence.compute_weights() # Start extracting sky fluxes sky_bkg = np.zeros(sequence.n_sky) sky_bkg_std = np.zeros(sequence.n_sky) for i in trange(sequence.n_sky, desc='Estimate sky background from cube'): file = sequence.sky_files[i] bkg, d_bkg = estimate_sky_background(file, method=method, mask_sources=mask_sources, path=file_path) sky_bkg[i] = bkg sky_bkg_std[i] = d_bkg logger.debug( f"Shapes:\nF: {sky_bkg.shape}\ndF: {sky_bkg_std.shape}") # Compute weighted sky background for each science file weighted_sky_bkg = np.dot(weights, sky_bkg) weighted_sky_bkg_var = np.dot(np.square(weights), np.square(sky_bkg_std)) # Store sky background estimates sky_bkg_table = Table(data=[ sequence.sky_files, weighted_sky_bkg, weighted_sky_bkg_var ], names=['FILE', 'BKG', 'VAR']) sky_bkg_table_name = f"sky_bkg_{sequence.object}_{sequence.setup}.fits" sky_bkg_table.write(os.path.join(tmp_dir, sky_bkg_table_name), overwrite=True) # Plot sky flux estimates for i, file in enumerate(sequence.sky_files): plt.text(sequence.sky_time_stamps[i], sky_bkg[i], file, rotation=90, alpha=.5) for i, file in enumerate(sequence.science_files): plt.text(sequence.science_time_stamps[i], weighted_sky_bkg[i], file, rotation=90, alpha=.66) plt.errorbar(x=sequence.sky_time_stamps, y=sky_bkg, yerr=sky_bkg_std, fmt='None', ecolor='tab:blue', alpha=.5) plt.plot(sequence.sky_time_stamps, sky_bkg, 'D', label='Sky', c='tab:blue') plt.errorbar(x=sequence.science_time_stamps, y=weighted_sky_bkg, yerr=np.sqrt(weighted_sky_bkg_var), fmt='None', ecolor='tab:orange', alpha=.66) plt.plot(sequence.science_time_stamps, weighted_sky_bkg, 'D', label='Science', c='tab:orange') plt.xlabel('Time (s)') plt.ylabel('Flux (counts)') plt.legend() save_figure( os.path.join(tmp_dir, sky_bkg_table_name.replace('.fits', '.png'))) if show: plt.show() plt.close() # Subtract sky from product files for i, science_file in enumerate(sequence.science_files): for out_file in out_files: if science_file in out_file: science_file = out_file logger.info( f"Applying sky background subtraction on file {science_file}" ) with fits.open(science_file, mode='update') as hdu_list: hdu_list[0].data = hdu_list[0].data.astype( float) - weighted_sky_bkg[i] if 'VAR' in hdu_list: hdu_list['VAR'].data = hdu_list[ 'VAR'].data + weighted_sky_bkg_var[i] else: # Construct new HDU shape = np.array(hdu_list[0].data.shape)[[-2, -1]] data = np.full(shape=shape, fill_value=weighted_sky_bkg_var[i]) hdu = fits.ImageHDU(data=data, name='VAR') hdu_list.append(hdu) hdu_list[0].header.set('SKYCORR', str(datetime.now())) hdu_list[0].header.set('SKYBKG', weighted_sky_bkg[i], "Sky background") hdu_list[0].header.set('SKYVAR', weighted_sky_bkg_var[i], "Sky background variance") hdu_list.flush() elif method in ['image', 'frame']: raise NotImplementedError( "Sky subtraction in image mode is not implemented yet!") else: raise ValueError(f"Sky subtraction method {method} is not understood!")
from astropy.io import fits import numpy as np from astropy.table import Table from collections import OrderedDict data = fits.open("catalogue.fits") snData = fits.open("../line_fluxes/sn_R1000_lines_combined.fits") lineArr = ["C4_1548","C4_1551","O3_1661","O3_1666","C3_1907_C3_1910","O2_3726_O2_3729","Ne3_3869","O3_4363","HBaB_4861","O3_4959","O3_5007","HBaA_6563","N2_6584","S2_6716","S2_6731"] eMPTout1 = np.genfromtxt("output_dither_MDC_00.txt", dtype=None, names=True) eMPTout2 = np.genfromtxt("output_dither_MDC_01.txt", dtype=None, names=True) eMPTout3 = np.genfromtxt("output_dither_MDC_02.txt", dtype=None, names=True) outputDict = OrderedDict() outputDict["ID"] = snData[1].data["ID"] id_for_peter = np.zeros_like(snData[1].data["ID"])-1 for i in range(len(id_for_peter)): tempIdx = np.where(eMPTout1["ID_cat"] == snData[1].data["ID"][i])[0] if len(tempIdx) > 0: id_for_peter[i] = eMPTout1["ID_emma"][tempIdx]-1 else: tempIdx = np.where(eMPTout2["ID_cat"] == snData[1].data["ID"][i])[0] if len(tempIdx) > 0: id_for_peter[i] = eMPTout2["ID_emma"][tempIdx]-1 else: tempIdx = np.where(eMPTout3["ID_cat"] == snData[1].data["ID"][i])[0] if len(tempIdx) > 0: id_for_peter[i] = eMPTout3["ID_emma"][tempIdx]-1 outputDict["ID_for_peter"] = id_for_peter outputDict["redshift"] = data['GALAXY PROPERTIES'].data['redshift']
def buildEmptyDRZ(input, output): """ Create an empty DRZ file. This module creates an empty DRZ file in a valid FITS format so that the HST pipeline can handle the Multidrizzle zero expossure time exception where all data has been excluded from processing. Parameters ---------- input : str filename of the initial input to process_input output : str filename of the default empty _drz.fits file to be generated """ # Identify the first input image inputfile = parseinput.parseinput(input)[0] if not inputfile: print('\n******* ERROR *******', file=sys.stderr) print( 'No input file found! Check specification of parameter ' '"input". ', file=sys.stderr) print('Quitting...', file=sys.stderr) print('******* ***** *******\n', file=sys.stderr) return # raise IOError, "No input file found!" # Set up output file here... if output is None: if len(input) == 1: oname = fileutil.buildNewRootname(input[0]) else: oname = 'final' output = fileutil.buildNewRootname(oname, extn='_drz.fits') else: if '_drz' not in output: output = fileutil.buildNewRootname(output, extn='_drz.fits') print('Building emtpy DRZ file with output name: %s' % output) # Open the first image (of the excludedFileList?) to use as a template to build # the DRZ file. try : log.info('Building empty DRZ file from %s' % inputfile[0]) img = fits.open(inputfile[0], memmap=False) except: raise IOError('Unable to open file %s \n' % inputfile) # Create the fitsobject fitsobj = fits.HDUList() # Copy the primary header hdu = img[0].copy() fitsobj.append(hdu) # Modify the 'NEXTEND' keyword of the primary header to 3 for the #'sci, wht, and ctx' extensions of the newly created file. fitsobj[0].header['NEXTEND'] = 3 # Create the 'SCI' extension hdu = fits.ImageHDU(header=img['sci', 1].header.copy()) hdu.header['EXTNAME'] = 'SCI' fitsobj.append(hdu) # Create the 'WHT' extension hdu = fits.ImageHDU(header=img['sci', 1].header.copy()) hdu.header['EXTNAME'] = 'WHT' fitsobj.append(hdu) # Create the 'CTX' extension hdu = fits.ImageHDU(header=img['sci', 1].header.copy()) hdu.header['EXTNAME'] = 'CTX' fitsobj.append(hdu) # Add HISTORY comments explaining the creation of this file. fitsobj[0].header.add_history("** AstroDrizzle has created this empty " "DRZ product because**") fitsobj[0].header.add_history("** all input images were excluded from " "processing.**") # Change the filename in the primary header to reflect the name of the output # filename. fitsobj[0].header['FILENAME'] = str(output) # +"_drz.fits" # Change the ROOTNAME keyword to the ROOTNAME of the output PRODUCT fitsobj[0].header['ROOTNAME'] = str(output.split('_drz.fits')[0]) # Modify the ASN_MTYP keyword to contain "PROD-DTH" so it can be properly # ingested into the archive catalog. # stis has this keyword in the [1] header, so I am directing the code #t o first look in the primary, then the 1 try: fitsobj[0].header['ASN_MTYP'] = 'PROD-DTH' except: fitsobj[1].header['ASN_MTYP'] = 'PROD-DTH' # If the file is already on disk delete it and replace it with the # new file dirfiles = os.listdir(os.curdir) if dirfiles.count(output) > 0: os.remove(output) log.info(" Replacing %s..." % output) # Write out the empty DRZ file fitsobj.writeto(output) print(textutil.textbox( 'ERROR:\nAstroDrizzle has created an empty DRZ product because all ' 'input images were excluded from processing or a user requested the ' 'program to stop.') + '\n', file=sys.stderr) return
def load_allstar(self): self.apogee_dr = apogee_default_dr(dr=self.apogee_dr) allstarpath = allstar(dr=self.apogee_dr) hdulist = fits.open(allstarpath) print(f'Loading allStar DR{self.apogee_dr} catalog') return hdulist
def compile(self): h5name_check(self.filename) hdulist = self.load_allstar() indices = self.filter_apogeeid_list(hdulist) info = chips_pix_info(dr=self.apogee_dr) total_pix = (info[1] - info[0]) + (info[3] - info[2]) + (info[5] - info[4]) default_length = 900000 spec = np.zeros((default_length, total_pix), dtype=np.float32) spec_err = np.zeros((default_length, total_pix), dtype=np.float32) RA = np.zeros(default_length, dtype=np.float32) DEC = np.zeros(default_length, dtype=np.float32) SNR = np.zeros(default_length, dtype=np.float32) individual_flag = np.zeros(default_length, dtype=np.float32) Kmag = np.zeros(default_length, dtype=np.float32) AK_TARG = np.zeros(default_length, dtype=np.float32) # Data array teff = np.zeros(default_length, dtype=np.float32) logg = np.zeros(default_length, dtype=np.float32) MH = np.zeros(default_length, dtype=np.float32) alpha_M = np.zeros(default_length, dtype=np.float32) C = np.zeros(default_length, dtype=np.float32) C1 = np.zeros(default_length, dtype=np.float32) N = np.zeros(default_length, dtype=np.float32) O = np.zeros(default_length, dtype=np.float32) Na = np.zeros(default_length, dtype=np.float32) Mg = np.zeros(default_length, dtype=np.float32) Al = np.zeros(default_length, dtype=np.float32) Si = np.zeros(default_length, dtype=np.float32) P = np.zeros(default_length, dtype=np.float32) S = np.zeros(default_length, dtype=np.float32) K = np.zeros(default_length, dtype=np.float32) Ca = np.zeros(default_length, dtype=np.float32) Ti = np.zeros(default_length, dtype=np.float32) Ti2 = np.zeros(default_length, dtype=np.float32) V = np.zeros(default_length, dtype=np.float32) Cr = np.zeros(default_length, dtype=np.float32) Mn = np.zeros(default_length, dtype=np.float32) Fe = np.zeros(default_length, dtype=np.float32) Co = np.zeros(default_length, dtype=np.float32) Ni = np.zeros(default_length, dtype=np.float32) Cu = np.zeros(default_length, dtype=np.float32) Ge = np.zeros(default_length, dtype=np.float32) Ce = np.zeros(default_length, dtype=np.float32) Rb = np.zeros(default_length, dtype=np.float32) Y = np.zeros(default_length, dtype=np.float32) Nd = np.zeros(default_length, dtype=np.float32) parallax = np.zeros(default_length, dtype=np.float32) fakemag = np.zeros(default_length, dtype=np.float32) # Error array teff_err = np.zeros(default_length, dtype=np.float32) logg_err = np.zeros(default_length, dtype=np.float32) MH_err = np.zeros(default_length, dtype=np.float32) alpha_M_err = np.zeros(default_length, dtype=np.float32) C_err = np.zeros(default_length, dtype=np.float32) C1_err = np.zeros(default_length, dtype=np.float32) N_err = np.zeros(default_length, dtype=np.float32) O_err = np.zeros(default_length, dtype=np.float32) Na_err = np.zeros(default_length, dtype=np.float32) Mg_err = np.zeros(default_length, dtype=np.float32) Al_err = np.zeros(default_length, dtype=np.float32) Si_err = np.zeros(default_length, dtype=np.float32) P_err = np.zeros(default_length, dtype=np.float32) S_err = np.zeros(default_length, dtype=np.float32) K_err = np.zeros(default_length, dtype=np.float32) Ca_err = np.zeros(default_length, dtype=np.float32) Ti_err = np.zeros(default_length, dtype=np.float32) Ti2_err = np.zeros(default_length, dtype=np.float32) V_err = np.zeros(default_length, dtype=np.float32) Cr_err = np.zeros(default_length, dtype=np.float32) Mn_err = np.zeros(default_length, dtype=np.float32) Fe_err = np.zeros(default_length, dtype=np.float32) Co_err = np.zeros(default_length, dtype=np.float32) Ni_err = np.zeros(default_length, dtype=np.float32) Cu_err = np.zeros(default_length, dtype=np.float32) Ge_err = np.zeros(default_length, dtype=np.float32) Ce_err = np.zeros(default_length, dtype=np.float32) Rb_err = np.zeros(default_length, dtype=np.float32) Y_err = np.zeros(default_length, dtype=np.float32) Nd_err = np.zeros(default_length, dtype=np.float32) parallax_err = np.zeros(default_length, dtype=np.float32) fakemag_err = np.zeros(default_length, dtype=np.float32) array_counter = 0 start_time = time.time() # provide a cont mask so no need to read every loop if self.cont_mask is None: maskpath = os.path.join(os.path.dirname(astroNN.__path__[0]), 'astroNN', 'data', f'dr{self.apogee_dr}_contmask.npy') self.cont_mask = np.load(maskpath) for counter, index in enumerate(indices): nvisits = 1 apogee_id = hdulist[1].data['APOGEE_ID'][index] location_id = hdulist[1].data['LOCATION_ID'][index] if counter % 100 == 0: print( f'Completed {counter + 1} of {indices.shape[0]}, {(time.time() - start_time):.{2}f}s elapsed' ) if self.continuum is False: path = combined_spectra(dr=self.apogee_dr, location=location_id, apogee=apogee_id, verbose=0) if path is False: # if path is not found then we should skip continue combined_file = fits.open(path) _spec = combined_file[ 1].data # Pseudo-continuum normalized flux _spec_err = combined_file[2].data # Spectrum error array _spec = gap_delete( _spec, dr=self.apogee_dr) # Delete the gap between sensors _spec_err = gap_delete(_spec_err, dr=self.apogee_dr) inSNR = combined_file[0].header['SNR'] combined_file.close() else: path = visit_spectra(dr=self.apogee_dr, location=location_id, apogee=apogee_id, verbose=0) if path is False: # if path is not found then we should skip continue apstar_file = fits.open(path) nvisits = apstar_file[0].header['NVISITS'] if nvisits == 1: _spec = apstar_file[1].data _spec_err = apstar_file[2].data _spec_mask = apstar_file[3].data inSNR = np.ones(nvisits) inSNR[0] = apstar_file[0].header['SNR'] else: _spec = apstar_file[1].data[1:] _spec_err = apstar_file[2].data[1:] _spec_mask = apstar_file[3].data[1:] inSNR = np.ones(nvisits + 1) inSNR[0] = apstar_file[0].header['SNR'] for i in range(nvisits): inSNR[i + 1] = apstar_file[0].header[f'SNRVIS{i + 1}'] # Deal with spectra thats all zeros flux ii = 0 while ii < _spec.shape[0]: if np.count_nonzero(_spec[ii]) == 0: nvisits -= 1 _spec = np.delete(_spec, ii, 0) _spec_err = np.delete(_spec_err, ii, 0) _spec_mask = np.delete(_spec_mask, ii, 0) inSNR = np.delete(inSNR, ii, 0) ii -= 1 ii += 1 # Just for the sake of program to work, the real nvisits still nvisits nvisits += 1 # Normalize spectra and Set some bitmask to 0 _spec, _spec_err = self.apstar_normalization( _spec, _spec_err, _spec_mask) apstar_file.close() if nvisits == 1: individual_flag[array_counter:array_counter + nvisits] = 0 else: individual_flag[array_counter:array_counter + 1] = 0 individual_flag[array_counter + 1:array_counter + nvisits] = 1 spec[array_counter:array_counter + nvisits, :] = _spec spec_err[array_counter:array_counter + nvisits, :] = _spec_err SNR[array_counter:array_counter + nvisits] = inSNR RA[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['RA'][index], nvisits) DEC[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['DEC'][index], nvisits) parallax[array_counter:array_counter + nvisits] = np.tile( -9999, nvisits) parallax_err[array_counter:array_counter + nvisits] = np.tile( -9999, nvisits) fakemag[array_counter:array_counter + nvisits] = np.tile( -9999, nvisits) fakemag_err[array_counter:array_counter + nvisits] = np.tile( -9999, nvisits) Kmag[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['K'][index], nvisits) AK_TARG[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['AK_TARG'][index], nvisits) if self.spectra_only is not True: teff[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['PARAM'][index, 0], nvisits) logg[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['PARAM'][index, 1], nvisits) MH[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['PARAM'][index, 3], nvisits) alpha_M[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['PARAM'][index, 6], nvisits) C[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 0], nvisits) C1[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 1], nvisits) N[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 2], nvisits) O[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 3], nvisits) Na[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 4], nvisits) Mg[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 5], nvisits) Al[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 6], nvisits) Si[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 7], nvisits) P[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 8], nvisits) S[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 9], nvisits) K[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 10], nvisits) Ca[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 11], nvisits) Ti[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 12], nvisits) Ti2[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 13], nvisits) V[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 14], nvisits) Cr[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 15], nvisits) Mn[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 16], nvisits) Fe[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 17], nvisits) Co[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 18], nvisits) Ni[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 19], nvisits) Cu[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 20], nvisits) Ge[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 21], nvisits) Ce[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 22], nvisits) Rb[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 23], nvisits) Y[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 24], nvisits) Nd[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H'][index, 25], nvisits) if self.use_err is True: teff_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['TEFF_ERR'][index], nvisits) logg_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['LOGG_ERR'][index], nvisits) MH_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['M_H_ERR'][index], nvisits) alpha_M_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['ALPHA_M_ERR'][index], nvisits) C_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 0], nvisits) C1_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 1], nvisits) N_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 2], nvisits) O_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 3], nvisits) Na_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 4], nvisits) Mg_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 5], nvisits) Al_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 6], nvisits) Si_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 7], nvisits) P_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 8], nvisits) S_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 9], nvisits) K_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 10], nvisits) Ca_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 11], nvisits) Ti_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 12], nvisits) Ti2_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 13], nvisits) V_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 14], nvisits) Cr_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 15], nvisits) Mn_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 16], nvisits) Fe_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 17], nvisits) Co_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 18], nvisits) Ni_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 19], nvisits) Cu_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 20], nvisits) Ge_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 21], nvisits) Ce_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 22], nvisits) Rb_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 23], nvisits) Y_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 24], nvisits) Nd_err[array_counter:array_counter + nvisits] = np.tile( hdulist[1].data['X_H_ERR'][index, 25], nvisits) array_counter += nvisits spec = spec[0:array_counter] spec_err = spec_err[0:array_counter] individual_flag = individual_flag[0:array_counter] RA = RA[0:array_counter] DEC = DEC[0:array_counter] SNR = SNR[0:array_counter] if self.spectra_only is not True: teff = teff[0:array_counter] logg = logg[0:array_counter] Kmag = Kmag[0:array_counter] AK_TARG = AK_TARG[0:array_counter] MH = MH[0:array_counter] alpha_M = alpha_M[0:array_counter] C = C[0:array_counter] C1 = C1[0:array_counter] N = N[0:array_counter] O = O[0:array_counter] Na = Na[0:array_counter] Mg = Mg[0:array_counter] Al = Al[0:array_counter] Si = Si[0:array_counter] P = P[0:array_counter] S = S[0:array_counter] K = K[0:array_counter] Ca = Ca[0:array_counter] Ti = Ti[0:array_counter] Ti2 = Ti2[0:array_counter] V = V[0:array_counter] Cr = Cr[0:array_counter] Mn = Mn[0:array_counter] Fe = Fe[0:array_counter] Co = Co[0:array_counter] Ni = Ni[0:array_counter] Cu = Cu[0:array_counter] Ge = Ge[0:array_counter] Ce = Ce[0:array_counter] Rb = Rb[0:array_counter] Y = Y[0:array_counter] Nd = Nd[0:array_counter] parallax = parallax[0:array_counter] fakemag = fakemag[0:array_counter] teff_err = teff_err[0:array_counter] logg_err = logg_err[0:array_counter] MH_err = MH_err[0:array_counter] alpha_M_err = alpha_M_err[0:array_counter] C_err = C_err[0:array_counter] C1_err = C1_err[0:array_counter] N_err = N_err[0:array_counter] O_err = O_err[0:array_counter] Na_err = Na_err[0:array_counter] Mg_err = Mg_err[0:array_counter] Al_err = Al_err[0:array_counter] Si_err = Si_err[0:array_counter] P_err = P_err[0:array_counter] S_err = S_err[0:array_counter] K_err = K_err[0:array_counter] Ca_err = Ca_err[0:array_counter] Ti_err = Ti_err[0:array_counter] Ti2_err = Ti2_err[0:array_counter] V_err = V_err[0:array_counter] Cr_err = Cr_err[0:array_counter] Mn_err = Mn_err[0:array_counter] Fe_err = Fe_err[0:array_counter] Co_err = Co_err[0:array_counter] Ni_err = Ni_err[0:array_counter] Cu_err = Cu_err[0:array_counter] Ge_err = Ge_err[0:array_counter] Ce_err = Ce_err[0:array_counter] Rb_err = Rb_err[0:array_counter] Y_err = Y_err[0:array_counter] Nd_err = Nd_err[0:array_counter] parallax_err = parallax_err[0:array_counter] fakemag_err = fakemag_err[0:array_counter] if self.use_esa_gaia is True: gaia_ra, gaia_dec, gaia_parallax, gaia_err = gaiadr2_parallax( cuts=True, keepdims=False) m1, m2, sep = xmatch(RA, gaia_ra, maxdist=2, colRA1=RA, colDec1=DEC, colRA2=gaia_ra, colDec2=gaia_dec, swap=False) parallax[m1] = gaia_parallax[m2] parallax_err[m1] = gaia_err[m2] fakemag[m1], fakemag_err[m1] = mag_to_fakemag( extinction_correction(Kmag[m1], AK_TARG[m1]), parallax[m1], parallax_err[m1]) elif self.use_anderson_2017 is True: gaia_ra, gaia_dec, gaia_parallax, gaia_err = anderson_2017_parallax( ) m1, m2, sep = xmatch(RA, gaia_ra, maxdist=2, colRA1=RA, colDec1=DEC, epoch1=2000., colRA2=gaia_ra, colDec2=gaia_dec, epoch2=2000., swap=False) parallax[m1] = gaia_parallax[m2] parallax_err[m1] = gaia_err[m2] fakemag[m1], fakemag_err[m1] = mag_to_fakemag( extinction_correction(Kmag[m1], AK_TARG[m1]), parallax[m1], parallax_err[m1]) print(f'Creating {self.filename}.h5') h5f = h5py.File(f'{self.filename}.h5', 'w') h5f.create_dataset('spectra', data=spec) h5f.create_dataset('spectra_err', data=spec_err) h5f.create_dataset('in_flag', data=individual_flag) h5f.create_dataset('index', data=indices) if self.spectra_only is not True: h5f.create_dataset('SNR', data=SNR) h5f.create_dataset('RA', data=RA) h5f.create_dataset('DEC', data=DEC) h5f.create_dataset('Kmag', data=Kmag) h5f.create_dataset('AK_TARG', data=AK_TARG) h5f.create_dataset('teff', data=teff) h5f.create_dataset('logg', data=logg) h5f.create_dataset('M', data=MH) h5f.create_dataset('alpha', data=alpha_M) h5f.create_dataset('C', data=C) h5f.create_dataset('C1', data=C1) h5f.create_dataset('N', data=N) h5f.create_dataset('O', data=O) h5f.create_dataset('Na', data=Na) h5f.create_dataset('Mg', data=Mg) h5f.create_dataset('Al', data=Al) h5f.create_dataset('Si', data=Si) h5f.create_dataset('P', data=P) h5f.create_dataset('S', data=S) h5f.create_dataset('K', data=K) h5f.create_dataset('Ca', data=Ca) h5f.create_dataset('Ti', data=Ti) h5f.create_dataset('Ti2', data=Ti2) h5f.create_dataset('V', data=V) h5f.create_dataset('Cr', data=Cr) h5f.create_dataset('Mn', data=Mn) h5f.create_dataset('Fe', data=Fe) h5f.create_dataset('Co', data=Co) h5f.create_dataset('Ni', data=Ni) h5f.create_dataset('Cu', data=Cu) h5f.create_dataset('Ge', data=Ge) h5f.create_dataset('Ce', data=Ce) h5f.create_dataset('Rb', data=Rb) h5f.create_dataset('Y', data=Y) h5f.create_dataset('Nd', data=Nd) h5f.create_dataset('parallax', data=parallax) h5f.create_dataset('fakemag', data=fakemag) if self.use_err is True: h5f.create_dataset('AK_TARG_err', data=np.zeros_like(AK_TARG)) h5f.create_dataset('teff_err', data=teff_err) h5f.create_dataset('logg_err', data=logg_err) h5f.create_dataset('M_err', data=MH_err) h5f.create_dataset('alpha_err', data=alpha_M_err) h5f.create_dataset('C_err', data=C_err) h5f.create_dataset('C1_err', data=C1_err) h5f.create_dataset('N_err', data=N_err) h5f.create_dataset('O_err', data=O_err) h5f.create_dataset('Na_err', data=Na_err) h5f.create_dataset('Mg_err', data=Mg_err) h5f.create_dataset('Al_err', data=Al_err) h5f.create_dataset('Si_err', data=Si_err) h5f.create_dataset('P_err', data=P_err) h5f.create_dataset('S_err', data=S_err) h5f.create_dataset('K_err', data=K_err) h5f.create_dataset('Ca_err', data=Ca_err) h5f.create_dataset('Ti_err', data=Ti_err) h5f.create_dataset('Ti2_err', data=Ti2_err) h5f.create_dataset('V_err', data=V_err) h5f.create_dataset('Cr_err', data=Cr_err) h5f.create_dataset('Mn_err', data=Mn_err) h5f.create_dataset('Fe_err', data=Fe_err) h5f.create_dataset('Co_err', data=Co_err) h5f.create_dataset('Ni_err', data=Ni_err) h5f.create_dataset('Cu_err', data=Cu_err) h5f.create_dataset('Ge_err', data=Ge_err) h5f.create_dataset('Ce_err', data=Ce_err) h5f.create_dataset('Rb_err', data=Rb_err) h5f.create_dataset('Y_err', data=Y_err) h5f.create_dataset('Nd_err', data=Nd_err) h5f.create_dataset('parallax_err', data=parallax_err) h5f.create_dataset('fakemag_err', data=fakemag_err) h5f.close() print(f'Successfully created {self.filename}.h5 in {currentdir}')
def _generator(self, return_type, save_with_name="", save_location='', clobber=False, overwrite=False, do_not_scale_image_data=True, return_fname=False, ccd_kwargs=None, **kwd): """ Generator that yields each {name} in the collection. If any of the parameters ``save_with_name``, ``save_location`` or ``overwrite`` evaluates to ``True`` the generator will write a copy of each FITS file it is iterating over. In other words, if ``save_with_name`` and/or ``save_location`` is a string with non-zero length, and/or ``overwrite`` is ``True``, a copy of each FITS file will be made. Parameters ---------- save_with_name : str, optional string added to end of file name (before extension) if FITS file should be saved after iteration. Unless ``save_location`` is set, files will be saved to location of the source files ``self.location``. Default is ``''``. save_location : str, optional Directory in which to save FITS files; implies that FITS files will be saved. Note this provides an easy way to copy a directory of files--loop over the {name} with ``save_location`` set. Default is ``''``. overwrite : bool, optional If ``True``, overwrite input FITS files. Default is ``False``. clobber : bool, optional Alias for ``overwrite``. Default is ``False``. do_not_scale_image_data : bool, optional If ``True``, prevents fits from scaling images. Default is ``{default_scaling}``. Default is ``True``. return_fname : bool, optional If True, return the tuple (header, file_name) instead of just header. The file name returned is the name of the file only, not the full path to the file. Default is ``False``. ccd_kwargs : dict, optional Dict with parameters for `~astropy.nddata.fits_ccddata_reader`. For instance, the key ``'unit'`` can be used to specify the unit of the data. If ``'unit'`` is not given then ``'adu'`` is used as the default unit. See `~astropy.nddata.fits_ccddata_reader` for a complete list of parameters that can be passed through ``ccd_kwargs``. **kwd : Any additional keywords are used to filter the items returned; see `files_filtered` examples for details. Returns ------- `{return_type}` If ``return_fname`` is ``False``, yield the next {name} in the collection. (`{return_type}`, str) If ``return_fname`` is ``True``, yield a tuple of ({name}, ``file name``) for the next item in the collection. """ # store mask so we can reset at end--must COPY, otherwise # current_mask just points to the mask of summary if not self.summary: return current_mask = {} for col in self.summary.columns: current_mask[col] = self.summary[col].mask if kwd: self._find_keywords_by_values(**kwd) ccd_kwargs = ccd_kwargs or {} for full_path in self._paths(): add_kwargs = {'do_not_scale_image_data': do_not_scale_image_data} # We need to open the file here, get the appropriate values and then # close it again before it "yields" otherwise it's not garantueed # that the generator actually advances and closes the file again. # For example if one uses "next" on the generator manually the # file handle could "leak". if return_type == 'header': return_thing = fits.getheader(full_path, self.ext) elif return_type == 'data': return_thing = fits.getdata(full_path, self.ext, **add_kwargs) elif return_type == 'ccd': return_thing = fits_ccddata_reader(full_path, hdu=self.ext, **ccd_kwargs) elif return_type == 'hdu': with fits.open(full_path, **add_kwargs) as hdulist: ext_index = hdulist.index_of(self.ext) # Need to copy the HDU to prevent lazy loading problems # and "IO operations on closed file" errors return_thing = hdulist[ext_index].copy() else: raise ValueError('no generator for {}'.format(return_type)) file_name = path.basename(full_path) if return_fname: yield return_thing, file_name else: yield return_thing if save_location: destination_dir = save_location else: destination_dir = path.dirname(full_path) basename = path.basename(full_path) if save_with_name: base, ext = path.splitext(basename) basename = base + save_with_name + ext new_path = path.join(destination_dir, basename) # I really should have called the option overwrite from # the beginning. The hack below ensures old code works, # at least... if clobber or overwrite: if _ASTROPY_LT_1_3: nuke_existing = {'clobber': True} else: nuke_existing = {'overwrite': True} else: nuke_existing = {} if return_type == 'ccd': pass elif (new_path != full_path) or nuke_existing: with fits.open(full_path, **add_kwargs) as hdulist: ext_index = hdulist.index_of(self.ext) if return_type == 'hdu': hdulist[ext_index] = return_thing elif return_type == 'data': hdulist[ext_index].data = return_thing elif return_type == 'header': hdulist[ext_index].header = return_thing try: hdulist.writeto(new_path, **nuke_existing) except IOError: logger.error('error writing file %s', new_path) raise # reset mask for col in self.summary.columns: self.summary[col].mask = current_mask[col]
def sdss_score(flist, **kwargs): """Score a list of imaging fields from zero to one. """ import time import numpy as np from astropy import log from astropy.io import fits from warnings import warn from ..sdssio import sdss_name, sdss_calib from ...pydlutils.sdss import sdss_flagval if 'silent' not in kwargs: kwargs['silent'] = True lat = 32.780361 lng = 360.0 - 105.820417 tzone = 7 scores = 1 # # Read the PHOTO status bits # if not kwargs['silent']: log.info('Setting PHOTO status bits') t1 = time.time() nlist = flist[1].header.get('NAXIS2') fdata = flist[1].data for k in range(nlist): if not kwargs['silent'] and ((k % 1000) == 0): log.info("Setting PHOTO status {0:d} {1:d}".format(k, nlist)) thisfile = sdss_name('fpFieldStat', fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k]) try: fpfield = fits.open(thisfile) except IOError: warn("Retrying fpFieldStat file for RUN = {0:d} CAMCOL = {1:d} FIELD = {2:d} RERUN = {3}".format(fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k])) try: fpfield = fits.open(thisfile) except IOError: warn("Bad fpFieldStat file for RUN = {0:d} CAMCOL = {1:d} FIELD = {2:d} RERUN = {3}".format(fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k])) fdata.field('PHOTO_STATUS')[k] = -1 if not kwargs['silent']: log.info('Trying tsField instead.') thisfile = sdss_name('tsField', fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k]) try: tsfield = fits.open(thisfile) except IOError: warn('Bad tsField file.') else: if not kwargs['silent']: log.info('tsField found, using frames_status.') fdata.field('PHOTO_STATUS')[k] = tsfield[1].data.field('frames_status')[0] else: fdata.field('PHOTO_STATUS')[k] = fpfield[1].data.field('status')[0] else: fdata.field('PHOTO_STATUS')[k] = fpfield[1].data.field('status')[0] if not kwargs['silent']: log.info("Time to set PHOTO status = {0:f} sec".format(time.time()-t1)) # # Read in the PSP status # if not kwargs['silent']: log.info('Setting PSP status bits') t2 = time.time() for k in range(nlist): if not kwargs['silent'] and ((k % 1000) == 0): log.info("Setting PSP status {0:d} {1:d}".format(k, nlist)) thisfile = sdss_name('psField', fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k]) try: psfield = fits.open(thisfile) except IOError: warn("Retrying psField file for RUN = {0:d} CAMCOL = {1:d} FIELD = {2:d} RERUN = {3}".format(fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k])) try: psfield = fits.open(thisfile) except IOError: warn("Bad psField file for RUN = {0:d} CAMCOL = {1:d} FIELD = {2:d} RERUN = {3}".format(fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k])) fdata.field('PSP_STATUS')[k] = -1 fdata.field('PSF_FWHM')[k] = -1 fdata.field('SKYFLUX')[k] = -1 pixscale = 0.396 * np.sqrt(fdata.field('XBIN')[k]**2 + fdata.field('YBIN')[k]**2)/np.sqrt(2.0) calibinfo = sdss_calib(fdata.field('RUN')[k], fdata.field('CAMCOL')[k], fdata.field('FIELD')[k], fdata.field('RERUN')[k], **kwargs) fdata.field('PSP_STATUS')[k] = psfield[6].data.field('status')[0] fdata.field('PSF_FWHM')[k] = psfield[6].data.field('psf_width')[0] fdata.field('SKYFLUX')[k] = (psfield[6].data.field('sky')[0] * calibinfo['NMGYPERCOUNT']/pixscale**2) if not kwargs['silent']: log.info("Time to set PSP status = {0:f} sec".format(time.time()-t2)) # # Decide if each field exists in all 5 bands. # bad_bits = sdss_flagval('image_status', ['bad_rotator', 'bad_astrom', 'bad_focus', 'shutters']) if 'ignoreframesstatus' in kwargs: ignoreframesstatus = np.zeros(fdata.field('PHOTO_STATUS').shape) == 0 else: ignoreframesstatus = np.zeros(fdata.field('PHOTO_STATUS').shape) == 1 qexist = (fdata.field('PHOTO_STATUS') == 0) | ignoreframesstatus for k in range(5): qexist &= (fdata.field('IMAGE_STATUS')[:, k] & bad_bits) == 0 # # Decide if each field is phtometric in all 5 bands. # unphot_bits = sdss_flagval('image_status', ['cloudy', 'unknown', 'ff_petals', 'dead_ccd', 'noisy_ccd']) qphot = fdata.field('SUN_ANGLE') < -12 for k in range(5): qphot &= (fdata.field('IMAGE_STATUS')[:, k] & unphot_bits) == 0 for k in range(5): qphot &= (((fdata.field('PSP_STATUS')[:, k] & 31) <= 2) | (fdata.field('XBIN') > 1) | ignoreframesstatus) # # Now set the score for each field # sensitivity = (0.7 / (fdata.field('PSF_FWHM')[:, 2] * np.sqrt(fdata.field('SKYFLUX')[:, 2]))) < 0.4 fdata.field('SCORE')[:] = qexist * (0.1 + 0.5*qphot + sensitivity) ibinned = np.find(fdata.field('XBIN') > 1) if len(ibinned) > 0: fdata.field('SCORE')[ibinned] *= 0.1 # # Look for any NaN values, which could happen for example if there # is a corrupted psField file and PSF_FWHM or SKYFLUX was negative. # ibad = np.find(~np.isfinite(fdata.field('SCORE'))) if len(ibad) > 0: warn("Changing NaN scores for {0:d} fields to zero.".format(len(ibad))) fdata.field('SCORE')[ibad] = 0 return fdata.field('SCORE')
#!/usr/bin/env python # -*- coding: utf8 -*- #import ds9 # doesn't bloody work import numpy as np from astropy.io import fits import pyregion import pyregion._region_filter as filter # define some SPIRE-typical constants beam = [465, 822, 1769] # beam area, in arcsec^2 ebar = [1.0432705,1.0521442,1.1025926] # effective beam area ratio (beta=2 SPIRE UM, table 5.8, page 96; see also M. Smith email, 16/04/14) cnoise = [0.0058,0.0063,0.0068] # confusion noise, in Jy/beam, as from Nguyen et al. 2010 kk = 2.*np.pi/(360.*3600.) # 1 arcsec in rad img = raw_input('Name of the SPIRE data: ') reg_name = raw_input('Name of the DS9 file containing the regions for flux and sky measurements: ') hdulist = fits.open(img) wavel = input('Wavelength of the image: ') if wavel == 250: wind = 0 if wavel == 350: wind = 1 if wavel == 500: wind = 2 header = hdulist[1].header fluxmap = hdulist[1].data errmap = hdulist[2].data nx = header["NAXIS1"] ny = header["NAXIS2"] size = (ny, nx) # this is the size of the image pixsz = np.abs(header["CDELT1"])*3600. ## converts from MJy/sr to Jy/pix
def create_product_page(prodname, zoom_size=128, wcsname="", gcolor='magenta', fsize=8): """Create a matplotlib Figure() object which summarizes this product FITS file.""" # obtain image data to display with fits.open(prodname) as prod: data = prod[1].data phdr = prod[0].header if 'texptime' not in phdr: return None, None targname = phdr['targname'] inst = phdr['instrume'] det = phdr['detector'] texptime = phdr['texptime'] inexp = phdr['d001data'].split('[')[0] wcstype = prod[1].header['wcstype'] wcs = wcsutil.HSTWCS(prod, ext=1) hdrtab = prod['hdrtab'].data filters = ';'.join([phdr[f] for f in phdr['filter*']]) dateobs = phdr['date-obs'] # human-readable date expstart = phdr['expstart'] # MJD float value asnid = phdr.get('asn_id', '') center = (data.shape[0] // 2, data.shape[1] // 2) prod_path = os.path.split(prodname)[0] data = np.nan_to_num(data, nan=0.0) # Get GAIA catalog refcat = amutils.create_astrometric_catalog(prodname, existing_wcs=wcs, output=None) refx, refy = wcs.all_world2pix(refcat['RA'], refcat['DEC'], 0) # Remove points outside the full-size image area rx = [] ry = [] zx = [] zy = [] for x, y in zip(refx, refy): if 0 < x < data.shape[1] and 0 < y < data.shape[0]: rx.append(x) ry.append(y) if -zoom_size < x - center[1] < zoom_size and -zoom_size < y - center[0] < zoom_size: zx.append(x - center[1] + zoom_size) zy.append(y - center[0] + zoom_size) # Define subplot regions on page fig = plt.figure(constrained_layout=True, figsize=(8.5, 11)) gs = fig.add_gridspec(ncols=4, nrows=5) # title plots rootname = os.path.basename(prodname) img_title = "{} image of {} with WCSNAME={}".format(rootname, targname, wcsname) fig.suptitle(img_title, ha='center', va='top', fontsize=fsize) # Define image display fig_img = fig.add_subplot(gs[:3, :]) fig_zoom = fig.add_subplot(gs[3:, 2:]) fig_summary = fig.add_subplot(gs[3:, :2]) # Compute display range dmax = (data.max() // 10) if data.max() <= 1000. else 100 dscaled = np.log10(np.clip(data, -0.1, dmax) + 0.10001) # identify zoom region around center of data zoom = dscaled[center[0] - zoom_size:center[0] + zoom_size, center[1] - zoom_size:center[1] + zoom_size] # display full image fig_img.imshow(dscaled, cmap='gray', origin='lower') # display zoomed section fig_zoom.imshow(zoom, cmap='gray', origin='lower') # define markerstyle mstyle = markers.MarkerStyle(marker='o') mstyle.set_fillstyle('none') # plot GAIA sources onto full size image fig_img.scatter(rx, ry, marker=mstyle, alpha=0.35, c=gcolor, s=3) fig_zoom.scatter(zx, zy, marker=mstyle, alpha=0.35, c=gcolor) # Print summary info pname = os.path.split(prodname)[1] fig_summary.text(0.01, 0.95, "Summary for {}".format(pname), fontsize=fsize) fig_summary.text(0.01, 0.9, "WCSNAME: {}".format(wcsname), fontsize=fsize) fig_summary.text(0.01, 0.85, "TARGET: {}".format(targname), fontsize=fsize) fig_summary.text(0.01, 0.8, "Instrument: {}/{}".format(inst, det), fontsize=fsize) fig_summary.text(0.01, 0.75, "Filters: {}".format(filters), fontsize=fsize) fig_summary.text(0.01, 0.7, "Total Exptime: {}".format(texptime), fontsize=fsize) fig_summary.text(0.01, 0.65, "WCSTYPE: {}".format(wcstype), fontsize=fsize) fig_summary.text(0.01, 0.5, "Total # of GAIA sources: {}".format(len(refx)), fontsize=fsize) fig_summary.text(0.01, 0.45, "# of GAIA matches: {}".format(len(rx)), fontsize=fsize) # Get extended information about observation hdrtab_cols = hdrtab.columns.names mtflag = get_col_val(hdrtab, 'mtflag', default="") gyromode = get_col_val(hdrtab, 'gyromode', default='N/A') # populate JSON summary info summary = dict(wcsname=wcsname, targname=targname, asnid=asnid, dateobs=dateobs, expstart=expstart, instrument=(inst, det), exptime=texptime, wcstype=wcstype, num_gaia=len(refx), filters=filters, rms_ra=-1, rms_dec=-1, nmatch=-1, catalog="") obs_kws = ['gyromode', 'fgslock', 'aperture', 'mtflag', 'subarray', 'obstype', 'obsmode', 'scan_typ', 'photmode'] for kw in obs_kws: summary[kw] = get_col_val(hdrtab, kw, default="") if 'FIT' in wcsname: # Look for FIT RMS and other stats from headerlet exp = fits.open(os.path.join(prod_path, inexp)) for ext in exp: if 'extname' in ext.header and ext.header['extname'] == 'HDRLET' \ and ext.header['wcsname'] == wcsname: hdrlet = ext.headerlet rms_ra = hdrlet[0].header['rms_ra'] rms_dec = hdrlet[0].header['rms_dec'] nmatch = hdrlet[0].header['nmatch'] catalog = hdrlet[0].header['catalog'] fit_vals = dict(rms_ra=rms_ra, rms_dec=rms_dec, nmatch=nmatch, catalog=catalog) summary.update(fit_vals) break exp.close() try: fig_summary.text(0.01, 0.4, "RMS: RA={:0.3f}mas, DEC={:0.3f}mas".format(rms_ra, rms_dec), fontsize=fsize) fig_summary.text(0.01, 0.35, "# matches: {}".format(nmatch), fontsize=fsize) fig_summary.text(0.01, 0.3, "Matched to {} catalog".format(catalog), fontsize=fsize) except: fig_summary.text(0.01, 0.35, "No MATCH to GAIA") print("Data without a match to GAIA: {},{}".format(inexp, wcsname)) return fig, summary
if man_dec is not None: man_dec = float(man_dec) man_flipx = args.flipx man_flipy = args.flipy man_rotate = float(args.rotate) man_target = args.target keep_wcs = args.keep_wcs telescope = args.telescope filenames = args.images # read telescope information from fits headers instruments = [] for filename in filenames: try: hdulist = fits.open(filename, verify='ignore', ignore_missing_end=True) except IOError: raise IOError('File %s does not exist! Abort.' % filename) header = hdulist[0].header for key in _pp_conf.instrument_keys: if key in header: instruments.append(header[key]) hdulist.close() if telescope is None: try: telescope = _pp_conf.instrument_identifiers[instruments[0]] except: print('cannot identify telescope/instrument; '
def ingest(filename): # create path and filename formatter if hasattr(settings, 'PATH_FORMATTER') and settings.PATH_FORMATTER is not None: path_fmt = FilenameFormatter(settings.PATH_FORMATTER) else: raise ValueError('No path formatter configured.') filename_fmt = None if hasattr(settings, 'FILENAME_FORMATTER' ) and settings.FILENAME_FORMATTER is not None: filename_fmt = FilenameFormatter(settings.FILENAME_FORMATTER) # get archive root root = settings.ARCHIVE_ROOT # open file log.info('Opening new file to ingest...') fits_file = fits.open(filename) # get path for archive path = path_fmt(fits_file['SCI'].header) # get filename for archive if isinstance(filename_fmt, FilenameFormatter): name = filename_fmt(fits_file['SCI'].header) else: tmp = os.path.basename(fits_file['SCI'].header['FNAME']) name = tmp[:tmp.find('.')] if '.' in tmp else tmp log.info('Formatted filename to %s.', name) # create new filename and set it in header out_filename = name + '.fits.fz' fits_file['SCI'].header['FNAME'] = name # find or create image if Frame.objects.filter(basename=name).exists(): img = Frame.objects.get(basename=name) else: img = Frame(basename=name) # set headers img.path = path img.add_fits_header(fits_file['SCI'].header) # write to database log.info('Writing to database...') img.save() # link related img.link_related(fits_file['SCI'].header) # create path if necessary file_path = os.path.join(root, path) if not os.path.exists(file_path): os.makedirs(file_path) # write FITS file to byte stream and close with io.BytesIO() as bio: log.info('Writing file...') fits_file.writeto(bio) fits_file.close() # pipe data into fpack log.info('Fpacking file...') proc = subprocess.Popen(['/usr/bin/fpack', '-S', '-'], stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=open( os.path.join(file_path, out_filename), 'wb')) proc.communicate(bytes(bio.getbuffer())) # all good store it if proc.returncode == 0: log.info('Stored image as %s...', img.basename) return img.basename else: raise ValueError('Could not fpack file %s.' % filename)
import numpy as np import math import matplotlib from astropy.io import fits #Import image source positions: y, x = np.loadtxt("ssa22_XY.reg", skiprows=0, unpack = True) segmap = '/Users/Brett/Desktop/Research/ssa22/Code/check.fits' image = '/Users/Brett/Desktop/Research/ssa22/Code/Mosaic_North2_drz_sci.fits' hdulist_seg = fits.open(segmap) hdulist_img = fits.open(image) seg_data = hdulist_seg[0].data img_data = hdulist_img[0].data n1 = len(x) M=0 for i in xrange(n1): source = seg_data[x[i],y[i]] if source != 0: here = np.where( seg_data == source) flux = img_data[here]
def prepare(filenames, obsparam, header_update, keep_wcs=False, flipx=False, flipy=False, rotate=0, man_ra=None, man_dec=None, diagnostics=False, display=False): """ prepare wrapper output: diagnostic properties """ # start logging logging.info('preparing data with parameters: %s' % (', '.join([('%s: %s' % (var, str(val))) for var, val in list(locals().items())]))) # change FITS file extensions to .fits for idx, filename in enumerate(filenames): if filename.split('.')[-1] in ['fts', 'FTS', 'FITS', 'fit', 'FIT']: os.rename(filename, '.'.join(filename.split('.')[:-1]) + '.fits') filenames[idx] = '.'.join(filename.split('.')[:-1]) + '.fits' logging.info('change filename from "%s" to "%s"' % (filename, filenames[idx])) # identify keywords for GENERIC telescopes # open one sample image file hdulist = fits.open(filenames[0], verify='ignore', ignore_missing_end='True') header = hdulist[0].header # check if this is a single-extension FITS file if float(header['NAXIS']) > 2.: logging.error('This is not a single-extension FITS file. Please ' 'extract individual extensions and run them ' 'individually.') sys.exit() # keywords that have to be implanted into each image implants = {} # if GENERIC telescope, ask user for header keywords if obsparam['telescope_keyword'] is 'GENERIC': keywords = { 'pixel scale in arcsec/px before binning': 'secpix', 'binning factor in both axes': 'binning', 'image center RA (keyword or degrees)': 'ra', 'image center DEC (keyword or degrees)': 'dec', 'filter used (clear if none was used)': 'filter', 'observation midtime': 'date_keyword', 'exposure time (seconds)': 'exptime' } for description, keyword in list(keywords.items()): try: if obsparam[keyword] in header: continue except: pass inp = input('%s? > ' % description) if keyword is 'secpix': obsparam[keyword] = (float(inp), float(inp)) if keyword is 'binning': implants['BINX'] = (float(inp), 'PP: user-defined') implants['BINY'] = (float(inp), 'PP: user-defined') if keyword is 'ra': try: implants['OBJCTRA'] = (float(inp), 'PP: user_defined') obsparam['radec_separator'] = 'XXX' except TypeError: obsparam['ra'] = inp # # check for separator # try: # dummy = float(header[inp]) # obsparam['radec_separator'] = 'XXX' # except ValueError: # if ':' in header[inp]: # obsparam['radec_separator'] = ':' # if ' ' in header[inp].strip(): # obsparam['radec_separator'] = ' ' if keyword is 'dec': try: implants['OBJCTDEC'] = (float(inp), 'PP: user_defined') obsparam['radec_separator'] = 'XXX' except TypeError: obsparam['dec'] = inp if keyword is 'filter': implants[obsparam['filter']] = (inp, 'PP: user-defined') if keyword is 'date_keyword': obsparam['date_keyword'] = inp if keyword is 'exptime': implants['EXPTIME'] = (float(inp), 'PP: user-defined') implants['INSTRUME'] = ('GENERIC', 'PP: manually set') # prepare image headers for photometry pipeline for filename in filenames: if display: print('preparing', filename) # open image file hdulist = fits.open(filename, mode='update', verify='silentfix', ignore_missing_end=True) header = hdulist[0].header # add other headers, if available if len(hdulist) > 1: for i in range(len(hdulist)): try: header += hdulist[i].header except: pass # account for flips and rotation in telescope configuration # if instrument has several chips... if 'chip_id' in obsparam: chip_id = header[obsparam['chip_id']] this_flipx = obsparam['flipx'][chip_id] this_flipy = obsparam['flipy'][chip_id] this_rotate = obsparam['rotate'][chip_id] # if not... else: this_flipx = obsparam['flipx'] this_flipy = obsparam['flipy'] this_rotate = obsparam['rotate'] if flipx: this_flipx = numpy.invert(this_flipx) if flipy: this_flipy = numpy.invert(this_flipy) if rotate > 0: this_rotate += rotate # read image data imdata = hdulist[0].data # check if image is a cube, or a single frame put into a cube if len(imdata.shape) > 2: # this image is a cube if imdata.shape[0] == 1: # this is a single image put into a cube # turn this into a single-frame fits file imdata = imdata[0] else: # this is really a cube; don't know what to do raise TypeError(("%s is a cube FITS file; don't know how to " "handle this file...") % filename) # add header keywords for Source Extractor if 'EPOCH' not in header: header['EPOCH'] = (2000, 'PP: required for registration') # if 'EQUINOX' not in header: # header['EQUINOX'] = (2000, 'PP: required for registration') # add header keywords for SCAMP header['PHOTFLAG'] = ('F', 'PP: data is not photometric (SCAMP)') header['PHOT_K'] = (0.05, 'PP: assumed extinction coefficient') if not keep_wcs: # remove keywords that might collide with fake wcs for key in list(header.keys()): if re.match('^CD[1,2]_[1,2]', key) is not None: # if key not in obsparam.values(): # header.remove(key) if not toolbox.if_val_in_dict(key, obsparam): header.remove(key) elif 'PV' in key and '_' in key: header.remove(key) elif key in [ 'CTYPE1', 'CRPIX1', 'CRVAL1', 'CROTA1', 'CROTA2', 'CFINT1', 'CTYPE2', 'CRPIX2', 'CRVAL2', 'CFINT2', 'LTM1_1', 'LTM2_2', 'WAT0_001', 'LTV1', 'LTV2', 'PIXXMIT', 'PIXOFFST', 'PC1_1', 'PC1_2', 'PC2_1', 'PC2_2', #'CUNIT1', 'CUNIT2', 'A_ORDER', 'A_0_0', 'A_0_1', 'A_0_2', 'A_1_0', 'A_1_1', 'A_2_0', 'B_ORDER', 'B_0_0', 'B_0_1', 'B_0_2', 'B_1_0', 'B_1_1', 'B_2_0', 'AP_ORDER', 'AP_0_0', 'AP_0_1', 'AP_0_2', 'AP_1_0', 'AP_1_1', 'AP_2_0', 'BP_ORDER', 'BP_0_0', 'BP_0_1', 'BP_0_2', 'BP_1_0', 'BP_1_1', 'BP_2_0', 'CDELT1', 'CDELT2', 'CRDELT1', 'CRDELT2' ]: if not toolbox.if_val_in_dict(key, obsparam): header.remove(key) # normalize CUNIT keywords try: if 'degree' in header['CUNIT1'].lower(): header['CUNIT1'] = ('deg') if 'degree' in header['CUNIT2'].lower(): header['CUNIT2'] = ('deg') except KeyError: pass # if GENERIC telescope, add implants to header if obsparam['telescope_keyword'] is 'GENERIC': for key, val in list(implants.items()): header[key] = (val[0], val[1]) # read out image binning mode binning = toolbox.get_binning(header, obsparam) # add pixel resolution keyword header['SECPIXX'] = (obsparam['secpix'][0] * binning[0], 'PP: x pixscale after binning') header['SECPIXY'] = (obsparam['secpix'][1] * binning[1], 'PP: y pixscale after binning') # create observation midtime jd if not keep_wcs or 'MIDTIMJD' not in header: if obsparam['date_keyword'].find('|') == -1: header['MIDTIMJD'] = \ (toolbox.dateobs_to_jd(header[obsparam['date_keyword']]) + float(header[obsparam['exptime']])/2./86400., 'PP: obs midtime') else: datetime = (header[obsparam['date_keyword'].split('|')[0]] + 'T' + header[obsparam['date_keyword'].split('|')[1]]) datetime = datetime.replace('/', '-') header['MIDTIMJD'] = ( toolbox.dateobs_to_jd(datetime) + float(header[obsparam['exptime']]) / 2. / 86400., 'PP: obs midtime') # other keywords header['TELINSTR'] = (obsparam['telescope_instrument'], 'PP: tel/instr name') header['TEL_KEYW'] = (obsparam['telescope_keyword'], 'PP: tel/instr keyword') header[obsparam['filter']] = header[obsparam['filter']].strip() header['FILTER'] = (header[obsparam['filter']], 'PP:copied') header['EXPTIME'] = (header[obsparam['exptime']], 'PP: copied') if obsparam['airmass'] in header: header['AIRMASS'] = (header[obsparam['airmass']], 'PP: copied') else: header['AIRMASS'] = (1, 'PP: fake airmass') # check if filter can be translated by PP try: obsparam['filter_translations'][header[obsparam['filter']]] except KeyError: logging.warning('cannot translate filter keyword \"' + header[obsparam['filter']] + '\"') #header[obsparam['filter']] = 'clear' header['FILTER'] = (header[obsparam['filter']], 'PP:copied') # perform header update for key, value in list(header_update.items()): if key in header: header['_' + key[:6]] = (header[key], 'PP: old value for %s' % key) header[key] = (value, 'PP: manually updated') # check if OBJECT keyword is available if 'OBJECT' not in header: header['OBJECT'] = 'None' elif len(header['OBJECT'].strip()) == 0: header['OBJECT'] = 'None' # # check if RA, Dec, airmass headers are available; # # else: query horizons # # to get approximate information # if (obsparam['ra'] not in header or # obsparam['dec'] not in header or # obsparam['airmass'] not in header): # logging.info('Either RA, Dec, or airmass missing from image ' + # 'header; pull approximate information for Horizons') # # obtain approximate ra and dec (and airmass) from JPL Horizons # eph = callhorizons.query(header[obsparam['object']]. # replace('_', ' ')) # eph.set_discreteepochs(header['MIDTIMJD']) # try: # n = eph.get_ephemerides(obsparam['observatory_code']) # except ValueError: # logging.warning('Target (%s) is not an asteroid' % # header[obsparam['object']]) # n = None # if n is None: # raise KeyError((('%s is not an asteroid known ' # 'to JPL Horizons') % # header[obsparam['object']])) # header[obsparam['ra']] = (eph['RA'][0], # 'PP: queried from Horizons') # header[obsparam['dec']] = (eph['DEC'][0], # 'PP: queried from Horizons') # header[obsparam['airmass']] = (eph['airmass'][0], # 'PP: queried from Horizons') # add fake wcs information that is necessary to run SCAMP # read out ra and dec from header if obsparam['radec_separator'] == 'XXX': ra_deg = float(header[obsparam['ra']]) dec_deg = float(header[obsparam['dec']]) else: ra_string = header[obsparam['ra']].split( obsparam['radec_separator']) dec_string = header[obsparam['dec']].split( obsparam['radec_separator']) ra_deg = 15. * (float(ra_string[0]) + old_div(float(ra_string[1]), 60.) + old_div(float(ra_string[2]), 3600.)) dec_deg = (abs(float(dec_string[0])) + old_div(float(dec_string[1]), 60.) + old_div(float(dec_string[2]), 3600.)) if dec_string[0].find('-') > -1: dec_deg = -1 * dec_deg # transform to equinox J2000, if necessary if 'EQUINOX' in header: equinox = float(header['EQUINOX']) if equinox != 2000.: anyeq = SkyCoord(ra=ra_deg * u.deg, dec=dec_deg * u.deg, frame=FK5, equinox=Time(equinox, format='jyear', scale='utc')) coo = anyeq.transform_to(ICRS) ra_deg = coo.ra.deg dec_deg = coo.dec.deg header['EQUINOX'] = (2000.0, 'PP: normalized to ICRS') if man_ra is not None and man_dec is not None: ra_deg = float(man_ra) dec_deg = float(man_dec) # special treatment for UKIRT/WFCAM if obsparam['telescope_keyword'] == 'UKIRTWFCAM': try: ra_deg = (float(header['TELRA']) / 24. * 360. - old_div(float(header['JITTER_X']), 3600.)) dec_deg = (float(header['TELDEC']) - old_div(float(header['JITTER_Y']), 3600.)) except KeyError: # JITTER keywords not in combined images pass # apply flips xnorm, ynorm = 1, 1 if this_flipx: xnorm = -1 if this_flipy: ynorm = -1 # check if instrument has a chip offset ra_offset, dec_offset = 0, 0 if (man_ra is None or man_dec is None) and \ 'chip_offset_fixed' in obsparam: cid = header[obsparam['chip_id']] ra_offset = float(obsparam['chip_offset_fixed'][cid][0]) dec_offset = float(obsparam['chip_offset_fixed'][cid][1]) if not keep_wcs: # create fake header header['RADECSYS'] = ('FK5', 'PP: fake wcs coordinates') header['RADESYS'] = ('FK5', 'PP: fake wcs coordinates') header['CTYPE1'] = ('RA---TAN', 'PP: fake Coordinate type') header['CTYPE2'] = ('DEC--TAN', 'PP: fake Coordinate type') header['CRVAL1'] = (ra_deg + ra_offset, 'PP: fake Coordinate reference value') header['CRVAL2'] = (dec_deg + dec_offset, 'PP: fake Coordinate reference value') header['CRPIX1'] = (int( old_div(float(header[obsparam['extent'][0]]), 2)), 'PP: fake Coordinate reference pixel') header['CRPIX2'] = (int( old_div(float(header[obsparam['extent'][1]]), 2)), 'PP: fake Coordinate reference pixel') # plugin default distortion parameters, if available if 'distort' in obsparam: if 'functionof' in obsparam['distort']: pv_dict = obsparam['distort'][header[obsparam['distort'] ['functionof']]] else: pv_dict = obsparam['distort'] try: for pv_key, pv_val in pv_dict.items(): header[pv_key] = (pv_val, 'PP: default distortion') except KeyError: logging.error(('No distortion coefficients available for ' '%s %s') % (obsparam['distort']['functionof'], header[obsparam['distort']['functionof']])) header['CD1_1'] = (xnorm * numpy.cos(this_rotate / 180. * numpy.pi) * obsparam['secpix'][0] * binning[0] / 3600., 'PP: fake Coordinate matrix') header['CD1_2'] = (ynorm * -numpy.sin(this_rotate / 180. * numpy.pi) * obsparam['secpix'][1] * binning[1] / 3600., 'PP: fake Coordinate matrix') header['CD2_1'] = (xnorm * numpy.sin(this_rotate / 180. * numpy.pi) * obsparam['secpix'][0] * binning[0] / 3600., 'PP: fake Coordinate matrix') header['CD2_2'] = (ynorm * numpy.cos(this_rotate / 180. * numpy.pi) * obsparam['secpix'][1] * binning[1] / 3600., 'PP: fake Coordinate matrix') # crop center from LOWELL42 frames if obsparam['telescope_keyword'] == 'LOWELL42': imdata = imdata[100:-100, 100:-100] logging.info('cropping LOWELL42 data') # overwrite imdata in case something has been modified hdulist[0].data = imdata hdulist.flush() hdulist.close() logging.info('created fake wcs information for image %s' % filename) # create diagnostics if diagnostics: diag.create_index(filenames, os.getcwd(), obsparam, display) logging.info('Done! -----------------------------------------------------') return None
import pylab as pl import numpy as np import astropy.io.fits as fits from astropy.table import Table, join, Column from desitarget.targets import encode_targetid from phot_sys import set_photsys from make_tiny import make_tiny print('\n\nWelcome.\n\n') scratch = os.environ['CSCRATCH'] ## hsc = Table(fits.open(scratch + '/BGS/SV-ASSIGN/elgs/hsc_north.fits')[1].data) hsc = hsc[hsc['DEC'] > -30.] hsc = set_photsys(hsc) hsc = hsc[hsc['PHOTSYS'] == 'BMZLS'] cols = hsc.columns cols = ['TARGETID', 'RA','DEC', 'DCHISQ', 'EBV', 'FLUX_G', 'FLUX_R', 'FLUX_Z', 'NOBS_G','NOBS_R','NOBS_Z', 'SHAPEEXP_R', 'TYPE','mizuki_photoz_best','FRANKENZ','PHOTSYS', 'PSFSIZE_G', 'PSFSIZE_R', 'PSFSIZE_Z'] hsc = hsc[cols] hsc = hsc[hsc['TYPE'] == 'REX'] hsc['MIZUKI'] = hsc['mizuki_photoz_best'] hsc = hsc[hsc['SHAPEEXP_R'] > 2.5] urls = ['http://viewer.legacysurvey.org/?ra={:.4f}&dec={:.4f}&zoom=16&layer=dr8'.format(i['RA'], i['DEC']) for i in hsc] urls = [make_tiny(url) for url in urls] hsc['TURL'] = urls
import numpy as np from astropy.io import fits import matplotlib.pyplot as plt from utils import * import scipy.ndimage.morphology as sp from matplotlib.colors import LogNorm import csv import pandas as pd from scipy.optimize import curve_fit hdulist = fits.open("A1_mosaic.fits") magzpt = hdulist[0].header['MAGZPT'] magzrr = hdulist[0].header['MAGZRR'] image = hdulist[0].data def linfit(x, a, b): return (a * x + b) """ fluxmaster = [] for i in range(100,1500,200): imageslice = image[100:300, i:i+200] fluxarray = em.fluxarray(imageslice, Rohan=True, im_show=True) for j in fluxarray: fluxmaster.append(j) print(fluxmaster) """ d1 = pd.read_csv("catalogue.csv")
This is a python procedure to read the galaxy file, add AGN to it (multiple times) ,the Agn being variable means the variability can be simulated using this model. ''' from astropy.io import fits import matplotlib.pyplot as plt from glob import glob import numpy as np s = np.zeros(15) x = np.arange(0, 15) #To open the galaxy.fits file and introduce a gaussian psf (the AGN ) inside it. # 15 night continuous observations. galaxy = fits.open('gal_moffat.fits') gal_data = galaxy[0].data #15 night continuous AGN observations with small variability in the amplitude. t = glob('agn*.fits') for i in range(0, 15): k = t[i] agn = fits.open(k) agn_data = agn[0].data com = gal_data + agn_data print('%i th image being processed' % i) new = fits.PrimaryHDU(com) myfit = fits.HDUList([new]) myfit.writeto('com%i.fits' % i, overwrite=True) myfit.close()
"""return the key of dictionary dic given the value""" return [k for k, v in dic.iteritems() if v == val] filein = sys.argv[1] filename, ext = os.path.splitext(filein) if ext == ".gz": filename = os.path.splitext(filename)[0] fileout = filename + "_filtered.fits" print filename # Binning of the resulting spectrum (in keV) newbinsize = 1.0 N_primaries = float(filein.split(".")[0].split("_")[1]) hdulist = pyfits.open(filein) hdr = hdulist[1].header data = hdulist[1].data eventid = data.field("EventID") endep = data.field("En_dep") X_ID = data.field("X_ID") Y_ID = data.field("Y_ID") theta = data.field("Theta_Primary") phi = data.field("Phi_Primary") hdulist.close() # Initialize output arrays eventid_out = [] endep_out = [] theta_out = [] phi_out = []
def keppixseries(infile, outfile=None, plotfile=None, plottype='global', filterlc=False, function='boxcar', cutoff=1.0, overwrite=False, verbose=False, logfile='keppixseries.log'): """ keppixseries -- individual time series photometry for all pixels within a target mask keppixseries plots a light curve for each individual pixel in a target mask. Light curves are extracted from a target pixel file obtained from the Kepler data archive at MAST. If required, the data can be fed through a boxcar, gaussian or sinc function high bandpass filter in order to remove low frequency signal from the data. keppixseries is a diagnostic tool for identifying source contaminants in the background or foreground of the target. It can be employed to identify pixels for inclusion or exclusion when re-extracting a Kepler light curve from target pixel files. Parameters ---------- infile : str The name of a MAST standard format FITS file containing Kepler Target Pixel data within the first data extension. outfile : str The name of the output FITS file. This file has two data extensions. The first called 'PIXELSERIES' contains a table with columns of barycenter-corrected time, barycenter time correction, cadence number, cadence quality flag and a series of photometric light curves, one for each pixel within the target mask. Each pixel is labeled COLx_ROWy, where :math:`x` is the pixel column number and :math:`y` is the pixel row number on the CCD module/output. The second extension contains the mask definition map copied directly from the input target pixel file. plotfile : str Name of an optional diagnostic output plot file containing the results of keppixseries. An example is provided in Figure 1. Typically this is a PNG format file. If no diagnostic file is required, plotfile can be 'None'. The plot will be generated regardless of the value of this field, but the plot will not be saved to a file if ``plotfile='None'``. plottype : str keppixseries can plot light curves of three types. The choice is made using this argument. The options are: * local - All individual pixel light curves are scaled separately to provide the most dynamic range for each pixel. * global - All pixel light curves are scaled between zero and the maximum flux attained by the brightest pixel in the mask. This option provides the relative contribution to the archived light curve by each pixel. * full - All pixels light curves are scaled between zero and the maximum flux attained by that pixel. This provides the fraction of variability within each individual pixel. filterlc : bool If True, the light curve for each pixel will be treated by a high band-pass filter to remove long-term trends from e.g. differential velocity aberration. function : str The functional form of the high pass-band filter: * boxcar * gauss * sinc cutoff : float The frequency of the high pass-band cutoff in units of :math:`days^{-1}`. overwrite : bool Overwrite the output file? verbose : bool Print informative messages and warnings to the shell and logfile? logfile = str Name of the logfile containing error and warning messages. Examples -------- .. code-block :: bash $ keppixseries kplr008256049-2010174085026_lpd-targ.fits.gz .. image:: ../_static/images/api/keppixseries.png :align: center """ if outfile is None: outfile = infile.split('.')[0] + "-{}.fits".format(__all__[0]) # log the call hashline = '--------------------------------------------------------------' kepmsg.log(logfile, hashline, verbose) call = ('KEPPIXSERIES -- ' + ' infile={}'.format(infile) + ' outfile={}'.format(outfile) + ' plotfile={}'.format(plotfile) + ' plottype={}'.format(plottype) + ' filterlc={}'.format(filterlc) + ' function={}'.format(function) + ' cutoff={}'.format(cutoff) + ' overwrite={}'.format(overwrite) + ' verbose={}'.format(verbose) + ' logfile={}'.format(logfile)) kepmsg.log(logfile, call + '\n', verbose) # start time kepmsg.clock('KEPPIXSERIES started at', logfile, verbose) # overwrite output file if overwrite: kepio.overwrite(outfile, logfile, verbose) if kepio.fileexists(outfile): errmsg = ('ERROR -- KEPPIXSERIES: {} exists. Use --overwrite'.format( outfile)) kepmsg.err(logfile, errmsg, verbose) # open TPF FITS file kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, barytime = \ kepio.readTPF(infile, 'TIME', logfile, verbose) kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, tcorr = \ kepio.readTPF(infile, 'TIMECORR', logfile, verbose) kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, cadno = \ kepio.readTPF(infile, 'CADENCENO', logfile, verbose) kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, fluxpixels = \ kepio.readTPF(infile, 'FLUX', logfile, verbose) kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, errpixels = \ kepio.readTPF(infile, 'FLUX_ERR', logfile, verbose) kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, qual = \ kepio.readTPF(infile, 'QUALITY', logfile, verbose) # read mask defintion data from TPF file maskimg, pixcoord1, pixcoord2 = kepio.readMaskDefinition( infile, logfile, verbose) # print target data print('') print(' KepID: {}'.format(kepid)) print(' RA (J2000): {}'.format(ra)) print('Dec (J2000): {}'.format(dec)) print(' KepMag: {}'.format(kepmag)) print(' SkyGroup: {}'.format(skygroup)) print(' Season: {}'.format(season)) print(' Channel: {}'.format(channel)) print(' Module: {}'.format(module)) print(' Output: {}'.format(output)) print('') # how many quality = 0 rows? npts = 0 nrows = len(fluxpixels) for i in range(nrows): if (qual[i] == 0 and np.isfinite(barytime[i]) and np.isfinite(fluxpixels[i, ydim * xdim // 2])): npts += 1 time = np.empty((npts)) timecorr = np.empty((npts)) cadenceno = np.empty((npts)) quality = np.empty((npts)) pixseries = np.empty((ydim, xdim, npts)) errseries = np.empty((ydim, xdim, npts)) # construct output light curves nptsx = 0 for i in tqdm(range(ydim)): for j in range(xdim): npts = 0 for k in range(nrows): if (qual[k] == 0 and np.isfinite(barytime[k]) and np.isfinite(fluxpixels[k, int(ydim * xdim / 2)])): time[npts] = barytime[k] timecorr[npts] = tcorr[k] cadenceno[npts] = cadno[k] quality[npts] = qual[k] pixseries[i, j, npts] = fluxpixels[k, nptsx] errseries[i, j, npts] = errpixels[k, nptsx] npts += 1 nptsx += 1 # define data sampling if filterlc: tpf = pyfits.open(infile) cadence = kepkey.cadence(tpf[1], infile, logfile, verbose) tr = 1.0 / (cadence / 86400) timescale = 1.0 / (cutoff / tr) # define convolution function if function == 'boxcar': filtfunc = np.ones(int(np.ceil(timescale))) elif function == 'gauss': timescale /= 2 dx = np.ceil(timescale * 10 + 1) filtfunc = filtfunc([1.0, dx / 2 - 1.0, timescale], np.linspace(0, dx - 1, dx)) elif function == 'sinc': dx = np.ceil(timescale * 12 + 1) fx = np.linspace(0, dx - 1, dx) fx = fx - dx / 2 + 0.5 fx /= timescale filtfunc = np.sinc(fx) filtfunc /= np.sum(filtfunc) # pad time series at both ends with noise model for i in range(ydim): for j in range(xdim): ave, sigma = (np.mean(pixseries[i, j, :len(filtfunc)]), np.std(pixseries[i, j, :len(filtfunc)])) padded = np.append( kepstat.randarray( np.ones(len(filtfunc)) * ave, np.ones(len(filtfunc)) * sigma), pixseries[i, j, :]) ave, sigma = (np.mean(pixseries[i, j, -len(filtfunc):]), np.std(pixseries[i, j, -len(filtfunc):])) padded = np.append( padded, kepstat.randarray( np.ones(len(filtfunc)) * ave, np.ones(len(filtfunc)) * sigma)) # convolve data convolved = np.convolve(padded, filtfunc, 'same') # remove padding from the output array outdata = convolved[len(filtfunc):-len(filtfunc)] # subtract low frequencies outmedian = np.median(outdata) pixseries[i, j, :] = pixseries[i, j, :] - outdata + outmedian # construct output file print("Writing output file {}...".format(outfile)) if ydim * xdim < 1000: instruct = pyfits.open(infile, 'readonly') kepkey.history(call, instruct[0], outfile, logfile, verbose) hdulist = pyfits.HDUList(instruct[0]) cols = [] cols.append( pyfits.Column(name='TIME', format='D', unit='BJD - 2454833', disp='D12.7', array=time)) cols.append( pyfits.Column(name='TIMECORR', format='E', unit='d', disp='E13.6', array=timecorr)) cols.append( pyfits.Column(name='CADENCENO', format='J', disp='I10', array=cadenceno)) cols.append(pyfits.Column(name='QUALITY', format='J', array=quality)) for i in range(ydim): for j in range(xdim): colname = 'COL{}_ROW{}'.format(i + column, j + row) cols.append( pyfits.Column(name=colname, format='E', disp='E13.6', array=pixseries[i, j, :])) hdu1 = pyfits.BinTableHDU.from_columns(pyfits.ColDefs(cols)) try: hdu1.header['INHERIT'] = (True, 'inherit the primary header') except: pass try: hdu1.header['EXTNAME'] = ('PIXELSERIES', 'name of extension') except: pass try: hdu1.header['EXTVER'] = ( instruct[1].header['EXTVER'], 'extension version number (not format version)') except: pass try: hdu1.header['TELESCOP'] = (instruct[1].header['TELESCOP'], 'telescope') except: pass try: hdu1.header['INSTRUME'] = (instruct[1].header['INSTRUME'], 'detector type') except: pass try: hdu1.header['OBJECT'] = (instruct[1].header['OBJECT'], 'string version of KEPLERID') except: pass try: hdu1.header['KEPLERID'] = (instruct[1].header['KEPLERID'], 'unique Kepler target identifier') except: pass try: hdu1.header['RADESYS'] = ( instruct[1].header['RADESYS'], 'reference frame of celestial coordinates') except: pass try: hdu1.header['RA_OBJ'] = (instruct[1].header['RA_OBJ'], '[deg] right ascension from KIC') except: pass try: hdu1.header['DEC_OBJ'] = (instruct[1].header['DEC_OBJ'], '[deg] declination from KIC') except: pass try: hdu1.header['EQUINOX'] = (instruct[1].header['EQUINOX'], 'equinox of celestial coordinate system') except: pass try: hdu1.header['TIMEREF'] = ( instruct[1].header['TIMEREF'], 'barycentric correction applied to times') except: pass try: hdu1.header['TASSIGN'] = (instruct[1].header['TASSIGN'], 'where time is assigned') except: pass try: hdu1.header['TIMESYS'] = (instruct[1].header['TIMESYS'], 'time system is barycentric JD') except: pass try: hdu1.header['BJDREFI'] = (instruct[1].header['BJDREFI'], 'integer part of BJD reference date') except: pass try: hdu1.header['BJDREFF'] = ( instruct[1].header['BJDREFF'], 'fraction of the day in BJD reference date') except: pass try: hdu1.header['TIMEUNIT'] = (instruct[1].header['TIMEUNIT'], 'time unit for TIME, TSTART and TSTOP') except: pass try: hdu1.header['TSTART'] = (instruct[1].header['TSTART'], 'observation start time in BJD-BJDREF') except: pass try: hdu1.header['TSTOP'] = (instruct[1].header['TSTOP'], 'observation stop time in BJD-BJDREF') except: pass try: hdu1.header['LC_START'] = (instruct[1].header['LC_START'], 'mid point of first cadence in MJD') except: pass try: hdu1.header['LC_END'] = (instruct[1].header['LC_END'], 'mid point of last cadence in MJD') except: pass try: hdu1.header['TELAPSE'] = (instruct[1].header['TELAPSE'], '[d] TSTOP - TSTART') except: pass try: hdu1.header['LIVETIME'] = (instruct[1].header['LIVETIME'], '[d] TELAPSE multiplied by DEADC') except: pass try: hdu1.header['EXPOSURE'] = (instruct[1].header['EXPOSURE'], '[d] time on source') except: pass try: hdu1.header['DEADC'] = (instruct[1].header['DEADC'], 'deadtime correction') except: pass try: hdu1.header['TIMEPIXR'] = (instruct[1].header['TIMEPIXR'], 'bin time beginning=0 middle=0.5 end=1') except: pass try: hdu1.header['TIERRELA'] = (instruct[1].header['TIERRELA'], '[d] relative time error') except: pass try: hdu1.header['TIERABSO'] = (instruct[1].header['TIERABSO'], '[d] absolute time error') except: pass try: hdu1.header['INT_TIME'] = ( instruct[1].header['INT_TIME'], '[s] photon accumulation time per frame') except: pass try: hdu1.header['READTIME'] = (instruct[1].header['READTIME'], '[s] readout time per frame') except: pass try: hdu1.header['FRAMETIM'] = (instruct[1].header['FRAMETIM'], '[s] frame time (INT_TIME + READTIME)') except: pass try: hdu1.header['NUM_FRM'] = (instruct[1].header['NUM_FRM'], 'number of frames per time stamp') except: pass try: hdu1.header['TIMEDEL'] = (instruct[1].header['TIMEDEL'], '[d] time resolution of data') except: pass try: hdu1.header['DATE-OBS'] = (instruct[1].header['DATE-OBS'], 'TSTART as UTC calendar date') except: pass try: hdu1.header['DATE-END'] = (instruct[1].header['DATE-END'], 'TSTOP as UTC calendar date') except: pass try: hdu1.header['BACKAPP'] = (instruct[1].header['BACKAPP'], 'background is subtracted') except: pass try: hdu1.header['DEADAPP'] = (instruct[1].header['DEADAPP'], 'deadtime applied') except: pass try: hdu1.header['VIGNAPP'] = ( instruct[1].header['VIGNAPP'], 'vignetting or collimator correction applied') except: pass try: hdu1.header['GAIN'] = (instruct[1].header['GAIN'], '[electrons/count] channel gain') except: pass try: hdu1.header['READNOIS'] = (instruct[1].header['READNOIS'], '[electrons] read noise') except: pass try: hdu1.header['NREADOUT'] = (instruct[1].header['NREADOUT'], 'number of read per cadence') except: pass try: hdu1.header['TIMSLICE'] = (instruct[1].header['TIMSLICE'], 'time-slice readout sequence section') except: pass try: hdu1.header['MEANBLCK'] = (instruct[1].header['MEANBLCK'], '[count] FSW mean black level') except: pass hdulist.append(hdu1) hdulist.writeto(outfile) kepkey.new('EXTNAME', 'APERTURE', 'name of extension', instruct[2], outfile, logfile, verbose) pyfits.append(outfile, instruct[2].data, instruct[2].header) instruct.close() else: warnmsg = ('WARNING -- KEPPIXSERIES: output FITS file requires > 999' 'columns. Non-compliant with FITS convention.') kepmsg.warn(logfile, warnmsg) # plot pixel array fmin = 1.0e33 fmax = -1.033 plt.figure() plt.clf() dx = 0.93 / xdim dy = 0.94 / ydim ax = plt.axes([0.06, 0.05, 0.93, 0.94]) plt.gca().xaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False)) plt.gca().yaxis.set_major_formatter(plt.ScalarFormatter(useOffset=False)) plt.gca().xaxis.set_major_locator(plt.MaxNLocator(integer=True)) plt.gca().yaxis.set_major_locator(plt.MaxNLocator(integer=True)) plt.xlim(np.min(pixcoord1) - 0.5, np.max(pixcoord1) + 0.5) plt.ylim(np.min(pixcoord2) - 0.5, np.max(pixcoord2) + 0.5) plt.xlabel('time', {'color': 'k'}) plt.ylabel('arbitrary flux', {'color': 'k'}) for i in range(ydim): for j in range(xdim): tmin = np.amin(time) tmax = np.amax(time) try: np.isfinite(np.amin(pixseries[i, j, :])) np.isfinite(np.amin(pixseries[i, j, :])) fmin = np.amin(pixseries[i, j, :]) fmax = np.amax(pixseries[i, j, :]) except: ugh = 1 xmin = tmin - (tmax - tmin) / 40 xmax = tmax + (tmax - tmin) / 40 ymin = fmin - (fmax - fmin) / 20 ymax = fmax + (fmax - fmin) / 20 if kepstat.bitInBitmap(maskimg[i, j], 2): plt.axes([0.06 + float(j) * dx, 0.05 + i * dy, dx, dy], facecolor='lightslategray') elif maskimg[i, j] == 0: plt.axes([0.06 + float(j) * dx, 0.05 + i * dy, dx, dy], facecolor='black') else: plt.axes([0.06 + float(j) * dx, 0.05 + i * dy, dx, dy]) if j == int(xdim / 2) and i == 0: plt.setp(plt.gca(), xticklabels=[], yticklabels=[]) elif j == 0 and i == int(ydim / 2): plt.setp(plt.gca(), xticklabels=[], yticklabels=[]) else: plt.setp(plt.gca(), xticklabels=[], yticklabels=[]) ptime = time * 1.0 ptime = np.insert(ptime, [0], ptime[0]) ptime = np.append(ptime, ptime[-1]) pflux = pixseries[i, j, :] * 1.0 pflux = np.insert(pflux, [0], -1000.0) pflux = np.append(pflux, -1000.0) plt.plot(time, pixseries[i, j, :], color='#0000ff', linestyle='-', linewidth=0.5) if not kepstat.bitInBitmap(maskimg[i, j], 2): plt.fill(ptime, pflux, fc='lightslategray', linewidth=0.0, alpha=1.0) plt.fill(ptime, pflux, fc='#FFF380', linewidth=0.0, alpha=1.0) if 'loc' in plottype: plt.xlim(xmin, xmax) plt.ylim(ymin, ymax) if 'glob' in plottype: plt.xlim(xmin, xmax) plt.ylim(1.0e-10, np.nanmax(pixseries) * 1.05) if 'full' in plottype: plt.xlim(xmin, xmax) plt.ylim(1.0e-10, ymax * 1.05) # render plot plt.show() plt.savefig(plotfile) # stop time kepmsg.clock('KEPPIXSERIES ended at', logfile, verbose)
def read_files(filter, basedir): """ Read in the files for a given filter and base directory. The structure of the directory is such that within this directory there are cat/ and psf/ sub-directories containing the relevant catalogs Note: Do not include the final slash! e.g. on NERSC the basedir looks like either /global/cscratch1/sd/amichoi/UltraVISTA or /global/cscratch1/sd/amichoi/VIDEO """ survey = os.path.split(basedir)[1] print(survey) # First read in the SExtractor for PSFEx catalog if survey == 'UltraVISTA': sex_file = os.path.join(basedir, 'cat/UVISTA_%s_21_01_16_psfcat.fits' % filter) elif survey == 'VIDEO': sex_file = os.path.join( basedir, 'cat/VIDEO_%s_10_36.80_-5.01_psfcat.fits' % filter) dat = fits.open(sex_file) cols = dat[2].columns #print(cols) sex = Table(dat[2].data) print("Length of sex file: ", len(sex)) #read in catalog containing list of stars made from Sextractor and PSFEx if survey == 'UltraVISTA': star_file = os.path.join( basedir, 'psf_minsn500/UVISTA_%s_21_01_16_psfex-starlist.fits' % filter) elif survey == 'VIDEO': star_file = os.path.join( basedir, 'psf/VIDEO_%s_10_36.80_-5.01_psfex-starlist.fits' % (filter)) dat = fits.open(star_file) cols = dat[2].columns #print(cols) star = Table(dat[2].data) print("Length of star file: ", len(star)) sex['X_IMAGE'] = sex['X_IMAGE'].astype(int) star['X_IMAGE'] = star['X_IMAGE'].astype(int) sex['Y_IMAGE'] = sex['Y_IMAGE'].astype(int) star['Y_IMAGE'] = star['Y_IMAGE'].astype(int) sexstarmerge = join(sex, star, keys=['X_IMAGE', 'Y_IMAGE'], join_type='inner') #sexstarmerge = join(sex, star, join_type='inner') print("length of merged cat: ", len(sexstarmerge)) cols = tuple(name for name in sexstarmerge.colnames if len(sexstarmerge[name].shape) <= 1) t2 = sexstarmerge[cols] sexstardf = t2.to_pandas() return sexstarmerge, sex, star
from time import strftime, gmtime ## specify source files path_to_cal = 'ArchCal/' # source directory dates = [f for f in os.listdir(path_to_cal) if not f.startswith('.')] # index date folders in ArchCal path_to_cal += max(dates)+'/' # specify path as most recent date filenames = [f for f in os.listdir(path_to_cal) if not f.startswith('.')] # list of filenames to process print('Searching %s for calibraton files...' % path_to_cal) bias,dark,Red,Green,Blue,R,V,B,Halpha,Lum,filters = [],[],[],[],[],[],[],[],[],[],[] # initialize lists # lists are used to store the data for each calibration file and then combine into a master ## sort the calibration images by type and store them in arrays for filename in filenames: img = fits.open(path_to_cal+filename) # open each image data = img[0].data # split into data and header header = img[0].header if header['IMAGETYP']=='Bias Frame': bias_header = img[0].header # save header for each image type to attach to master version bias.append(data) # add data array to type list if header['IMAGETYP']=='Dark Frame': dark_header = img[0].header dark.append(data) if header['IMAGETYP']=='Flat Field': flat_header = img[0].header filters.append(header['FILTER']) # store the filters found in this directory in a list # so that we don't attempt to create new master flats with filters we did not have raw flats for code = header['FILTER']+'.append(data)' # string operations to add data to filter-specific list exec(code) del img
SPLUS-s29s31-50.22486407163766--37.134297664400115-256_R.fits (1) result = [] hAlphaFlux = [] bands = ['_F660_','_I_','_R_'] i = 0 coef_F660 = 125.3 coef_R = 1419.0 fnu_R = 0 for file in glob.glob("*_swp.fits"): for i in range(len(bands)): if bands[i] in file: mainOutput = sys.stdout fileName = "FILENAME = " + file openFile = fits.open(file) header = openFile[1].header data = openFile[1].data header = str(header) fileNameList = [] fileNameList.append(fileName) headerList = [] headerList.append(header) dataList = [] dataList.append(data) result.append(fileNameList + headerList + dataList) for i in range(len(result)): fileName2 = str(result[i])
def load_images(images, ra, dec, wcs_dict, orbit=None, dra=None, ddec=None, target=DS9_NAME, regions=None, rejected=False, basedate=None): """ Load a list of images into a ds9 session. Note: orbit overrides what is in ra/dec/dra/ddec :param images: list of image filenames (full path) :param wcs_dict: Dictionary of WCS objects (wcs_dict[images[0]] will be WCS associated with image in frame 1) :param orbit: a BKObrbit object used to put the circle on the target based on the MJD of the image. :param ra: RA of source being measured (deg; used put a circle on ds9 image) :param dec: DEC of the source being measured (deg; used put a circle on ds9 image) :param dra: draw an error ellipse at ra/dec using dra/ddec sizes if no orbit given. :param ddec: draw an error ellipse at ra/dec using dra/ddec sizes if no orbit given. :param target: name of the ds9 session to put images into (aka 'validate') :param regions: name of file that holds a list of regions that should be maked on the display. :param rejected: Is this a rejected / null observation. :param basedate: date that the ra/dec is valid for (aka epoch of coordinate) :return: """ ds9 = get_ds9(target) ds9.set('frame delete all') ds9.set('zscale') for image in images: ds9.set('frame new') ds9.set(f"file {image}") with fits.open(image) as hdulist: header = hdulist[1].header obsdate = Time(hdulist[0].header['DATE-AVG'], scale='tai').utc if basedate is None: basedate = obsdate wcs_dict[image] = WCS(header) colour = 'yellow' if orbit is not None: orbit.predict(obsdate) ra1 = orbit.coordinate.ra.degree dec1 = orbit.coordinate.dec.degree uncertainty_ellipse = (orbit.dra.to('arcsec').value, orbit.ddec.to('arcsec').value, orbit.pa.to('degree').value + 90) colour = 'green' for obs in orbit.observations: record_index = obs.date.mpc[0:13] if record_index == obsdate.mpc[0:13]: colour = not obs.null_observation and 'cyan' or 'red' circle = f'icrs; circle({obs.coordinate.ra.degree}d,{obs.coordinate.dec.degree}d,5p) # color={colour}' ds9.set('regions', circle) break else: ra1 = ra + dra * (obsdate - basedate).to('hour').value / 3600. dec1 = dec + ddec * (obsdate - basedate).to('hour').value / 3600.0 uncertainty_ellipse = 3, 3, 0 colour = 'pink' colour = rejected and 'red' or colour ds9.set( 'regions', f'icrs; ellipse({ra1},{dec1},' f'{uncertainty_ellipse[0]}",' f'{uncertainty_ellipse[1]}",' f'{uncertainty_ellipse[2]}) # color={colour}') ds9.set(f'pan to {ra1} {dec1} wcs icrs') logging.debug(f'Loading regions from {regions}') if regions is not None: ds9.set(f'regions {regions}') ds9.set('frame match wcs') ds9.set('frame first')
def measure_strehl(HDUlist_or_filename=None, ext=0, slice=0, center=None, display=True, verbose=True, cache_perfect=False): """ Estimate the Strehl ratio for a PSF. This requires computing a simulated PSF with the same properties as the one under analysis. Note that this calculation will not be very accurate unless both PSFs are well sampled, preferably several times better than Nyquist. See `Roberts et al. 2004 SPIE 5490 <http://adsabs.harvard.edu/abs/2004SPIE.5490..504R>`_ for a discussion of the various possible pitfalls when calculating Strehl ratios. WARNING: This routine attempts to infer how to calculate a perfect reference PSF based on FITS header contents. It will likely work for simple direct imaging cases with WebbPSF but will not work (yet) for more complicated cases such as coronagraphy, anything with image or pupil masks, etc. Code contributions to add such cases are welcomed. Parameters ---------- HDUlist_or_filename : string Either a fits.HDUList object or a filename of a FITS file on disk ext : int Extension in that FITS file slice : int, optional If that extension is a 3D datacube, which slice (plane) of that datacube to use center : tuple center to compute around. Default is image center. If the center is on the crosshairs between four pixels, then the mean of those four pixels is used. Otherwise, if the center is in a single pixel, then that pixel is used. verbose, display : bool control whether to print the results or display plots on screen. cache_perfect : bool use caching for perfect images? greatly speeds up multiple calcs w/ same config Returns --------- strehl : float Strehl ratio as a floating point number between 0.0 - 1.0 """ from .webbpsf_core import Instrument from poppy import display_PSF if isinstance(HDUlist_or_filename, six.string_types): HDUlist = fits.open(HDUlist_or_filename) elif isinstance(HDUlist_or_filename, fits.HDUList): HDUlist = HDUlist_or_filename else: raise ValueError("input must be a filename or HDUlist") image = HDUlist[ext].data header = HDUlist[ext].header if image.ndim >= 3: # handle datacubes gracefully image = image[slice, :, :] if center is None: # get exact center of image #center = (image.shape[1]/2, image.shape[0]/2) center = tuple((a - 1) / 2.0 for a in image.shape[::-1]) # Compute a comparison image _log.info("Now computing image with zero OPD for comparison...") inst = Instrument(header['INSTRUME']) inst.filter = header['FILTER'] inst.pupilopd = None # perfect image inst.pixelscale = header['PIXELSCL'] * header[ 'OVERSAMP'] # same pixel scale pre-oversampling cache_key = (header['INSTRUME'], header['FILTER'], header['PIXELSCL'], header['OVERSAMP'], header['FOV'], header['NWAVES']) try: comparison_psf = _Strehl_perfect_cache[cache_key] except KeyError: comparison_psf = inst.calcPSF(fov_arcsec=header['FOV'], oversample=header['OVERSAMP'], nlambda=header['NWAVES']) if cache_perfect: _Strehl_perfect_cache[cache_key] = comparison_psf comparison_image = comparison_psf[0].data if (int(center[1]) == center[1]) and (int(center[0]) == center[0]): # individual pixel meas_peak = image[center[1], center[0]] ref_peak = comparison_image[center[1], center[0]] else: # average across a group of 4 bot = [np.floor(f) for f in center] top = [np.ceil(f) + 1 for f in center] meas_peak = image[bot[1]:top[1], bot[0]:top[0]].mean() ref_peak = comparison_image[bot[1]:top[1], bot[0]:top[0]].mean() strehl = (meas_peak / ref_peak) if display: plt.clf() plt.subplot(121) display_PSF(HDUlist, title="Observed PSF") plt.subplot(122) display_PSF(comparison_psf, title="Perfect PSF") plt.gcf().suptitle("Strehl ratio = %.3f" % strehl) if verbose: print("Measured peak: {0:.3g}".format(meas_peak)) print("Reference peak: {0:.3g}".format(ref_peak)) print(" Strehl ratio: {0:.3f}".format(strehl)) return strehl
def measure_image(p_name, images, wcs_dict, discovery=False, target=DS9_NAME, zpt=26.9): """ :param p_name: provisional name of the source being measured :param images: list of images being measured. Expect that image[0] is in ds9 Frame 1 :param wcs_dict: dictionary of WCS objects, one for each image. wcs_dict[images[0]] is the WCS objet for image in frame 1. :param discovery: Is this a discovery image (add a '*' to the MPC record) :param target: Name of the DS9 session, set by xpa (i.e. validate) :param zpt: zeropoint of the frame. :return: an mpc_ephem Observation record. :rtype: list(ObsRecord) """ ds9 = get_ds9(target) # Build a map of allowed key strokes allowed_keys = { 'x': ('', 'centroid at this location'), 'q': ('', 'Quit this image set'), 'p': ('', 'Previous frame'), 'n': ('', 'Next frame'), 'r': ('', 'Create a NULL observation.') } for key in [x.split() for x in config.read("MPC.NOTE1OPTIONS")]: allowed_keys[key[0].lower()] = key obs = {} while True: try: result = ds9.get('iexam key coordinate image') key, x, y = result.split() x = float(x) y = float(y) logging.debug(f"DS9 Returned: {result} -> {key} {x} {y}") except Exception as ex: logging.debug(f"DS9 get exception: {ex}") break if key == 'n': ds9.set('frame next') continue if key == 'p': ds9.set('frame prev') continue if key == 'q': break if key == 'Q': raise SystemExit("Q hit") if key not in allowed_keys: logging.info(f"Allowed keys: ") for key in allowed_keys: print(f"{key} -> {allowed_keys[key][1]}") continue note1 = allowed_keys[key][0] frame_no = int(ds9.get('frame')) - 1 image = images[frame_no] ds9.set('regions', f'image; circle {x} {y} 5') centroid = not note1 == 'H' phot = daophot.phot_mag(image, [ x, ], [ y, ], aperture=5, sky_inner_radius=15, sky_annulus_width=10, apcor=0.3, zmag=zpt, maxcount=1000, extno=1, exptime=90.0, centroid=centroid) phot_failure = (phot['PIER'][0] != 0 or phot.mask[0]['MAG'] or phot.mask[0]['MERR']) sky_failure = phot['SIER'][0] != 0 cen_failure = phot['CIER'][0] != 0 if phot_failure or sky_failure or cen_failure: logging.warning(f"iraf.daophot.phot error:\n {phot}") cen_x = x cen_y = y obs_mag = None obs_mag_err = None note1 = "H" else: cen_x = phot['XCENTER'][0] cen_y = phot['YCENTER'][0] obs_mag = phot['MAG'][0] obs_mag_err = phot['MERR'][0] colour = "{blue}" ds9.set('regions', f'image; circle {cen_x} {cen_y} 4 # color={colour}') obsdate = Time(Time(fits.open(image)[0].header['DATE-AVG'], scale='tai').utc, format='mjd', precision=6).mpc ra = dec = "UNDEFINED" try: ra, dec = wcs_dict[image].all_pix2world(cen_x, cen_y, 1) except Exception as ex: logging.warning( f"Failure converting {cen_x, cen_y} to RA/DEC for {image}") logging.warning(ex) logging.warning(f"Got: {ra},{dec}") record_key = obsdate[0:13] null_obs = key in ['r', 'b'] obs[record_key] = Observation( null_observation=null_obs, provisional_name=p_name, comment="{} {} {}".format(*util.from_provisional_name(p_name)), note1=note1, note2='C', date=obsdate, ra=ra, dec=dec, mag=obs_mag, mag_err=obs_mag_err, observatory_code='568', discovery=discovery, xpos=x, ypos=y, frame=os.path.splitext(os.path.basename(image))[0]) return obs
def get_CHMap_events(map_path): '''Return all the SPOCA_CoronalHole, SPOCA_CoronalHoleDetection, SPOCA_CoronalHoleStatistics and SPOCA_CoronalHoleRun events in a SPoCA CHMap''' # Open the FITS file hdus = fits.open(map_path) # Create a sunpy Map for converting the pixel coordinates image_hdu = hdus[image_hdu_name] map = Map(image_hdu.data, image_hdu.header) # Get the regions by id regions_hdu = hdus[region_hdu_name] regions = {region['ID']: region for region in regions_hdu.data} # Get chaincodes by id chaincodes_hdu = hdus[chaincode_hdu_name] chaincodes = { id: (chaincodes_hdu.data['X%07d' % id], chaincodes_hdu.data['Y%07d' % id]) for id in regions_hdu.data['ID'] } # Get region stats by channel then by id region_stats = { hdus[region_stats_hdu_name].header['CHANNEL']: { region_stat['ID']: region_stat for region_stat in hdus[region_stats_hdu_name].data } for region_stats_hdu_name in region_stats_hdu_names if region_stats_hdu_name in hdus } # Create the CH events CH_events = list() # Keep the list of detections for the SPOCA_CoronalHoleRun event detection_names = list() for id, region in regions.items(): events = dict() spoca_coronal_hole_detection_name = 'SPOCA_CoronalHoleDetection_{date}_{id}'.format( date=region['DATE_OBS'], id=id) events[ 'spoca_coronal_hole_detection'] = get_spoca_coronal_hole_detection( map, region, next(iter(region_stats.values()))[id], chaincodes[id], name=spoca_coronal_hole_detection_name) spoca_coronal_hole_name = 'SPOCA_CoronalHole_{color}'.format( color=region['TRACKED_COLOR']) events['spoca_coronal_hole'] = get_spoca_coronal_hole( spoca_coronal_hole_detection_name, region['DATE_OBS'] + 'Z', name=spoca_coronal_hole_name) events['spoca_coronal_hole_statistics'] = list() for channel, stats in region_stats.items(): spoca_coronal_hole_statistics_name = 'SPOCA_CoronalHoleStatistics_{date}_{id}_{channel}'.format( date=region['DATE_OBS'], id=id, channel=channel) events['spoca_coronal_hole_statistics'].append( get_spoca_coronal_hole_statistics( spoca_coronal_hole_detection_name, channel, stats[id], name=spoca_coronal_hole_statistics_name)) CH_events.append(events) detection_names.append(spoca_coronal_hole_detection_name) # Create the run event image_date = image_hdu.header['DATE_OBS'].split('.')[ 0] # We don't want the subsecond spoca_coronal_hole_run_name = 'SPOCA_CoronalHoleRun_{date}'.format( date=image_date) run_event = get_spoca_coronal_hole_run(image_date + 'Z', detection_names, name=spoca_coronal_hole_run_name) return run_event, CH_events
def add_fffits_metadata(ff_filename, config, platepars_recalibrated, fallback_platepar): """ Add FITS metadata and WCS to FF files generated by RMS Args: ff_filename (str): full or relative path to FF file config (RMS.Config): config instance platepars_recalibrated (dict): dictionary with recalibrated platepars fallback_platepar (RMS.Platepar): platepar with fitted stars Returns: None """ ff_basename = os.path.basename(ff_filename) platepar_recalibrated = Platepar() try: platepar_data = platepars_recalibrated[ff_basename] with open("platepar_tmp.cal", "w") as f: json.dump(platepar_data, f) platepar_recalibrated.read("platepar_tmp.cal") except (FileNotFoundError, KeyError): platepar_recalibrated = fallback_platepar logger.warning(f"Using non-recalibrated platepar for {ff_basename}") fftime = getMiddleTimeFF(ff_basename, config.fps) fit_xy = np.array(fallback_platepar.star_list)[:, 1:3] _, fit_ra, fit_dec, _ = xyToRaDecPP([fftime] * len(fit_xy), fit_xy[:, 0], fit_xy[:, 1], [1] * len(fit_xy), platepar_recalibrated, extinction_correction=False) x0 = platepar_recalibrated.X_res / 2 y0 = platepar_recalibrated.Y_res / 2 _, ra0, dec0, _ = xyToRaDecPP([fftime], [x0], [y0], [1], platepar_recalibrated, extinction_correction=False) w = fit_wcs(fit_xy[:, 0], fit_xy[:, 1], fit_ra, fit_dec, x0, y0, ra0[0], dec0[0], 5, projection="ZEA") hdu_list = fits.open(ff_filename, scale_back=True) obstime = Time(filenameToDatetime(ff_basename)) header_meta = {} header_meta["OBSERVER"] = config.stationID.strip() header_meta["INSTRUME"] = "Global Meteor Network" header_meta["MJD-OBS"] = obstime.mjd header_meta["DATE-OBS"] = obstime.fits header_meta["NFRAMES"] = 256 header_meta["EXPTIME"] = 256 / config.fps header_meta["SITELONG"] = round(config.longitude, 2) header_meta["SITELAT"] = round(config.latitude, 2) for hdu in hdu_list: if hdu.header[ "NAXIS"] == 0: # First header is not an image so should not get WCS new_header = Header() else: new_header = w.to_fits(relax=True)[0].header for key, value in header_meta.items(): new_header.append((key, value)) for key, value in new_header.items(): if key in hdu.header: continue hdu.header[key] = value hdu_list.writeto(ff_filename, overwrite=True)
def main(orbit=None, **kwargs): """ This is the driver program. Gets the images from VOSpace or the local filesystem. expected kwargs: pointing, index, chip, rate, angle, ra, dec, p_name, discovery, nstk, zpt, dbimages :param kwargs: these are the 'args' sent on the commandline or in an input file. :param orbit: the orbit of the object that should be in the frames reference by kwargs provided. :type orbit: BKOrbit :return: list of observations :rtype list(ObsRecod) """ pointing = kwargs['pointing'] chip = kwargs['chip'] index = kwargs['index'] rate = kwargs['rate'] angle = kwargs['angle'] ra = kwargs['ra'] dec = kwargs['dec'] p_name = kwargs['p_name'] discovery = kwargs['discovery'] nstk = kwargs['nstk'] rejected = kwargs.get('rejected', False) zpt = kwargs.get('zpt', 26.9) dbimages = kwargs.get('dbimages', 'vos:NewHorizons/dbimages/') client = Client() int_rate = int(rate * 10) int_angle = int((angle % 360) * 10) images = [] # Load the 3 images associated with this point/chip/rate/angle set. epoch = None for idx in range(nstk): expnum = f'{int(pointing)}{int_rate:02d}{int_angle:04d}{idx}' image = f'{expnum}p{chip:02d}.fits' url = f'{dbimages}/{pointing:05d}/{chip:03d}/{index:04d}/{image}' logging.info(f"Looking for image at {url}") try: if os.access(url, os.R_OK): image = url elif not os.access(image, os.R_OK): # get from VOSpace is not already on disk client.copy(url, image) except Exception as ex: logging.error(str(ex)) # Return empty set on VOSpace copy error. return {} images.append(image) epoch = Time(fits.open(images[len(images) // 2])[0].header['DATE-AVG'], scale='tai').utc regions = f'{dbimages}/{pointing:05d}.reg' try: if not os.access(regions, os.R_OK): regions = client.copy(regions, '.', disposition=True) except Exception as ex: logging.debug(f"{ex}") regions = None wcs_dict = {} if orbit is not None: epoch = orbit.epoch.mjd if epoch is None: epoch = Time("2020-05-22T00:00:00") epoch = Time(kwargs.get('epoch', epoch), format='mjd') load_images(images, ra, dec, wcs_dict, orbit, dra=rate * math.cos(math.radians(angle)), ddec=rate * math.sin(math.radians(angle)), regions=regions, rejected=rejected, basedate=epoch) obs = measure_image(p_name, images, wcs_dict, discovery=discovery, zpt=zpt) return obs
def diag_t(eventfile, par_list, tbin_size, t1, t2, mode, diag_vars): """ Get the diagnostic plots for a desired time interval. eventfile - path to the event file. Will extract ObsID from this for the NICER files. par_list - A list of parameters we'd like to extract from the FITS file (e.g., from eventcl, PI_FAST, TIME, PI,) tbin_size - the size of the time bins (in seconds!) >> e.g., tbin_size = 2 means bin by 2s >> e.g., tbin_size = 0.05 means bin by 0.05s! t1 - lower time boundary t2 - upper time boundary mode - whether we want to show or save the plot. diag_vars - a dictionary where each key = 'att','mkf','hk', or 'cl', and diag_vars[key] provides the list of variables to loop over. """ if type(tbin_size) != int and type(tbin_size) != np.float: raise TypeError("tbin_size should be a float or integer!") if 'PI' and 'TIME' not in par_list: raise ValueError( "You should have BOTH 'PI' and 'TIME' in the parameter list!") if type(par_list) != list and type(par_list) != np.ndarray: raise TypeError("par_list should either be a list or an array!") if mode != 'show' and mode != 'save': raise ValueError("Mode should either be 'show' or 'save'!") parent_folder = str(pathlib.Path(eventfile).parent) event_header = fits.open(eventfile)[1].header obj_name = event_header['OBJECT'] obsid = event_header['OBS_ID'] #get the binned light curve binned_t, binned_counts = Lv1_data_bin.binning_t(eventfile, par_list, tbin_size, t1, t2) #define the variables that we'd like to compare their behavior with the light curve att_var = diag_vars['att'] mkf_var = diag_vars['mkf'] hk_var = diag_vars['hk'] ### FOR ATTITUDE dict_att = Lv0_nicer_housekeeping.get_att(eventfile, att_var) times_att = dict_att['TIME'] shifted_t_att = times_att - times_att[0] filtered_t = shifted_t_att[(shifted_t_att >= t1) & (shifted_t_att <= t2)] for i in range(1, len(att_var)): #as in, don't compare time with time... filtered_att = dict_att[att_var[i]][(shifted_t_att >= t1) & (shifted_t_att <= t2)] if len(filtered_t) != len(filtered_att): raise ValueError( "The lengths of arrays filtered t and filtered att for variable " + str(att_var[i]) + ' are different, with ' + str(len(filtered_t)) + ' and ' + str(len(filtered_att)) + ' respectively.') if mode == 'show': Lv3_diagnostics_display.display_t(eventfile, att_var[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_att, '.att') plt.show() if mode == 'save': filename = parent_folder + '/diag_att_' + obsid + '_bin' + str( tbin_size) + 's_' + str(t1) + 's-' + str(t2) + 's.pdf' with PdfPages(filename) as pdf: for i in range(1, len(att_var)): filtered_att = dict_att[att_var[i]][(shifted_t_att >= t1) & (shifted_t_att <= t2)] Lv3_diagnostics_display.display_t(eventfile, att_var[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_att, '.att') pdf.savefig() plt.close() ### FOR FILTER dict_mkf = Lv0_nicer_housekeeping.get_mkf(eventfile, mkf_var) times_mkf = dict_mkf['TIME'] shifted_t_mkf = times_mkf - times_mkf[0] filtered_t = shifted_t_mkf[(shifted_t_mkf >= t1) & (shifted_t_mkf <= t2)] for i in range(1, len(mkf_var)): #as in, don't compare time with time... filtered_mkf = dict_mkf[mkf_var[i]][(shifted_t_mkf >= t1) & (shifted_t_mkf <= t2)] if len(filtered_t) != len(filtered_mkf): raise ValueError( "The lengths of arrays filtered t and filtered mkf for variable " + str(mkf_var[i]) + ' are different, with ' + str(len(filtered_t)) + ' and ' + str(len(filtered_mkf)) + ' respectively.') if mode == 'show': Lv3_diagnostics_display.display_t(eventfile, mkf_var[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_mkf, '.mkf') plt.show() if mode == 'save': filename = parent_folder + '/diag_mkf_' + obsid + '_bin' + str( tbin_size) + 's_' + str(t1) + 's-' + str(t2) + 's.pdf' with PdfPages(filename) as pdf: for i in range(1, len(mkf_var)): filtered_mkf = dict_mkf[mkf_var[i]][(shifted_t_mkf >= t1) & (shifted_t_mkf <= t2)] Lv3_diagnostics_display.display_t(eventfile, mkf_var[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_mkf, '.mkf') pdf.savefig() plt.close() ### FOR HK if mode == 'show': for i in range(7): dict_hk = Lv0_nicer_housekeeping.get_hk(eventfile, str(i), hk_var) times_hk = dict_hk['TIME'] shifted_t_hk = times_hk - times_hk[0] filtered_t = shifted_t_hk[(shifted_t_hk >= t1) & (shifted_t_hk <= t2)] for j in range( 1, len(hk_var)): #as in, don't compare time with time... filtered_hk = dict_hk[hk_var[j]][(shifted_t_hk >= t1) & (shifted_t_hk <= t2)] if len(filtered_t) != len(filtered_hk): raise ValueError( "The lengths of arrays filtered t and filtered att for variable " + str(hk_var[j]) + ' are different, with ' + str(len(filtered_t)) + ' and ' + str(len(filtered_hk)) + ' respectively. This is for HK MPU=' + str(i)) Lv3_diagnostics_display.display_t(obsid, hk_var[j], t1, t2, binned_t, binned_counts, filtered_t, filtered_hk, ['.hk', str(i)]) plt.show() if mode == 'save': filename = parent_folder + '/diag_hk_' + obsid + '_bin' + str( tbin_size) + 's_' + str(t1) + 's-' + str(t2) + 's.pdf' with PdfPages(filename) as pdf: for i in range(7): dict_hk = Lv0_nicer_housekeeping.get_hk( eventfile, str(i), hk_var) times_hk = dict_hk['TIME'] shifted_t_hk = times_hk - times_hk[0] filtered_t = shifted_t_hk[(shifted_t_hk >= t1) & (shifted_t_hk <= t2)] for j in range( 1, len(hk_var)): #as in, don't compare time with time... filtered_hk = dict_hk[hk_var[j]][(shifted_t_hk >= t1) & (shifted_t_hk <= t2)] if len(filtered_t) != len(filtered_hk): raise ValueError( "The lengths of arrays filtered t and filtered att for variable " + str(hk_var[j]) + ' are different, with ' + str(len(filtered_t)) + ' and ' + str(len(filtered_hk)) + ' respectively. This is for HK MPU=' + str(i)) Lv3_diagnostics_display.display_t(eventfile, hk_var[j], t1, t2, binned_t, binned_counts, filtered_t, filtered_hk, ['.hk', str(i)]) pdf.savefig() plt.close() ### FOR EVENT_CL (BARY) data_dict = Lv0_fits2dict.fits2dict(eventfile, 1, par_list) times_cl = data_dict['TIME'] shifted_t_cl = times_cl - times_cl[0] filtered_t = shifted_t_cl[(shifted_t_cl >= t1) & (shifted_t_cl <= t2)] for i in range(1, len(par_list)): #as in, don't compare time with time... filtered_cl = data_dict[par_list[i]][(shifted_t_cl >= t1) & (shifted_t_cl <= t2)] if len(filtered_t) != len(filtered_cl): raise ValueError( "The lengths of arrays filtered t and filtered cl for variable " + str(eventcl_var[i]) + ' are different, with ' + str(len(filtered_t)) + ' and ' + str(len(filtered_cl)) + ' respectively.') if mode == 'show': Lv3_diagnostics_display.display_t(eventfile, par_list[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_cl, '.cl') plt.show() if mode == 'save': filename = parent_folder + '/diag_cl_' + obsid + '_bin' + str( tbin_size) + 's_' + str(t1) + 's-' + str(t2) + 's.pdf' with PdfPages(filename) as pdf: for i in range(1, len(par_list)): filtered_cl = data_dict[par_list[i]][(shifted_t_cl >= t1) & (shifted_t_cl <= t2)] Lv3_diagnostics_display.display_t(eventfile, par_file[i], t1, t2, binned_t, binned_counts, filtered_t, filtered_cl, '.cl') pdf.savefig() plt.close()
path = '/Users/baotong/Desktop/period_gc/' p90_list = 'mod_expweighted_90_mean_i4.psfmap' srcname_list = path + 'combineobs_info_box.txt' phy_x = np.loadtxt(srcname_list)[:, 2] phy_y = np.loadtxt(srcname_list)[:, 3] phy_x = np.rint(phy_x) phy_y = np.rint(phy_y) phy_x_int = phy_x.astype(np.int) phy_y_int = phy_y.astype(np.int) src_x = phy_x_int - 2896 src_y = phy_y_int - 2896 #将physical坐标转换为img坐标 hdul_p90 = fits.open(path + p90_list) p90_data = hdul_p90[0].data p90_data = p90_data.T src_radius = p90_data[src_x, src_y] src_radius *= 2.03252 #src_radius*=2 #pixel size=0.492arcsec #the default units of p90 fits is arcsec, 1/0.492=2.03252 print(np.sort(src_radius)) for i in range(len(phy_x)): with open(path + 'region/{0}.reg'.format(i + 1), 'w+') as f1: reg = 'circle(' + str(phy_x[i]) + ',' + str(phy_y[i]) + ',' + str( src_radius[i]) + ')' f1.writelines(reg) with open(path + 'region/all.reg', 'a+') as f2: