def transform_grid(ACT_fits, optical_fits): import tableio from astropy.io.fits import getheader import numpy header = getheader(ACT_fits) nx = header['NAXIS1'] ny = header['NAXIS2'] (iy, ix) = numpy.indices((ny, nx)) # flatten (x,y) indices arrays and add 1.0 and put in a tmp file x = ix.ravel() + 1 y = iy.ravel() + 1 tableio.put_data('/tmp/xy_file', (x, y), format="%7d %7d") # system call for xy2sky and sky2xy cmd1 = "xy2sky -d %s @/tmp/xy_file > /tmp/radec_file" % ACT_fits cmd2 = "sky2xy %s @/tmp/radec_file > /tmp/xynew_file" % optical_fits os.system(cmd1) os.system(cmd2) # Read in new grid and re-shape (ixnew, iynew) = tableio.get_data('/tmp/xynew_file', (4, 5)) ix_new = ixnew.reshape(ix.shape) - 1. iy_new = iynew.reshape(iy.shape) - 1. # Clean up files os.remove('/tmp/xy_file') os.remove('/tmp/radec_file') os.remove('/tmp/xynew_file') return ix_new, iy_new
def run(self): """Make the multicolor catalog.""" # define the zero point error zpoint_err = 0.01 self.logfile.write('Processing catalogs...') detection_variables = tableio.get_data(self.detCatalog,self.detectionColumns) nsources = len(detection_variables[0]) ncats = len(self.catalogList) #Create a series of arrays to keep the relevant data flux = Numeric.zeros((ncats,nsources))*1.0 # ISO flux ... for now. fluxerr = Numeric.zeros((ncats,nsources))*1.0 # ISO flux error... for now. m = Numeric.zeros((ncats,nsources))*1.0 # MAG_ISO em = Numeric.zeros((ncats,nsources))*1.0 # MAGERR_ISO m_corr = Numeric.zeros((ncats,nsources))*1.0 # extinction corrected magnitude em_corr = Numeric.zeros((ncats,nsources))*1.0 # extinction corrected magnitude error ap_corr = Numeric.zeros((ncats,nsources))*1.0 # aperture correction m_bpz = Numeric.zeros((ncats,nsources))*1.0 # MAG_BPZ extinction/aperture corrected em_bpz = Numeric.zeros((ncats,nsources))*1.0 # MAGERR_BPZ extinction/aperture corrected preds = [os.path.basename(self.detCatalog)] # this is a list of those indices of the catalogList which raised the filterError exception # i think i need to know this, but i'm not sure why yet.... self.badcats = [] for i in range(ncats): selectSet = {} pardict = {} imname = os.path.splitext(self.catalogList[i])[0] basefits= imname + '.fits' fitsfile= os.path.join(self.obsFits,basefits) detector= fUtil.getDetector(fitsfile) try: imfilter= fUtil.filterResponseName(fitsfile) except fUtil.filterError: self.badcats.append(i) imfilter = fUtil.twoFilterName(fitsfile) try: filterXCorr = extinction.filterFactor(imfilter) * self.eBV filterXCorrError = filterXCorr*0.16 except fUtil.filterError,err: self.errorList.append(("extintion.py","filterError: "+str(err))) self.logfile.write("filterError: "+str(err)) filterXCorr = 0.0 filterXCorrError = 0.0 try: filter = fUtil.filterName(fitsfile) ap_corr[i,:] = self._apcorr(detector,filter) except KeyError,err: print str(err) self.logfile.write(str(err)) # if we can't do it, well, we can't do it. pass
def KEfit(modelfile): import scipy import scipy.interpolate import tableio sout.write("# Getting K(z) and Ev(z) corrections from file: %s\n" % modelfile) e = {} k = {} (z, k_g, k_r, k_i, k_z, e_g, e_r, e_i, e_z) = tableio.get_data(modelfile, cols=(0, 12, 13, 14, 15, 17, 18, 19, 20)) # K-only correction at each age SED, k['g'] = scipy.interpolate.interp1d(z, k_g) k['r'] = scipy.interpolate.interp1d(z, k_r) k['i'] = scipy.interpolate.interp1d(z, k_i) k['z'] = scipy.interpolate.interp1d(z, k_z) # Evolution term alone e['g'] = scipy.interpolate.interp1d(z, e_g) e['r'] = scipy.interpolate.interp1d(z, e_r) e['i'] = scipy.interpolate.interp1d(z, e_i) e['z'] = scipy.interpolate.interp1d(z, e_z) return k, e
def KEfit(modelfile): import scipy import scipy.interpolate import tableio print("# Getting K(z) and Ev(z) corrections from file: %s\n" % modelfile) e = {} k = {} c = {} (z, c_gr, c_ri, c_iz, k_g, k_r, k_i, k_z, e_g, e_r, e_i, e_z) = tableio.get_data(modelfile, cols=(0, 3, 4, 5, 10, 11, 12, 13, 14, 15, 16, 17)) # K-only correction at each age SED, k['g'] = scipy.interpolate.interp1d(z, k_g) k['r'] = scipy.interpolate.interp1d(z, k_r) k['i'] = scipy.interpolate.interp1d(z, k_i) k['z'] = scipy.interpolate.interp1d(z, k_z) # Evolution term alone e['g'] = scipy.interpolate.interp1d(z, e_g) e['r'] = scipy.interpolate.interp1d(z, e_r) e['i'] = scipy.interpolate.interp1d(z, e_i) e['z'] = scipy.interpolate.interp1d(z, e_z) # Color redshift c['gr'] = scipy.interpolate.interp1d(z, c_gr) c['ri'] = scipy.interpolate.interp1d(z, c_ri) c['iz'] = scipy.interpolate.interp1d(z, c_iz) return k, e, c
def match_cats(self, field, dmax=4.0): matchcat = os.path.join(self.outpath, "Catalogs", "{}{}.mch".format(field, self.filter[-1])) with open(matchcat, 'w') as mch: (x1, y1, m1) = tableio.get_data(self.GaList, cols=(0, 1, 2)) (x2, y2, m2) = tableio.get_data(self.SExCat, cols=(1, 2, 3)) idx = [] ## Find matches for GaList on SEx one for i in range(len(x1)): d = numpy.sqrt((x1[i] - x2)**2 + (y1[i] - y2)**2) if d.min() < dmax: ix = numpy.argmin(d) idx.append(ix) mch.write( "%8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f\n" % (x1[i], y1[i], m1[i], x2[ix], y2[ix], m2[ix], d[ix])) print("# Matched cat on %s" % matchcat) idx = numpy.array(idx) print("# Matched %s/%s" % (len(idx), self.Ngal)) return
def _apcorrSetup(self,dcat,detectionSelectSet): """private method called by the constructor to help initialise aperture correction stuff. This method reads the detection catalog and greps the ISOAREA_IMAGE data out of that catalog. This method then converts the areas for the objects in the catalog into radii and returns a Numeric array to the constructor. These ISO areas are the same for all filters so this only has to happen once, hence the constructor calls this. The catalog arg passed is the detection catalog which is just a readlines() list, the second arg is the selectSet for the detection catalog header. """ iradList = [] isoAreaIndex = detectionSelectSet["ISOAREA_IMAGE"] - 1 # index is one less than the col no. isoAreaData = tableio.get_data(self.detCatalog,isoAreaIndex) for area in isoAreaData: iradList.append(max(2.0 ,(math.sqrt(area/math.pi)))) return iradList
def evolfit(modelfile): import scipy import scipy.interpolate import tableio e = {} (z, e_g, e_r, e_i, e_z) = tableio.get_data(modelfile, cols=(0, 14, 15, 16, 17)) e['g'] = scipy.interpolate.interp1d(z, e_g) e['r'] = scipy.interpolate.interp1d(z, e_r) e['i'] = scipy.interpolate.interp1d(z, e_i) e['z'] = scipy.interpolate.interp1d(z, e_z) return e
def sky2xy_list(ra, dec, fitsfile): import os import tableio inlist = "/tmp/%s_list.sky2xy" % os.environ['USER'] outlist = "/tmp/%s_list.sky2xy.out" % os.environ['USER'] tableio.put_data(inlist, (ra, dec), header='', format="%s %s J2000", append='no') cmd = "sky2xy %s @%s > %s" % (fitsfile, inlist, outlist) os.popen(cmd) (x, y) = tableio.get_data(outlist, cols=(4, 5)) #print cmd os.system("rm %s" % inlist) os.system("rm %s" % outlist) return x, y
def _apcorr(self, detector, filter): """Calculate the aperture corrections for this dataset for the given detector and filter. This is a factor to be added to the magnitudes calculated above array [m]. This "apcor" is to be in magnitudes and apcor must be <= 0. This new magnitude or array of magnitudes is then written into the multicolor.cat file and presented to BPZ as the magnitude to use, as indicated in the multicolor.columns file. Function returns a numeric array of aperture corrections. The fiducial radius specified herein is specified in Bugzilla bug #2708, as are the other specifications for this method. """ apcorr = [] apcorrArray = None irad_fiducial = 14 aperFiles = { "UVIS": os.path.join(self.aperData, "newUVIS_2011.ee_new_csky.dat"), "IR": os.path.join(self.aperData, "newIR_1210.ee_new_csky.dat"), "WFC": os.path.join(self.aperData, "newWFC_0803.ee_new_csky.dat"), #{}"HRC" : os.path.join(self.aperData,"newHRC_0803.ee_new_csky.dat") } try: aperDataFile = aperFiles[detector] except KeyError: raise KeyError, "No known aperture data for detector, " + detector # now get the filter aperture data for the indicated detector. # raise a filterError if the lookup fails. aperCatalog = open(aperDataFile).readlines() selectSet = pUtil.makeHeaderDict(aperCatalog) colName = "EE_" + filter try: aperIndex = selectSet[colName] - 1 except KeyError: raise fUtil.filterError, "No aperture data for filter " + filter aperData = tableio.get_data(aperDataFile, aperIndex) # the aperData contains the encircled energy as a function of radius in # pixels. Conveniently, the pixel radius = index + 1 of the value we # want from the array for that radius. eeFiducial = aperData[irad_fiducial - 1] for irad in self.iradList: if irad > irad_fiducial: apcorr.append(0) continue # irad1, irad2 are the adjacent integers of the radius of the object. # used to do a linear interpolation on the encircled energy curve. irad1 = int(irad) irad2 = irad1 + 1 iradFrac = irad - irad1 ee_irad1 = aperData[irad1 - 1] ee_irad2 = aperData[irad2 - 1] k = (ee_irad2 - ee_irad1) ee_irad = k * iradFrac + ee_irad1 apcorr.append(min(0, +2.5 * math.log10(ee_irad / eeFiducial))) apcorrArray = Numeric.array(apcorr) return apcorrArray
def run(self, band): """Make the multicolor catalog.""" # define the zero point error zpoint_err = 0.01 self.logfile.write('Processing catalogs...') detection_variables = tableio.get_data(self.detCatalog, self.detectionColumns) nsources = len(detection_variables[0]) # WZ Should be a dimension check here with the filter catalogs before crashing ncats = len(self.catalogList) #Create a series of arrays to keep the relevant data flux = Numeric.zeros((ncats, nsources)) * 1.0 # ISO flux ... for now. fluxerr = Numeric.zeros( (ncats, nsources)) * 1.0 # ISO flux error... for now. m = Numeric.zeros((ncats, nsources)) * 1.0 # MAG_ISO em = Numeric.zeros((ncats, nsources)) * 1.0 # MAGERR_ISO m_corr = Numeric.zeros( (ncats, nsources)) * 1.0 # extinction corrected magnitude em_corr = Numeric.zeros( (ncats, nsources)) * 1.0 # extinction corrected magnitude error ap_corr = Numeric.zeros( (ncats, nsources)) * 1.0 # aperture correction m_bpz = Numeric.zeros( (ncats, nsources)) * 1.0 # MAG_BPZ extinction/aperture corrected em_bpz = Numeric.zeros( (ncats, nsources)) * 1.0 # MAGERR_BPZ extinction/aperture corrected preds = [os.path.basename(self.detCatalog)] # this is a list of those indices of the catalogList which raised the filterError exception # i think i need to know this, but i'm not sure why yet.... self.badcats = [] for i in range(ncats): selectSet = {} pardict = {} imname = os.path.splitext(self.catalogList[i])[0] j = string.rfind(imname, '_') basefits = imname[0:j] + '_sci.fits' fitsfile = os.path.join(self.obsFits, basefits) detector = fUtil.getDetector(fitsfile) try: imfilter = fUtil.filterResponseName(fitsfile) except fUtil.filterError: self.badcats.append(i) imfilter = fUtil.twoFilterName(fitsfile) try: filterXCorr = extinction.filterFactor(imfilter) * self.eBV filterXCorrError = filterXCorr * 0.16 except fUtil.filterError, err: self.errorList.append( ("extintion.py", "filterError: " + str(err))) self.logfile.write("filterError: " + str(err)) filterXCorr = 0.0 filterXCorrError = 0.0 try: f = pyfits.open(fitsfile) # WZ instr = f[0].header.get('INSTRUME') f.close() del f if instr == 'ACS': #filter = 'F814W' #WZ filter = fUtil.filterNameACS(fitsfile) elif instr == 'WFC3': #filter = fUtil.filterName(fitsfile) filter = fUtil.filterNameWFC3(fitsfile) if filter: print "Find filter ", filter else: raise filterError, "No filter found for fits image: " + fitsfile #filter = fUtil.filterName(fitsfile) ap_corr[i, :] = self._apcorr(detector, filter) except KeyError, err: print str(err) self.logfile.write(str(err)) # if we can't do it, well, we can't do it. pass
# i.e the catalog column numbers start at 1 not zero. catalogObj = open(catalog).readlines() selectSet = pUtil.makeHeaderDict(catalogObj) #Info for flux columns fluxList = [] fluxList.append(selectSet['FLUX_ISO'] - 1) fluxList.append(selectSet['FLUXERR_ISO'] - 1) self.fluxColumns = tuple( fluxList) # the get_data function interface requires a tuple # Build the various columns arrays with the get_data function. # We read raw fluxes and errors into the flux,fluxerr arrays. # They are afterwards transformed to magnitudes t1, t2 = tableio.get_data(catalog, self.fluxColumns) if (len(t1) != nsources): self.logfile.write("Catalog dimension mismatch: ", str(len(t1)), ' ', nsources) self.logfile.write( "Check patrameters in detectionCatalog.inpar and filterCatalog.inpar" ) #flux[i,:],fluxerr[i,:] = tableio.get_data(catalog,self.fluxColumns) flux[i, :], fluxerr[i, :] = t1, t2 flux[i, :] = pUtil.deNAN(flux[i, :]) # Those objects with flux equal or less than 0 are assigned a magnitude of 99 # and a limiting magnitude equal to their SExtractor photometric error. This # is interpreted by BPZ as a nondetection with zero flux and 1-sigma error # equal to the limiting magnitude
catalogObj = open(catalog).readlines() selectSet = pUtil.makeHeaderDict(catalogObj) #Info for flux columns fluxList = [] fluxList.append(selectSet['FLUX_ISO'] - 1) fluxList.append(selectSet['FLUXERR_ISO']- 1) self.fluxColumns = tuple(fluxList) # the get_data function interface requires a tuple # Build the various columns arrays with the get_data function. # We read raw fluxes and errors into the flux,fluxerr arrays. # They are afterwards transformed to magnitudes pdb.set_trace() #xingxing flux[i,:],fluxerr[i,:] = tableio.get_data(catalog,self.fluxColumns) flux[i,:] = pUtil.deNAN(flux[i,:]) # Those objects with flux equal or less than 0 are assigned a magnitude of 99 # and a limiting magnitude equal to their SExtractor photometric error. This # is interpreted by BPZ as a nondetection with zero flux and 1-sigma error # equal to the limiting magnitude nondetected = Numeric.less_equal(flux[i,:],0.0)*Numeric.greater(fluxerr[i,:],0.0) # Those objects with error flux and flux equal to 0 are assigned a magnitude of -99 # and a flux of 0, which is interpreted by SExtractor as a non-observed object nonobserved = Numeric.less_equal(fluxerr[i,:],0.0) # When flux error > 100*(flux), mark as nonobserved (Benitez, 24-Oct-03).
def read_dndM(self): (M, dndM) = tableio.get_data(self.dndMFile, cols=(0, 1)) os.remove(self.dndMFile) return M, dndM
def BuildColorCat(tilename, combcat, filters=['g', 'r', 'i', 'z', 'K'], newfirm=True): # The default output names colorCat = tilename + "_complete.catalog" print('Processing catalogs... for: ', tilename, file=sys.stderr) flux = {} fluxerr = {} m = {} em = {} # Get the detection catalog required columns outColumns = ['NUMBER', 'X_IMAGE', 'Y_IMAGE'] detCatalog = combcat['i'] detcols = SEx_head(detCatalog, verb=None) detectionList = [] for key in outColumns: detectionList.append(detcols[key]) # the get_data function requires a tuple detectionColumns = tuple(detectionList) detection_variables = tableio.get_data(detCatalog, detectionColumns) # Read in the MAG_ISO and MAG_ISOERR from each catalog for filter in filters: if not newfirm and filter == 'K': continue # get the zeropoint Info tmp = np.genfromtxt('photometry_control_star_{}.dat'.format( filter), names=True, dtype=None) zpoint = tmp['ZP'] # Get the columns sexcols = SEx_head(combcat[filter], verb=None) ## Info for flux columns fluxList = [] fluxList.append(sexcols['FLUX_ISO']) fluxList.append(sexcols['FLUXERR_ISO']) fluxColumns = tuple( fluxList) # the get_data function interface requires a tuple # Get the array using tableio flux[filter], fluxerr[filter] = tableio.get_data(combcat[filter], fluxColumns) m[filter] = flux[filter] * 0.0 em[filter] = flux[filter] * 0.0 # Fix the NAN values flux[filter] = deNAN(flux[filter]) # Those objects with flux equal or less than 0 are assigned a # magnitude of 99 and a limiting magnitude equal to their # SExtractor photometric error. This is interpreted by BPZ as a # nondetection with zero flux and 1-sigma error equal to the # limiting magnitude #nondetected = np.less_equal(flux[filter], 0.0) * \ # np.greater(fluxerr[filter], 0.0) # update: There are a lot of really small positive values. I am # going to modify this to look for values really close to zero. nondetected = (flux[filter] < 1E-3) & (fluxerr[filter] > 0.0) # Those objects with error flux and flux equal to 0 are assigned a # magnitude of -99 # and a flux of 0, which is interpreted by SExtractor as a # non-observed object nonobserved = np.less_equal(fluxerr[filter], 0.0) # When flux error > 100*(flux), mark as nonobserved (Benitez, # 24-Oct-03). nonobserved = np.where(fluxerr[filter] > 100 * (abs(flux[filter])), True, nonobserved) detected = np.logical_not(nonobserved + nondetected) print(filter, zpoint) flux[filter] = np.clip(flux[filter], 1e-100, 1e100) m[filter] = np.where(detected, -2.5 * np.log10(abs(flux[filter])) + zpoint, m[filter]) m[filter] = np.where(nondetected, 99.0, m[filter]) m[filter] = np.where(nonobserved, -99.0, m[filter]) # clip values from being too small or large, i.e. 0 or inf. fluxerr[filter] = np.clip(fluxerr[filter], 1e-100, 1e100) em[filter] = np.where( detected, 2.5 * np.log10(1.0 + abs(fluxerr[filter] / flux[filter])), em[filter]) em[filter] = np.where( nondetected, 2.5 * np.log10(abs(fluxerr[filter])) - zpoint, em[filter]) em[filter] = np.where(nonobserved, 0.0, em[filter]) if filter == 'K': outColumns.append(filter + '_KittPeak_MAG_ISO') outColumns.append(filter + '_KittPeak_MAGERR_ISO') else: outColumns.append(filter + '_MOSAICII_MAG_ISO') outColumns.append(filter + '_MOSAICII_MAGERR_ISO') # Prepare the header header = \ '## ' + '\n' + \ '## BPZ Catalog file for Observation: ' + tilename + \ '\n' + \ '## (This file was generated automatically by' + \ 'the BCS Rutgers pipeline)\n##\n' for i in range(len(outColumns)): header = header + '# ' + str(i + 1) + '\t' + outColumns[i] + '\n' # Prepare the data vars = list(detection_variables) for filter in filters: if not newfirm and filter == 'K': continue vars.append(m[filter]) vars.append(em[filter]) variables = tuple(vars) format = '%i\t %10.2f %10.2f' + '%10.4f ' * (len(variables) - 3) print('Writing data to multicolor catalog...', file=sys.stderr) tableio.put_data(colorCat, variables, header=header, format=format, append='no') print('Multicolor catalog complete.', file=sys.stderr) return
def read_cat(self): t1 = time.time() cols = (1, 2, 23, 27, 26, 28, 29, 30, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16, 17, 18, 19, 20, 21, 22, 31, 32, 33, 34, 35, 36) sout.write("# Reading cols:%s\n# Reading cats from: %s... \n" % (cols, self.catsfile)) (ra, dec, z_b, odds, t_b, z_ml, t_ml, chi, g, g_err, r, r_err, i, i_err, z, z_err, g_bpz, g_berr, r_bpz, r_berr, i_bpz, i_berr, z_bpz, z_berr, class_star, a_image, b_image, theta, x_image, y_image) = tableio.get_data(self.catsfile, cols=cols) (id) = tableio.get_str(self.catsfile, cols=(0, )) ############################################ # Choose the photo-z to use, ml or bayesian ############################################ sout.write("# Will use %s redshifts\n" % self.zuse) if self.zuse == "ML": z_ph = z_ml # t = t_ml elif self.zuse == "ZB": z_ph = z_b # t = t_b i_lim = self.maglim odds_lim = 0.80 # not currently used star_lim = self.starlim # Clean up according to BPZ sout.write("# Avoiding magnitudes -99 and 99 in BPZ \n") g_mask = numpy.where(lor(g_bpz == 99, g_bpz == -99), 0, 1) r_mask = numpy.where(lor(r_bpz == 99, r_bpz == -99), 0, 1) i_mask = numpy.where(lor(i_bpz == 99, i_bpz == -99), 0, 1) z_mask = numpy.where(lor(z_bpz == 99, z_bpz == -99), 0, 1) bpz_mask = g_mask * r_mask * i_mask * z_mask # Clean up to avoid 99 values and very faint i_mag values sout.write("# Avoiding magnitudes 99 in MAG_AUTO \n") #g_mask = numpy.where( g >= 99, 0 , 1) #r_mask = numpy.where( r >= 99, 0 , 1) #i_mask = numpy.where( i >= i_lim, 0 , 1) #z_mask = numpy.where( z >= 99, 0 , 1) sout.write("# Avoiding magnitudes i > %s in MAG_AUTO \n" % i_lim) # Clean by class_star sout.write("# Avoiding CLASS_STAR > %s \n" % star_lim) mask_star = numpy.where(class_star > star_lim, 0, 1) # Clean up by odds #sout.write( "# Avoiding ODDS < %s in BPZ \n" % odds_lim) odds_mask = numpy.where(odds > odds_lim, 1, 0) odds_mask = 1 # Avoid z> zlim objects too. #sout.write( "# Avoiding objects with z > %s " % self.zlim) zp_mask = numpy.where(z_ph > self.zlim, 0, 1) zp_mask = 1 # Clean up by BPZ type # sout.write('# Avoiding objects with type > %s' % t) tp_mask = 1 # The final 'good' mask mask_good = bpz_mask * odds_mask * mask_star * zp_mask * tp_mask idx = numpy.where(mask_good == 1) # Make ids a Char String in numarray self.id = nstr.array(id)[idx] # Only keep the 'good' one, avoid -99 and 99 values in BPZ mags self.ra = ra[idx] self.dec = dec[idx] self.z_b = z_b[idx] self.odds = odds[idx] self.z_ml = z_ml[idx] self.t_ml = t_ml[idx] self.t_b = t_b[idx] self.t_ml = t_ml[idx] self.chi = chi[idx] ############################################ # Choose the photo-z to use, ml or bayesian ############################################ if self.zuse == "ML": self.z_ph = self.z_ml self.type = self.t_ml elif self.zuse == "ZB": self.z_ph = self.z_b self.type = self.t_b self.g = g[idx] self.r = r[idx] self.i = i[idx] self.z = z[idx] self.g_err = g_err[idx] self.r_err = r_err[idx] self.i_err = i_err[idx] self.z_err = z_err[idx] self.g_bpz = g_bpz[idx] self.r_bpz = r_bpz[idx] self.i_bpz = i_bpz[idx] self.z_bpz = z_bpz[idx] self.g_berr = g_berr[idx] self.r_berr = r_berr[idx] self.i_berr = i_berr[idx] self.z_berr = z_berr[idx] self.class_star = class_star[idx] self.a_image = a_image[idx] self.b_image = b_image[idx] self.theta = theta[idx] self.x_image = x_image[idx] self.y_image = y_image[idx] # Color of selected galaxies self.gr = self.g_bpz - self.r_bpz self.ri = self.r_bpz - self.i_bpz self.iz = self.i_bpz - self.z_bpz # Min and and max values in RA/DEC self.ramin = self.ra.min() self.ramax = self.ra.max() self.decmin = self.dec.min() self.decmax = self.dec.max() self.idx_cat = idx sout.write(" \tDone: %s\n" % extras.elapsed_time_str(t1)) return
def BuildColorCat(self): # Change accordingly zp_error = 0.05 # The default output names self.colorCat = self.tilename + ".color" self.columnsFile = self.tilename + ".columns" print('Processing catalogs... for: ', self.tilename, file=sys.stderr) flux = {} fluxerr = {} m = {} em = {} # Get the detection catalog required columns outColumns = ['NUMBER', 'X_IMAGE', 'Y_IMAGE'] detCatalog = self.combcat['i'] detcols = SEx_head(detCatalog, verb=None) detectionList = [] for key in outColumns: detectionList.append(detcols[key]) detectionColumns = tuple( detectionList) # the get_data function requires a tuple detection_variables = tableio.get_data(detCatalog, detectionColumns) # Read in the MAG_ISO and MAG_ISOERR from each catalog for filter in self.filters: # Get the columns sexcols = SEx_head(self.combcat[filter], verb=None) ## Info for flux columns fluxList = [] fluxList.append(sexcols['FLUX_ISO']) fluxList.append(sexcols['FLUXERR_ISO']) fluxColumns = tuple( fluxList) # the get_data function interface requires a tuple # Get the array using tableio flux[filter], fluxerr[filter] = tableio.get_data( self.combcat[filter], fluxColumns) m[filter] = flux[filter] * 0.0 em[filter] = flux[filter] * 0.0 # Fix the NAN values flux[filter] = deNAN(flux[filter]) # Those objects with flux equal or less than 0 are assigned a magnitude of 99 # and a limiting magnitude equal to their SExtractor photometric error. This # is interpreted by BPZ as a nondetection with zero flux and 1-sigma error # equal to the limiting magnitude nondetected = Numeric.less_equal( flux[filter], 0.0) * Numeric.greater(fluxerr[filter], 0.0) # Those objects with error flux and flux equal to 0 are assigned a magnitude of -99 # and a flux of 0, which is interpreted by SExtractor as a non-observed object nonobserved = Numeric.less_equal(fluxerr[filter], 0.0) # When flux error > 100*(flux), mark as nonobserved (Benitez, 24-Oct-03). # Fix for fc11 -- y[:] has change meaning #nonobserved = Numeric.where(fluxerr[filter] > 100*(abs(flux[filter])),1.0,nonobserved[:]) nonobserved = Numeric.where( fluxerr[filter] > 100 * (abs(flux[filter])), 1.0, nonobserved) detected = Numeric.logical_not(nonobserved + nondetected) # Get the zero point for the final magnitudes zpoint = self.magbase print(filter, zpoint) flux[filter] = Numeric.clip(flux[filter], 1e-100, 1e100) m[filter] = Numeric.where( detected, -2.5 * Numeric.log10(abs(flux[filter])) + zpoint - self.XCorr[filter], m[filter]) m[filter] = Numeric.where(nondetected, 99.0, m[filter]) m[filter] = Numeric.where(nonobserved, -99.0, m[filter]) # clip values from being too small or large, i.e. 0 or inf. fluxerr[filter] = Numeric.clip(fluxerr[filter], 1e-100, 1e100) em[filter] = Numeric.where( detected, 2.5 * Numeric.log10(1.0 + abs(fluxerr[filter] / flux[filter])) + self.XCorrError[filter], em[filter]) em[filter] = Numeric.where( nondetected, 2.5 * Numeric.log10(abs(fluxerr[filter])) - zpoint, em[filter]) em[filter] = Numeric.where(nonobserved, 0.0, em[filter]) #outColumns.append(filter +'_SDSS_MAG_ISO') #outColumns.append(filter +'_SDSS_MAGERR_ISO') outColumns.append(filter + '_MOSAICII_MAG_ISO') outColumns.append(filter + '_MOSAICII_MAGERR_ISO') # Prepare the header header = \ '## ' + time.ctime() + '\n'+\ '## BPZ Catalog file for Observation: ' + self.tilename + '\n'+\ '## (This file was generated automatically by the BCS Rutgers pipeline)\n##\n' for i in range(len(outColumns)): header = header + '# ' + str(i + 1) + '\t' + outColumns[i] + '\n' # Prepare the data vars = list(detection_variables) for filter in self.filters: vars.append(m[filter]) vars.append(em[filter]) variables = tuple(vars) format = '%i\t %10.2f %10.2f' + '%10.4f ' * (len(variables) - 3) print('Writing data to multicolor catalog...', file=sys.stderr) tableio.put_data(self.colorCat, variables, header=header, format=format, append='no') print('Multicolor catalog complete.', file=sys.stderr) # And now write .columns file cfile = open(self.columnsFile, 'w') cfile.write('## ' + time.ctime() + '\n') cfile.write('## ' + 'BPZ' + ' .columns file for Observation: ' + self.tilename + '\n') cfile.write( '## (This file was generated automatically by the BCS Rutgers pipeline)\n##\n' ) i = len(detection_variables) for filter in self.filters: if filter == 'i': n_mo = str(i + 1) colmag = i + 1 colmagerr = i + 2 cfile.write('%s_MOSAICII\t %s,%s\t AB\t %.2f\t 0.0\n' % (filter, i + 1, i + 2, zp_error)) i = i + 2 cfile.write('M_0\t%s\n' % n_mo) cfile.close() return
def prepMatch(reffile, imlist): #WZ """ Calculate shift/rotation new reference: center position """ de = 100. # distance from edge circle = 500. # matching range in pixel rmin = 4. # FWHM minimum rmax = 15. # 10. # 6. # clx = 0.8 # must be a star pardir = '/home/zheng/drex/reffiles/pars' fitsfile = pyfits.open(reffile) detector = fitsfile[0].header.get('DETECTOR') fitsfile.close() #pdb.set_trace() weightfile = reffile.split(".fits")[0] + '_weight.fits' cmd = 'cp ' + weightfile + ' temp_weight.fits' sproc = popen2.Popen3(cmd, 1) output = sproc.fromchild.readlines() errs = sproc.childerr.readlines() if (detector == 'WFC'): cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_wfc.inpar' else: if (detector == 'UVIS'): cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_uvis.inpar' else: cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_ir.inpar' sproc = popen2.Popen3(cmd, 1) output = sproc.fromchild.readlines() errs = sproc.childerr.readlines() cmd = 'cp temp.cat temp0.cat' sproc = popen2.Popen3(cmd, 1) output = sproc.fromchild.readlines() errs = sproc.childerr.readlines() x0, y0, mag0, err0, fwhm0, ra0, dec0, cls0 = tableio.get_data( 'temp.cat', (1, 2, 3, 4, 5, 6, 7, 8)) xmin0 = min(x0) + de xmax0 = max(x0) - de ymin0 = min(y0) + de ymax0 = max(y0) - de #twf = open("tweak.dat","w") format = str('%4d %8.2f %8.2f %6.2f %6.2f %4.1f') #format2=str('%s %6.3f %5.3f %10.7f') for im in imlist: #outfile = im.split("_sci")[0]+'_ini.fits' fitsfile = pyfits.open(im) detector = fitsfile[0].header.get('DETECTOR') fitsfile.close() weightfile = im.split(".fits")[0] + '_weight.fits' cmd = 'cp ' + weightfile + ' temp_weight.fits' sproc = popen2.Popen3(cmd, 1) output = sproc.fromchild.readlines() errs = sproc.childerr.readlines() #pdb.set_trace() if (detector == 'WFC'): cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_wfc.inpar' else: if (detector == 'UVIS'): cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_uvis.inpar' else: cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_ir.inpar' sproc = popen2.Popen3(cmd, 1) output = sproc.fromchild.readlines() errs = sproc.childerr.readlines() if errs: print 'Program choked on ' + cmd else: n, x, y, mag, err, fwhm, ra, dec, cls = tableio.get_data( 'temp.cat', (0, 1, 2, 3, 4, 5, 6, 7, 8)) # pdb.set_trace() flag0 = numpy.zeros(len(x0), dtype=int) flag = numpy.zeros(len(n), dtype=int) index = numpy.zeros(len(n), dtype=int) xmin = min(x) + de if (xmin < xmin0): xmin = xmin0 xmax = max(x) - de if (xmax > xmax0): xmax = xmax0 ymin = min(y) + de if (ymin < ymin0): ymin = ymin0 ymax = max(y) - de if (ymax > ymax0): ymax = ymax0 print 'Min/max: ', xmin, xmax, ymin, ymax fname0 = im.split("_drz")[0] + '_1.cat' fname1 = im.split("_drz")[0] + '_2.cat' f0 = open(fname0, "w") f = open(fname1, "w") k = 0 for i in range(len(x)): # if (mag[i] < 90 and fwhm[i] > rmin and fwhm[i] < rmax): if (mag[i] < 90): for j in range(len(x0)): if (abs(x[i] - x0[j]) < circle and abs(y[i] - y0[j]) < circle and mag0[j] < 90 and fwhm0[j] > rmin and fwhm0[j] < rmax): #if (abs(x[i]-x0[j]) < circle and abs(y[i]-y0[j]) < circle and mag0[j] < 90): index[i] = j flag[i] = flag[i] + 1 flag0[j] = flag0[j] + 1 k = k + 1 print k, " sources found in ", im k = m = 0 for i in range(len(x)): if (flag[i] >= 1): #pdb.set_trace() j = index[i] if (flag0[j] >= 1 and fwhm0[j] < rmax and mag[i] < 90): #if (flag0[j] >= 1 and mag[i] < 90): print >> f, format % (k, x[i], y[i], mag[i], err[i], fwhm[i]) print >> f0, format % (k, x0[j], y0[j], mag0[j], err0[j], fwhm0[j]) k = k + 1 else: m = m + 1 print k, " sources used in ", im #,", ",m," missed" #pdb.set_trace() f0.close() f.close() return
def _magFix(self, catalogFile): """This private method receives a path to a catalog file and sifts through the MAGERR field looking for values > 10. It sets the corresponding MAG field = -99 and sets that object's MAGERR field to 0.0. catalogFile is a path not a file object.""" # fillHeader will return a list of tuples where which looks like # # [(1, 'NUMBER'), # (2, 'X_IMAGE'), # (3, 'Y_IMAGE'), # ... # (12, 'MAG_ISOCOR'), # (13, 'MAGERR_ISOCOR'), # (14, 'FLUX_APER', 1) # (15, 'FLUX_APER', 2), # (16, 'FLUX_APER', 3), # ... # ] # # The tuples are either of length 2 or 3. If len is 3, the 3rd item of the # tuple is the nth occurance of that column identifier. This occurs on those # columns of MAGs and MAGERRs for a series of increasingly larger apertures. # newFieldList will be a list of Numeric arrays containing the columns of the catalogs. # This list will contain fields which have not been altered, i.e. all fields other than # MAG_* and MAGERR_*, and the new MAG and MAGERR fields which have been corrected. # Once the list is complete, it is tuple-ized and send to the tableio pu_data function. newFieldList = [] newMagsList = [] newMagErrsList = [] newMagHeaders = [] newMagErrHeaders = [] newHeaders = [] magCols = [] magErrCols = [] selectSet = fillHeader(catalogFile) print "Searching catalog for required columns, MAG, MAGERR" for i in range(len(selectSet)): if len(selectSet[i]) == 2: column, name = selectSet[i] paramNames = name.split("_") if "MAG" in paramNames: magCols.append((column, name)) elif "MAGERR" in paramNames: magErrCols.append((column, name)) else: oldField = tableio.get_data(catalogFile, (column - 1)) newFieldList.append(oldField) newHeaders.append(name) continue else: column, name, id = selectSet[i] paramNames = name.split("_") if "MAG" in paramNames: magCols.append((column, name, id)) elif "MAGERR" in paramNames: magErrCols.append((column, name, id)) else: oldField = tableio.get_data(catalogFile, (column - 1)) newFieldList.append(oldField) newHeaders.append(name) continue # We now have # catalog field --> list # -------------------------------- # MAG_* --> magCols # MAGERR_* --> magErrCols # # The algorithm will be to step through the magErrCols columns, extracting those fields # via get_data and getting Numeric arrays. The matching mag columns are slurped as well. # We search the magErrCols arrays looking for >= 10 values and then marking the those mags # as -99.0 and the matching magerrs as 0.0 # See Bugzilla bug #2700 for item in magErrCols: magErrAperId = None # item may be of len 2 or 3 if len(item) == 2: magErrColId, magErrColName = item else: magErrColId, magErrColName, magErrAperId = item magErrKind = magErrColName.split("_")[1] # ISO, ISOCORR, etc. print "\n\nMAG type:", magErrKind if magErrAperId: print magErrColName, "Aper id is", magErrAperId print "Getting\t", magErrColName, "\tfield", magErrColId # MAGERR array: magErrs = tableio.get_data(catalogFile, magErrColId - 1) matchingMagColName = None matchingMagColId = None #----------------------- Search for matching MAG_* field -----------------------# for magitems in magCols: # We know that the magErrColName is MAGERR and if magErrNameId is true then # the tuple is of len 3, i.e. a MAGERR_APER field. We look for the matching # MAG_APER field id, 1, 2, 3... etc. if len(magitems) == 3: magColId, magColName, magAperId = magitems if magColName == "MAG_" + magErrKind: matchingMagColName = magColName #print "Found matching field type:",magColName,"in field",magColId if magAperId == magErrAperId: print "Found matching aperture id." print "MAG_APER id: ", magAperId, "MAGERR_APER id: ", magErrAperId matchingMagColId = magColId matchingMags = tableio.get_data( catalogFile, magColId - 1) break else: continue else: magColId, magColName = magitems if magColName == "MAG_" + magErrKind: print "Found matching field type:", magColName, "in field", magColId matchingMagColName = magColName matchingMagColId = magColId matchingMags = tableio.get_data( catalogFile, magColId - 1) break else: continue #--------------------------------------------------------------------------------# print " MAG err field:", magErrColName, magErrColId print " Mag field:", matchingMagColName, matchingMagColId # Now the grunt work on the arrays, # magErrs, matchingMags # # update: flagging all MAGs as -99 when the corresponding MAGERR > 10 # introduced a bug which unintentionally reset the magnitudes # SExtractor had flagged with a MAG = 99.0 and a MAGERR = 99.0 # This now checks for a MAGERR of 99 and does not reset the MAG value # if MAGERR = 99.0 but does for all other MAGERRS > 10.0 badMagErrs1 = Numeric.where(magErrs >= 10, 1, 0) badMagErrs2 = Numeric.where(magErrs != 99.0, 1, 0) badMagErrs = badMagErrs1 * badMagErrs2 del badMagErrs1, badMagErrs2 newMags = Numeric.where(badMagErrs, -99.0, matchingMags) newMagErrs = Numeric.where(badMagErrs, 0.0, magErrs) newMagsList.append(newMags) newMagHeaders.append(matchingMagColName) newMagErrsList.append(newMagErrs) newMagErrHeaders.append(magErrColName) # concatenate the lists. This is done to preserve the MAG_APER and MAGERR_APER # grouping of the original SExtractor catalog. newFieldList = newFieldList + newMagsList newFieldList = newFieldList + newMagErrsList newHeaders = newHeaders + newMagHeaders newHeaders = newHeaders + newMagErrHeaders newVariables = tuple(newFieldList) # rename the old catalog file as catalogFile.old os.rename(catalogFile, catalogFile + ".old") self.outputList[os.path.basename(catalogFile) + ".old"] = [os.path.basename(catalogFile)] fob = open(catalogFile, 'w') fob.write("## " + ptime() + "\n") fob.write("## " + self.modName + " catalog regenerated by _magFix method.\n") fob.write( '## (This file was generated automatically by the ACS Pipeline.)\n##\n' ) fob.write( "## This catalog has been photometrically corrected to remove\n") fob.write("## 'bad' magnitude values.\n") fob.write("##\n") for i in range(len(newHeaders)): fob.write("# " + str(i + 1) + "\t" + newHeaders[i] + "\n") fob.close() tableio.put_data(catalogFile, newVariables, append="yes") return