def transform_grid(ACT_fits, optical_fits):
import tableio
from astropy.io.fits import getheader
import numpy
header = getheader(ACT_fits)
nx = header['NAXIS1']
ny = header['NAXIS2']
(iy, ix) = numpy.indices((ny, nx))
# flatten (x,y) indices arrays and add 1.0 and put in a tmp file
x = ix.ravel() + 1
y = iy.ravel() + 1
tableio.put_data('/tmp/xy_file', (x, y), format="%7d %7d")
# system call for xy2sky and sky2xy
cmd1 = "xy2sky -d %s @/tmp/xy_file > /tmp/radec_file" % ACT_fits
cmd2 = "sky2xy %s @/tmp/radec_file > /tmp/xynew_file" % optical_fits
os.system(cmd1)
os.system(cmd2)
# Read in new grid and re-shape
(ixnew, iynew) = tableio.get_data('/tmp/xynew_file', (4, 5))
ix_new = ixnew.reshape(ix.shape) - 1.
iy_new = iynew.reshape(iy.shape) - 1.
# Clean up files
os.remove('/tmp/xy_file')
os.remove('/tmp/radec_file')
os.remove('/tmp/xynew_file')
return ix_new, iy_new
def run(self):
"""Make the multicolor catalog."""
# define the zero point error
zpoint_err = 0.01
self.logfile.write('Processing catalogs...')
detection_variables = tableio.get_data(self.detCatalog,self.detectionColumns)
nsources = len(detection_variables[0])
ncats = len(self.catalogList)
#Create a series of arrays to keep the relevant data
flux = Numeric.zeros((ncats,nsources))*1.0 # ISO flux ... for now.
fluxerr = Numeric.zeros((ncats,nsources))*1.0 # ISO flux error... for now.
m = Numeric.zeros((ncats,nsources))*1.0 # MAG_ISO
em = Numeric.zeros((ncats,nsources))*1.0 # MAGERR_ISO
m_corr = Numeric.zeros((ncats,nsources))*1.0 # extinction corrected magnitude
em_corr = Numeric.zeros((ncats,nsources))*1.0 # extinction corrected magnitude error
ap_corr = Numeric.zeros((ncats,nsources))*1.0 # aperture correction
m_bpz = Numeric.zeros((ncats,nsources))*1.0 # MAG_BPZ extinction/aperture corrected
em_bpz = Numeric.zeros((ncats,nsources))*1.0 # MAGERR_BPZ extinction/aperture corrected
preds = [os.path.basename(self.detCatalog)]
# this is a list of those indices of the catalogList which raised the filterError exception
# i think i need to know this, but i'm not sure why yet....
self.badcats = []
for i in range(ncats):
selectSet = {}
pardict = {}
imname = os.path.splitext(self.catalogList[i])[0]
basefits= imname + '.fits'
fitsfile= os.path.join(self.obsFits,basefits)
detector= fUtil.getDetector(fitsfile)
try:
imfilter= fUtil.filterResponseName(fitsfile)
except fUtil.filterError:
self.badcats.append(i)
imfilter = fUtil.twoFilterName(fitsfile)
try:
filterXCorr = extinction.filterFactor(imfilter) * self.eBV
filterXCorrError = filterXCorr*0.16
except fUtil.filterError,err:
self.errorList.append(("extintion.py","filterError: "+str(err)))
self.logfile.write("filterError: "+str(err))
filterXCorr = 0.0
filterXCorrError = 0.0
try:
filter = fUtil.filterName(fitsfile)
ap_corr[i,:] = self._apcorr(detector,filter)
except KeyError,err:
print str(err)
self.logfile.write(str(err))
# if we can't do it, well, we can't do it.
pass
def KEfit(modelfile):
import scipy
import scipy.interpolate
import tableio
sout.write("# Getting K(z) and Ev(z) corrections from file: %s\n" %
modelfile)
e = {}
k = {}
(z, k_g, k_r, k_i, k_z, e_g, e_r, e_i,
e_z) = tableio.get_data(modelfile,
cols=(0, 12, 13, 14, 15, 17, 18, 19, 20))
# K-only correction at each age SED,
k['g'] = scipy.interpolate.interp1d(z, k_g)
k['r'] = scipy.interpolate.interp1d(z, k_r)
k['i'] = scipy.interpolate.interp1d(z, k_i)
k['z'] = scipy.interpolate.interp1d(z, k_z)
# Evolution term alone
e['g'] = scipy.interpolate.interp1d(z, e_g)
e['r'] = scipy.interpolate.interp1d(z, e_r)
e['i'] = scipy.interpolate.interp1d(z, e_i)
e['z'] = scipy.interpolate.interp1d(z, e_z)
return k, e
def KEfit(modelfile):
import scipy
import scipy.interpolate
import tableio
print("# Getting K(z) and Ev(z) corrections from file: %s\n" % modelfile)
e = {}
k = {}
c = {}
(z, c_gr, c_ri, c_iz, k_g, k_r, k_i, k_z, e_g, e_r, e_i,
e_z) = tableio.get_data(modelfile,
cols=(0, 3, 4, 5, 10, 11, 12, 13, 14, 15, 16, 17))
# K-only correction at each age SED,
k['g'] = scipy.interpolate.interp1d(z, k_g)
k['r'] = scipy.interpolate.interp1d(z, k_r)
k['i'] = scipy.interpolate.interp1d(z, k_i)
k['z'] = scipy.interpolate.interp1d(z, k_z)
# Evolution term alone
e['g'] = scipy.interpolate.interp1d(z, e_g)
e['r'] = scipy.interpolate.interp1d(z, e_r)
e['i'] = scipy.interpolate.interp1d(z, e_i)
e['z'] = scipy.interpolate.interp1d(z, e_z)
# Color redshift
c['gr'] = scipy.interpolate.interp1d(z, c_gr)
c['ri'] = scipy.interpolate.interp1d(z, c_ri)
c['iz'] = scipy.interpolate.interp1d(z, c_iz)
return k, e, c
def match_cats(self, field, dmax=4.0):
matchcat = os.path.join(self.outpath, "Catalogs",
"{}{}.mch".format(field, self.filter[-1]))
with open(matchcat, 'w') as mch:
(x1, y1, m1) = tableio.get_data(self.GaList, cols=(0, 1, 2))
(x2, y2, m2) = tableio.get_data(self.SExCat, cols=(1, 2, 3))
idx = []
## Find matches for GaList on SEx one
for i in range(len(x1)):
d = numpy.sqrt((x1[i] - x2)**2 + (y1[i] - y2)**2)
if d.min() < dmax:
ix = numpy.argmin(d)
idx.append(ix)
mch.write(
"%8.2f %8.2f %8.2f %8.2f %8.2f %8.2f %8.2f\n" %
(x1[i], y1[i], m1[i], x2[ix], y2[ix], m2[ix], d[ix]))
print("# Matched cat on %s" % matchcat)
idx = numpy.array(idx)
print("# Matched %s/%s" % (len(idx), self.Ngal))
return
def _apcorrSetup(self,dcat,detectionSelectSet):
"""private method called by the constructor to help initialise aperture correction stuff.
This method reads the detection catalog and greps the ISOAREA_IMAGE data out of that catalog.
This method then converts the areas for the objects in the catalog into radii and returns
a Numeric array to the constructor. These ISO areas are the same for all filters so this
only has to happen once, hence the constructor calls this. The catalog arg passed is the
detection catalog which is just a readlines() list, the second arg is the selectSet for the
detection catalog header.
"""
iradList = []
isoAreaIndex = detectionSelectSet["ISOAREA_IMAGE"] - 1 # index is one less than the col no.
isoAreaData = tableio.get_data(self.detCatalog,isoAreaIndex)
for area in isoAreaData:
iradList.append(max(2.0 ,(math.sqrt(area/math.pi))))
return iradList
def evolfit(modelfile):
import scipy
import scipy.interpolate
import tableio
e = {}
(z, e_g, e_r, e_i, e_z) = tableio.get_data(modelfile,
cols=(0, 14, 15, 16, 17))
e['g'] = scipy.interpolate.interp1d(z, e_g)
e['r'] = scipy.interpolate.interp1d(z, e_r)
e['i'] = scipy.interpolate.interp1d(z, e_i)
e['z'] = scipy.interpolate.interp1d(z, e_z)
return e
def sky2xy_list(ra, dec, fitsfile):
import os
import tableio
inlist = "/tmp/%s_list.sky2xy" % os.environ['USER']
outlist = "/tmp/%s_list.sky2xy.out" % os.environ['USER']
tableio.put_data(inlist, (ra, dec),
header='',
format="%s %s J2000",
append='no')
cmd = "sky2xy %s @%s > %s" % (fitsfile, inlist, outlist)
os.popen(cmd)
(x, y) = tableio.get_data(outlist, cols=(4, 5))
#print cmd
os.system("rm %s" % inlist)
os.system("rm %s" % outlist)
return x, y
def _apcorr(self, detector, filter):
"""Calculate the aperture corrections for this dataset for the given detector and filter.
This is a factor to be added to the magnitudes calculated above array [m].
This "apcor" is to be in magnitudes and apcor must be <= 0. This new magnitude or
array of magnitudes is then written into the multicolor.cat file and presented to
BPZ as the magnitude to use, as indicated in the multicolor.columns file.
Function returns a numeric array of aperture corrections. The fiducial radius
specified herein is specified in Bugzilla bug #2708, as are the other specifications
for this method.
"""
apcorr = []
apcorrArray = None
irad_fiducial = 14
aperFiles = {
"UVIS": os.path.join(self.aperData,
"newUVIS_2011.ee_new_csky.dat"),
"IR": os.path.join(self.aperData, "newIR_1210.ee_new_csky.dat"),
"WFC": os.path.join(self.aperData, "newWFC_0803.ee_new_csky.dat"),
#{}"HRC" : os.path.join(self.aperData,"newHRC_0803.ee_new_csky.dat")
}
try:
aperDataFile = aperFiles[detector]
except KeyError:
raise KeyError, "No known aperture data for detector, " + detector
# now get the filter aperture data for the indicated detector.
# raise a filterError if the lookup fails.
aperCatalog = open(aperDataFile).readlines()
selectSet = pUtil.makeHeaderDict(aperCatalog)
colName = "EE_" + filter
try:
aperIndex = selectSet[colName] - 1
except KeyError:
raise fUtil.filterError, "No aperture data for filter " + filter
aperData = tableio.get_data(aperDataFile, aperIndex)
# the aperData contains the encircled energy as a function of radius in
# pixels. Conveniently, the pixel radius = index + 1 of the value we
# want from the array for that radius.
eeFiducial = aperData[irad_fiducial - 1]
for irad in self.iradList:
if irad > irad_fiducial:
apcorr.append(0)
continue
# irad1, irad2 are the adjacent integers of the radius of the object.
# used to do a linear interpolation on the encircled energy curve.
irad1 = int(irad)
irad2 = irad1 + 1
iradFrac = irad - irad1
ee_irad1 = aperData[irad1 - 1]
ee_irad2 = aperData[irad2 - 1]
k = (ee_irad2 - ee_irad1)
ee_irad = k * iradFrac + ee_irad1
apcorr.append(min(0, +2.5 * math.log10(ee_irad / eeFiducial)))
apcorrArray = Numeric.array(apcorr)
return apcorrArray
def run(self, band):
"""Make the multicolor catalog."""
# define the zero point error
zpoint_err = 0.01
self.logfile.write('Processing catalogs...')
detection_variables = tableio.get_data(self.detCatalog,
self.detectionColumns)
nsources = len(detection_variables[0])
# WZ Should be a dimension check here with the filter catalogs before crashing
ncats = len(self.catalogList)
#Create a series of arrays to keep the relevant data
flux = Numeric.zeros((ncats, nsources)) * 1.0 # ISO flux ... for now.
fluxerr = Numeric.zeros(
(ncats, nsources)) * 1.0 # ISO flux error... for now.
m = Numeric.zeros((ncats, nsources)) * 1.0 # MAG_ISO
em = Numeric.zeros((ncats, nsources)) * 1.0 # MAGERR_ISO
m_corr = Numeric.zeros(
(ncats, nsources)) * 1.0 # extinction corrected magnitude
em_corr = Numeric.zeros(
(ncats, nsources)) * 1.0 # extinction corrected magnitude error
ap_corr = Numeric.zeros(
(ncats, nsources)) * 1.0 # aperture correction
m_bpz = Numeric.zeros(
(ncats,
nsources)) * 1.0 # MAG_BPZ extinction/aperture corrected
em_bpz = Numeric.zeros(
(ncats,
nsources)) * 1.0 # MAGERR_BPZ extinction/aperture corrected
preds = [os.path.basename(self.detCatalog)]
# this is a list of those indices of the catalogList which raised the filterError exception
# i think i need to know this, but i'm not sure why yet....
self.badcats = []
for i in range(ncats):
selectSet = {}
pardict = {}
imname = os.path.splitext(self.catalogList[i])[0]
j = string.rfind(imname, '_')
basefits = imname[0:j] + '_sci.fits'
fitsfile = os.path.join(self.obsFits, basefits)
detector = fUtil.getDetector(fitsfile)
try:
imfilter = fUtil.filterResponseName(fitsfile)
except fUtil.filterError:
self.badcats.append(i)
imfilter = fUtil.twoFilterName(fitsfile)
try:
filterXCorr = extinction.filterFactor(imfilter) * self.eBV
filterXCorrError = filterXCorr * 0.16
except fUtil.filterError, err:
self.errorList.append(
("extintion.py", "filterError: " + str(err)))
self.logfile.write("filterError: " + str(err))
filterXCorr = 0.0
filterXCorrError = 0.0
try:
f = pyfits.open(fitsfile) # WZ
instr = f[0].header.get('INSTRUME')
f.close()
del f
if instr == 'ACS':
#filter = 'F814W' #WZ
filter = fUtil.filterNameACS(fitsfile)
elif instr == 'WFC3':
#filter = fUtil.filterName(fitsfile)
filter = fUtil.filterNameWFC3(fitsfile)
if filter:
print "Find filter ", filter
else:
raise filterError, "No filter found for fits image: " + fitsfile
#filter = fUtil.filterName(fitsfile)
ap_corr[i, :] = self._apcorr(detector, filter)
except KeyError, err:
print str(err)
self.logfile.write(str(err))
# if we can't do it, well, we can't do it.
pass
# i.e the catalog column numbers start at 1 not zero.
catalogObj = open(catalog).readlines()
selectSet = pUtil.makeHeaderDict(catalogObj)
#Info for flux columns
fluxList = []
fluxList.append(selectSet['FLUX_ISO'] - 1)
fluxList.append(selectSet['FLUXERR_ISO'] - 1)
self.fluxColumns = tuple(
fluxList) # the get_data function interface requires a tuple
# Build the various columns arrays with the get_data function.
# We read raw fluxes and errors into the flux,fluxerr arrays.
# They are afterwards transformed to magnitudes
t1, t2 = tableio.get_data(catalog, self.fluxColumns)
if (len(t1) != nsources):
self.logfile.write("Catalog dimension mismatch: ",
str(len(t1)), ' ', nsources)
self.logfile.write(
"Check patrameters in detectionCatalog.inpar and filterCatalog.inpar"
)
#flux[i,:],fluxerr[i,:] = tableio.get_data(catalog,self.fluxColumns)
flux[i, :], fluxerr[i, :] = t1, t2
flux[i, :] = pUtil.deNAN(flux[i, :])
# Those objects with flux equal or less than 0 are assigned a magnitude of 99
# and a limiting magnitude equal to their SExtractor photometric error. This
# is interpreted by BPZ as a nondetection with zero flux and 1-sigma error
# equal to the limiting magnitude
catalogObj = open(catalog).readlines()
selectSet = pUtil.makeHeaderDict(catalogObj)
#Info for flux columns
fluxList = []
fluxList.append(selectSet['FLUX_ISO'] - 1)
fluxList.append(selectSet['FLUXERR_ISO']- 1)
self.fluxColumns = tuple(fluxList) # the get_data function interface requires a tuple
# Build the various columns arrays with the get_data function.
# We read raw fluxes and errors into the flux,fluxerr arrays.
# They are afterwards transformed to magnitudes
pdb.set_trace() #xingxing
flux[i,:],fluxerr[i,:] = tableio.get_data(catalog,self.fluxColumns)
flux[i,:] = pUtil.deNAN(flux[i,:])
# Those objects with flux equal or less than 0 are assigned a magnitude of 99
# and a limiting magnitude equal to their SExtractor photometric error. This
# is interpreted by BPZ as a nondetection with zero flux and 1-sigma error
# equal to the limiting magnitude
nondetected = Numeric.less_equal(flux[i,:],0.0)*Numeric.greater(fluxerr[i,:],0.0)
# Those objects with error flux and flux equal to 0 are assigned a magnitude of -99
# and a flux of 0, which is interpreted by SExtractor as a non-observed object
nonobserved = Numeric.less_equal(fluxerr[i,:],0.0)
# When flux error > 100*(flux), mark as nonobserved (Benitez, 24-Oct-03).
def read_dndM(self):
(M, dndM) = tableio.get_data(self.dndMFile, cols=(0, 1))
os.remove(self.dndMFile)
return M, dndM
def BuildColorCat(tilename, combcat, filters=['g', 'r', 'i', 'z', 'K'],
newfirm=True):
# The default output names
colorCat = tilename + "_complete.catalog"
print('Processing catalogs... for: ', tilename, file=sys.stderr)
flux = {}
fluxerr = {}
m = {}
em = {}
# Get the detection catalog required columns
outColumns = ['NUMBER', 'X_IMAGE', 'Y_IMAGE']
detCatalog = combcat['i']
detcols = SEx_head(detCatalog, verb=None)
detectionList = []
for key in outColumns:
detectionList.append(detcols[key])
# the get_data function requires a tuple
detectionColumns = tuple(detectionList)
detection_variables = tableio.get_data(detCatalog,
detectionColumns)
# Read in the MAG_ISO and MAG_ISOERR from each catalog
for filter in filters:
if not newfirm and filter == 'K':
continue
# get the zeropoint Info
tmp = np.genfromtxt('photometry_control_star_{}.dat'.format(
filter), names=True, dtype=None)
zpoint = tmp['ZP']
# Get the columns
sexcols = SEx_head(combcat[filter], verb=None)
## Info for flux columns
fluxList = []
fluxList.append(sexcols['FLUX_ISO'])
fluxList.append(sexcols['FLUXERR_ISO'])
fluxColumns = tuple(
fluxList) # the get_data function interface requires a tuple
# Get the array using tableio
flux[filter], fluxerr[filter] = tableio.get_data(combcat[filter],
fluxColumns)
m[filter] = flux[filter] * 0.0
em[filter] = flux[filter] * 0.0
# Fix the NAN values
flux[filter] = deNAN(flux[filter])
# Those objects with flux equal or less than 0 are assigned a
# magnitude of 99 and a limiting magnitude equal to their
# SExtractor photometric error. This is interpreted by BPZ as a
# nondetection with zero flux and 1-sigma error equal to the
# limiting magnitude
#nondetected = np.less_equal(flux[filter], 0.0) * \
# np.greater(fluxerr[filter], 0.0)
# update: There are a lot of really small positive values. I am
# going to modify this to look for values really close to zero.
nondetected = (flux[filter] < 1E-3) & (fluxerr[filter] > 0.0)
# Those objects with error flux and flux equal to 0 are assigned a
# magnitude of -99
# and a flux of 0, which is interpreted by SExtractor as a
# non-observed object
nonobserved = np.less_equal(fluxerr[filter], 0.0)
# When flux error > 100*(flux), mark as nonobserved (Benitez,
# 24-Oct-03).
nonobserved = np.where(fluxerr[filter] > 100 *
(abs(flux[filter])), True,
nonobserved)
detected = np.logical_not(nonobserved + nondetected)
print(filter, zpoint)
flux[filter] = np.clip(flux[filter], 1e-100, 1e100)
m[filter] = np.where(detected,
-2.5 * np.log10(abs(flux[filter])) +
zpoint, m[filter])
m[filter] = np.where(nondetected, 99.0, m[filter])
m[filter] = np.where(nonobserved, -99.0, m[filter])
# clip values from being too small or large, i.e. 0 or inf.
fluxerr[filter] = np.clip(fluxerr[filter], 1e-100, 1e100)
em[filter] = np.where(
detected,
2.5 * np.log10(1.0 + abs(fluxerr[filter] / flux[filter])), em[filter])
em[filter] = np.where(
nondetected,
2.5 * np.log10(abs(fluxerr[filter])) - zpoint, em[filter])
em[filter] = np.where(nonobserved, 0.0, em[filter])
if filter == 'K':
outColumns.append(filter + '_KittPeak_MAG_ISO')
outColumns.append(filter + '_KittPeak_MAGERR_ISO')
else:
outColumns.append(filter + '_MOSAICII_MAG_ISO')
outColumns.append(filter + '_MOSAICII_MAGERR_ISO')
# Prepare the header
header = \
'## ' + '\n' + \
'## BPZ Catalog file for Observation: ' + tilename + \
'\n' + \
'## (This file was generated automatically by' + \
'the BCS Rutgers pipeline)\n##\n'
for i in range(len(outColumns)):
header = header + '# ' + str(i + 1) + '\t' + outColumns[i] + '\n'
# Prepare the data
vars = list(detection_variables)
for filter in filters:
if not newfirm and filter == 'K':
continue
vars.append(m[filter])
vars.append(em[filter])
variables = tuple(vars)
format = '%i\t %10.2f %10.2f' + '%10.4f ' * (len(variables) - 3)
print('Writing data to multicolor catalog...', file=sys.stderr)
tableio.put_data(colorCat,
variables,
header=header,
format=format,
append='no')
print('Multicolor catalog complete.', file=sys.stderr)
return
def read_cat(self):
t1 = time.time()
cols = (1, 2, 23, 27, 26, 28, 29, 30, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16,
17, 18, 19, 20, 21, 22, 31, 32, 33, 34, 35, 36)
sout.write("# Reading cols:%s\n# Reading cats from: %s... \n" %
(cols, self.catsfile))
(ra, dec, z_b, odds, t_b, z_ml, t_ml, chi, g, g_err, r, r_err, i, i_err, z,
z_err, g_bpz, g_berr, r_bpz, r_berr, i_bpz, i_berr, z_bpz, z_berr,
class_star, a_image, b_image, theta, x_image,
y_image) = tableio.get_data(self.catsfile, cols=cols)
(id) = tableio.get_str(self.catsfile, cols=(0, ))
############################################
# Choose the photo-z to use, ml or bayesian
############################################
sout.write("# Will use %s redshifts\n" % self.zuse)
if self.zuse == "ML":
z_ph = z_ml
# t = t_ml
elif self.zuse == "ZB":
z_ph = z_b
# t = t_b
i_lim = self.maglim
odds_lim = 0.80 # not currently used
star_lim = self.starlim
# Clean up according to BPZ
sout.write("# Avoiding magnitudes -99 and 99 in BPZ \n")
g_mask = numpy.where(lor(g_bpz == 99, g_bpz == -99), 0, 1)
r_mask = numpy.where(lor(r_bpz == 99, r_bpz == -99), 0, 1)
i_mask = numpy.where(lor(i_bpz == 99, i_bpz == -99), 0, 1)
z_mask = numpy.where(lor(z_bpz == 99, z_bpz == -99), 0, 1)
bpz_mask = g_mask * r_mask * i_mask * z_mask
# Clean up to avoid 99 values and very faint i_mag values
sout.write("# Avoiding magnitudes 99 in MAG_AUTO \n")
#g_mask = numpy.where( g >= 99, 0 , 1)
#r_mask = numpy.where( r >= 99, 0 , 1)
#i_mask = numpy.where( i >= i_lim, 0 , 1)
#z_mask = numpy.where( z >= 99, 0 , 1)
sout.write("# Avoiding magnitudes i > %s in MAG_AUTO \n" % i_lim)
# Clean by class_star
sout.write("# Avoiding CLASS_STAR > %s \n" % star_lim)
mask_star = numpy.where(class_star > star_lim, 0, 1)
# Clean up by odds
#sout.write( "# Avoiding ODDS < %s in BPZ \n" % odds_lim)
odds_mask = numpy.where(odds > odds_lim, 1, 0)
odds_mask = 1
# Avoid z> zlim objects too.
#sout.write( "# Avoiding objects with z > %s " % self.zlim)
zp_mask = numpy.where(z_ph > self.zlim, 0, 1)
zp_mask = 1
# Clean up by BPZ type
# sout.write('# Avoiding objects with type > %s' % t)
tp_mask = 1
# The final 'good' mask
mask_good = bpz_mask * odds_mask * mask_star * zp_mask * tp_mask
idx = numpy.where(mask_good == 1)
# Make ids a Char String in numarray
self.id = nstr.array(id)[idx]
# Only keep the 'good' one, avoid -99 and 99 values in BPZ mags
self.ra = ra[idx]
self.dec = dec[idx]
self.z_b = z_b[idx]
self.odds = odds[idx]
self.z_ml = z_ml[idx]
self.t_ml = t_ml[idx]
self.t_b = t_b[idx]
self.t_ml = t_ml[idx]
self.chi = chi[idx]
############################################
# Choose the photo-z to use, ml or bayesian
############################################
if self.zuse == "ML":
self.z_ph = self.z_ml
self.type = self.t_ml
elif self.zuse == "ZB":
self.z_ph = self.z_b
self.type = self.t_b
self.g = g[idx]
self.r = r[idx]
self.i = i[idx]
self.z = z[idx]
self.g_err = g_err[idx]
self.r_err = r_err[idx]
self.i_err = i_err[idx]
self.z_err = z_err[idx]
self.g_bpz = g_bpz[idx]
self.r_bpz = r_bpz[idx]
self.i_bpz = i_bpz[idx]
self.z_bpz = z_bpz[idx]
self.g_berr = g_berr[idx]
self.r_berr = r_berr[idx]
self.i_berr = i_berr[idx]
self.z_berr = z_berr[idx]
self.class_star = class_star[idx]
self.a_image = a_image[idx]
self.b_image = b_image[idx]
self.theta = theta[idx]
self.x_image = x_image[idx]
self.y_image = y_image[idx]
# Color of selected galaxies
self.gr = self.g_bpz - self.r_bpz
self.ri = self.r_bpz - self.i_bpz
self.iz = self.i_bpz - self.z_bpz
# Min and and max values in RA/DEC
self.ramin = self.ra.min()
self.ramax = self.ra.max()
self.decmin = self.dec.min()
self.decmax = self.dec.max()
self.idx_cat = idx
sout.write(" \tDone: %s\n" % extras.elapsed_time_str(t1))
return
def BuildColorCat(self):
# Change accordingly
zp_error = 0.05
# The default output names
self.colorCat = self.tilename + ".color"
self.columnsFile = self.tilename + ".columns"
print('Processing catalogs... for: ', self.tilename, file=sys.stderr)
flux = {}
fluxerr = {}
m = {}
em = {}
# Get the detection catalog required columns
outColumns = ['NUMBER', 'X_IMAGE', 'Y_IMAGE']
detCatalog = self.combcat['i']
detcols = SEx_head(detCatalog, verb=None)
detectionList = []
for key in outColumns:
detectionList.append(detcols[key])
detectionColumns = tuple(
detectionList) # the get_data function requires a tuple
detection_variables = tableio.get_data(detCatalog, detectionColumns)
# Read in the MAG_ISO and MAG_ISOERR from each catalog
for filter in self.filters:
# Get the columns
sexcols = SEx_head(self.combcat[filter], verb=None)
## Info for flux columns
fluxList = []
fluxList.append(sexcols['FLUX_ISO'])
fluxList.append(sexcols['FLUXERR_ISO'])
fluxColumns = tuple(
fluxList) # the get_data function interface requires a tuple
# Get the array using tableio
flux[filter], fluxerr[filter] = tableio.get_data(
self.combcat[filter], fluxColumns)
m[filter] = flux[filter] * 0.0
em[filter] = flux[filter] * 0.0
# Fix the NAN values
flux[filter] = deNAN(flux[filter])
# Those objects with flux equal or less than 0 are assigned a magnitude of 99
# and a limiting magnitude equal to their SExtractor photometric error. This
# is interpreted by BPZ as a nondetection with zero flux and 1-sigma error
# equal to the limiting magnitude
nondetected = Numeric.less_equal(
flux[filter], 0.0) * Numeric.greater(fluxerr[filter], 0.0)
# Those objects with error flux and flux equal to 0 are assigned a magnitude of -99
# and a flux of 0, which is interpreted by SExtractor as a non-observed object
nonobserved = Numeric.less_equal(fluxerr[filter], 0.0)
# When flux error > 100*(flux), mark as nonobserved (Benitez, 24-Oct-03).
# Fix for fc11 -- y[:] has change meaning
#nonobserved = Numeric.where(fluxerr[filter] > 100*(abs(flux[filter])),1.0,nonobserved[:])
nonobserved = Numeric.where(
fluxerr[filter] > 100 * (abs(flux[filter])), 1.0, nonobserved)
detected = Numeric.logical_not(nonobserved + nondetected)
# Get the zero point for the final magnitudes
zpoint = self.magbase
print(filter, zpoint)
flux[filter] = Numeric.clip(flux[filter], 1e-100, 1e100)
m[filter] = Numeric.where(
detected, -2.5 * Numeric.log10(abs(flux[filter])) + zpoint -
self.XCorr[filter], m[filter])
m[filter] = Numeric.where(nondetected, 99.0, m[filter])
m[filter] = Numeric.where(nonobserved, -99.0, m[filter])
# clip values from being too small or large, i.e. 0 or inf.
fluxerr[filter] = Numeric.clip(fluxerr[filter], 1e-100, 1e100)
em[filter] = Numeric.where(
detected,
2.5 * Numeric.log10(1.0 + abs(fluxerr[filter] / flux[filter]))
+ self.XCorrError[filter], em[filter])
em[filter] = Numeric.where(
nondetected,
2.5 * Numeric.log10(abs(fluxerr[filter])) - zpoint, em[filter])
em[filter] = Numeric.where(nonobserved, 0.0, em[filter])
#outColumns.append(filter +'_SDSS_MAG_ISO')
#outColumns.append(filter +'_SDSS_MAGERR_ISO')
outColumns.append(filter + '_MOSAICII_MAG_ISO')
outColumns.append(filter + '_MOSAICII_MAGERR_ISO')
# Prepare the header
header = \
'## ' + time.ctime() + '\n'+\
'## BPZ Catalog file for Observation: ' + self.tilename + '\n'+\
'## (This file was generated automatically by the BCS Rutgers pipeline)\n##\n'
for i in range(len(outColumns)):
header = header + '# ' + str(i + 1) + '\t' + outColumns[i] + '\n'
# Prepare the data
vars = list(detection_variables)
for filter in self.filters:
vars.append(m[filter])
vars.append(em[filter])
variables = tuple(vars)
format = '%i\t %10.2f %10.2f' + '%10.4f ' * (len(variables) - 3)
print('Writing data to multicolor catalog...', file=sys.stderr)
tableio.put_data(self.colorCat,
variables,
header=header,
format=format,
append='no')
print('Multicolor catalog complete.', file=sys.stderr)
# And now write .columns file
cfile = open(self.columnsFile, 'w')
cfile.write('## ' + time.ctime() + '\n')
cfile.write('## ' + 'BPZ' + ' .columns file for Observation: ' +
self.tilename + '\n')
cfile.write(
'## (This file was generated automatically by the BCS Rutgers pipeline)\n##\n'
)
i = len(detection_variables)
for filter in self.filters:
if filter == 'i':
n_mo = str(i + 1)
colmag = i + 1
colmagerr = i + 2
cfile.write('%s_MOSAICII\t %s,%s\t AB\t %.2f\t 0.0\n' %
(filter, i + 1, i + 2, zp_error))
i = i + 2
cfile.write('M_0\t%s\n' % n_mo)
cfile.close()
return
def prepMatch(reffile, imlist): #WZ
""" Calculate shift/rotation
new reference: center position
"""
de = 100. # distance from edge
circle = 500. # matching range in pixel
rmin = 4. # FWHM minimum
rmax = 15. # 10. # 6.
# clx = 0.8 # must be a star
pardir = '/home/zheng/drex/reffiles/pars'
fitsfile = pyfits.open(reffile)
detector = fitsfile[0].header.get('DETECTOR')
fitsfile.close()
#pdb.set_trace()
weightfile = reffile.split(".fits")[0] + '_weight.fits'
cmd = 'cp ' + weightfile + ' temp_weight.fits'
sproc = popen2.Popen3(cmd, 1)
output = sproc.fromchild.readlines()
errs = sproc.childerr.readlines()
if (detector == 'WFC'):
cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_wfc.inpar'
else:
if (detector == 'UVIS'):
cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_uvis.inpar'
else:
cmd = 'sex ' + reffile + ' -c ' + pardir + '/tweak_ir.inpar'
sproc = popen2.Popen3(cmd, 1)
output = sproc.fromchild.readlines()
errs = sproc.childerr.readlines()
cmd = 'cp temp.cat temp0.cat'
sproc = popen2.Popen3(cmd, 1)
output = sproc.fromchild.readlines()
errs = sproc.childerr.readlines()
x0, y0, mag0, err0, fwhm0, ra0, dec0, cls0 = tableio.get_data(
'temp.cat', (1, 2, 3, 4, 5, 6, 7, 8))
xmin0 = min(x0) + de
xmax0 = max(x0) - de
ymin0 = min(y0) + de
ymax0 = max(y0) - de
#twf = open("tweak.dat","w")
format = str('%4d %8.2f %8.2f %6.2f %6.2f %4.1f')
#format2=str('%s %6.3f %5.3f %10.7f')
for im in imlist:
#outfile = im.split("_sci")[0]+'_ini.fits'
fitsfile = pyfits.open(im)
detector = fitsfile[0].header.get('DETECTOR')
fitsfile.close()
weightfile = im.split(".fits")[0] + '_weight.fits'
cmd = 'cp ' + weightfile + ' temp_weight.fits'
sproc = popen2.Popen3(cmd, 1)
output = sproc.fromchild.readlines()
errs = sproc.childerr.readlines()
#pdb.set_trace()
if (detector == 'WFC'):
cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_wfc.inpar'
else:
if (detector == 'UVIS'):
cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_uvis.inpar'
else:
cmd = 'sex ' + im + ' -c ' + pardir + '/tweak_ir.inpar'
sproc = popen2.Popen3(cmd, 1)
output = sproc.fromchild.readlines()
errs = sproc.childerr.readlines()
if errs:
print 'Program choked on ' + cmd
else:
n, x, y, mag, err, fwhm, ra, dec, cls = tableio.get_data(
'temp.cat', (0, 1, 2, 3, 4, 5, 6, 7, 8))
# pdb.set_trace()
flag0 = numpy.zeros(len(x0), dtype=int)
flag = numpy.zeros(len(n), dtype=int)
index = numpy.zeros(len(n), dtype=int)
xmin = min(x) + de
if (xmin < xmin0):
xmin = xmin0
xmax = max(x) - de
if (xmax > xmax0):
xmax = xmax0
ymin = min(y) + de
if (ymin < ymin0):
ymin = ymin0
ymax = max(y) - de
if (ymax > ymax0):
ymax = ymax0
print 'Min/max: ', xmin, xmax, ymin, ymax
fname0 = im.split("_drz")[0] + '_1.cat'
fname1 = im.split("_drz")[0] + '_2.cat'
f0 = open(fname0, "w")
f = open(fname1, "w")
k = 0
for i in range(len(x)):
# if (mag[i] < 90 and fwhm[i] > rmin and fwhm[i] < rmax):
if (mag[i] < 90):
for j in range(len(x0)):
if (abs(x[i] - x0[j]) < circle
and abs(y[i] - y0[j]) < circle and mag0[j] < 90
and fwhm0[j] > rmin and fwhm0[j] < rmax):
#if (abs(x[i]-x0[j]) < circle and abs(y[i]-y0[j]) < circle and mag0[j] < 90):
index[i] = j
flag[i] = flag[i] + 1
flag0[j] = flag0[j] + 1
k = k + 1
print k, " sources found in ", im
k = m = 0
for i in range(len(x)):
if (flag[i] >= 1):
#pdb.set_trace()
j = index[i]
if (flag0[j] >= 1 and fwhm0[j] < rmax and mag[i] < 90):
#if (flag0[j] >= 1 and mag[i] < 90):
print >> f, format % (k, x[i], y[i], mag[i], err[i],
fwhm[i])
print >> f0, format % (k, x0[j], y0[j], mag0[j],
err0[j], fwhm0[j])
k = k + 1
else:
m = m + 1
print k, " sources used in ", im #,", ",m," missed"
#pdb.set_trace()
f0.close()
f.close()
return
def _magFix(self, catalogFile):
"""This private method receives a path to a catalog file and sifts through the
MAGERR field looking for values > 10. It sets the corresponding MAG field = -99 and
sets that object's MAGERR field to 0.0. catalogFile is a path not a file object."""
# fillHeader will return a list of tuples where which looks like
#
# [(1, 'NUMBER'),
# (2, 'X_IMAGE'),
# (3, 'Y_IMAGE'),
# ...
# (12, 'MAG_ISOCOR'),
# (13, 'MAGERR_ISOCOR'),
# (14, 'FLUX_APER', 1)
# (15, 'FLUX_APER', 2),
# (16, 'FLUX_APER', 3),
# ...
# ]
#
# The tuples are either of length 2 or 3. If len is 3, the 3rd item of the
# tuple is the nth occurance of that column identifier. This occurs on those
# columns of MAGs and MAGERRs for a series of increasingly larger apertures.
# newFieldList will be a list of Numeric arrays containing the columns of the catalogs.
# This list will contain fields which have not been altered, i.e. all fields other than
# MAG_* and MAGERR_*, and the new MAG and MAGERR fields which have been corrected.
# Once the list is complete, it is tuple-ized and send to the tableio pu_data function.
newFieldList = []
newMagsList = []
newMagErrsList = []
newMagHeaders = []
newMagErrHeaders = []
newHeaders = []
magCols = []
magErrCols = []
selectSet = fillHeader(catalogFile)
print "Searching catalog for required columns, MAG, MAGERR"
for i in range(len(selectSet)):
if len(selectSet[i]) == 2:
column, name = selectSet[i]
paramNames = name.split("_")
if "MAG" in paramNames:
magCols.append((column, name))
elif "MAGERR" in paramNames:
magErrCols.append((column, name))
else:
oldField = tableio.get_data(catalogFile, (column - 1))
newFieldList.append(oldField)
newHeaders.append(name)
continue
else:
column, name, id = selectSet[i]
paramNames = name.split("_")
if "MAG" in paramNames:
magCols.append((column, name, id))
elif "MAGERR" in paramNames:
magErrCols.append((column, name, id))
else:
oldField = tableio.get_data(catalogFile, (column - 1))
newFieldList.append(oldField)
newHeaders.append(name)
continue
# We now have
# catalog field --> list
# --------------------------------
# MAG_* --> magCols
# MAGERR_* --> magErrCols
#
# The algorithm will be to step through the magErrCols columns, extracting those fields
# via get_data and getting Numeric arrays. The matching mag columns are slurped as well.
# We search the magErrCols arrays looking for >= 10 values and then marking the those mags
# as -99.0 and the matching magerrs as 0.0
# See Bugzilla bug #2700
for item in magErrCols:
magErrAperId = None
# item may be of len 2 or 3
if len(item) == 2:
magErrColId, magErrColName = item
else:
magErrColId, magErrColName, magErrAperId = item
magErrKind = magErrColName.split("_")[1] # ISO, ISOCORR, etc.
print "\n\nMAG type:", magErrKind
if magErrAperId: print magErrColName, "Aper id is", magErrAperId
print "Getting\t", magErrColName, "\tfield", magErrColId
# MAGERR array:
magErrs = tableio.get_data(catalogFile, magErrColId - 1)
matchingMagColName = None
matchingMagColId = None
#----------------------- Search for matching MAG_* field -----------------------#
for magitems in magCols:
# We know that the magErrColName is MAGERR and if magErrNameId is true then
# the tuple is of len 3, i.e. a MAGERR_APER field. We look for the matching
# MAG_APER field id, 1, 2, 3... etc.
if len(magitems) == 3:
magColId, magColName, magAperId = magitems
if magColName == "MAG_" + magErrKind:
matchingMagColName = magColName
#print "Found matching field type:",magColName,"in field",magColId
if magAperId == magErrAperId:
print "Found matching aperture id."
print "MAG_APER id: ", magAperId, "MAGERR_APER id: ", magErrAperId
matchingMagColId = magColId
matchingMags = tableio.get_data(
catalogFile, magColId - 1)
break
else:
continue
else:
magColId, magColName = magitems
if magColName == "MAG_" + magErrKind:
print "Found matching field type:", magColName, "in field", magColId
matchingMagColName = magColName
matchingMagColId = magColId
matchingMags = tableio.get_data(
catalogFile, magColId - 1)
break
else:
continue
#--------------------------------------------------------------------------------#
print " MAG err field:", magErrColName, magErrColId
print " Mag field:", matchingMagColName, matchingMagColId
# Now the grunt work on the arrays,
# magErrs, matchingMags
#
# update: flagging all MAGs as -99 when the corresponding MAGERR > 10
# introduced a bug which unintentionally reset the magnitudes
# SExtractor had flagged with a MAG = 99.0 and a MAGERR = 99.0
# This now checks for a MAGERR of 99 and does not reset the MAG value
# if MAGERR = 99.0 but does for all other MAGERRS > 10.0
badMagErrs1 = Numeric.where(magErrs >= 10, 1, 0)
badMagErrs2 = Numeric.where(magErrs != 99.0, 1, 0)
badMagErrs = badMagErrs1 * badMagErrs2
del badMagErrs1, badMagErrs2
newMags = Numeric.where(badMagErrs, -99.0, matchingMags)
newMagErrs = Numeric.where(badMagErrs, 0.0, magErrs)
newMagsList.append(newMags)
newMagHeaders.append(matchingMagColName)
newMagErrsList.append(newMagErrs)
newMagErrHeaders.append(magErrColName)
# concatenate the lists. This is done to preserve the MAG_APER and MAGERR_APER
# grouping of the original SExtractor catalog.
newFieldList = newFieldList + newMagsList
newFieldList = newFieldList + newMagErrsList
newHeaders = newHeaders + newMagHeaders
newHeaders = newHeaders + newMagErrHeaders
newVariables = tuple(newFieldList)
# rename the old catalog file as catalogFile.old
os.rename(catalogFile, catalogFile + ".old")
self.outputList[os.path.basename(catalogFile) +
".old"] = [os.path.basename(catalogFile)]
fob = open(catalogFile, 'w')
fob.write("## " + ptime() + "\n")
fob.write("## " + self.modName +
" catalog regenerated by _magFix method.\n")
fob.write(
'## (This file was generated automatically by the ACS Pipeline.)\n##\n'
)
fob.write(
"## This catalog has been photometrically corrected to remove\n")
fob.write("## 'bad' magnitude values.\n")
fob.write("##\n")
for i in range(len(newHeaders)):
fob.write("# " + str(i + 1) + "\t" + newHeaders[i] + "\n")
fob.close()
tableio.put_data(catalogFile, newVariables, append="yes")
return
