def run_redmonster(this_thresh):
#global completeness
#global purity
global nfib
completeness = []
purity = []
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
zstar = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', npoly=4, zmin=-.005, zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
zfit_ssp = zfitter.ZFitter(zssp.zchi2arr, zssp.zbase)
zfit_ssp.z_refine(threshold=this_thresh)
zfit_star = zfitter.ZFitter(zstar.zchi2arr, zstar.zbase)
zfit_star.z_refine(threshold=this_thresh)
zpick = zpicker.ZPicker(specs, zssp, zfit_ssp, zstar, zfit_star)
ssp_flags = n.zeros(len(fiberid))
star_flags = n.zeros(len(fiberid))
for ifiber in range(len(fiberid)):
ssp_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zssp.zwarning[ifiber])) | int(zfit_ssp.zwarning[ifiber])
star_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zstar.zwarning[ifiber])) | int(zfit_star.zwarning[ifiber])
#purity.append( (len(n.where(ssp_flags == 0)))/float(nfib) )
#completeness.append( (len(n.where(abs(zpick.z[:,0]-zperson) <= .0001)))/float(nfib) )
purity = (len(n.where(ssp_flags == 0)))/float(nfib)
completeness = (len(n.where(abs(zpick.z[:,0]-zperson) <= .0001)))/float(nfib)
return completeness, purity
def parallel_rm(xxx_todo_changeme ):
(plate,mjd,fiberid) = xxx_todo_changeme
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
# Write chi2 file with zbase
prihdu = fits.PrimaryHDU(zssp.zchi2arr)
col1 = fits.Column(name='ZBASE', format='E', array=zssp.zbase)
cols = fits.ColDefs([col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
thdulist = fits.HDUList([prihdu,tbhdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-%s.fits' % (plate, zssp.type), clobber=True)
# ----
zstar = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', npoly=4, zmin=-.005, zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
# Write chi2 file with zbase
prihdu = fits.PrimaryHDU(zstar.zchi2arr)
col1 = fits.Column(name='ZBASE', format='E', array=zstar.zbase)
cols = fits.ColDefs([col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
thdulist = fits.HDUList([prihdu,tbhdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-%s.fits' % (plate, zstar.type), clobber=True)
# ----
zfit_ssp = zfitter.ZFitter(zssp.zchi2arr, zssp.zbase)
zfit_ssp.z_refine()
zfit_star = zfitter.ZFitter(zstar.zchi2arr, zstar.zbase)
zfit_star.z_refine()
ssp_flags = misc.comb_flags(specs, zssp, zfit_ssp)
star_flags = misc.comb_flags(specs, zstar, zfit_star)
zpick = zpicker.ZPicker(specs, zssp, zfit_ssp, ssp_flags, zstar, zfit_star, star_flags)
# Write flags file
prihdu = fits.PrimaryHDU(zpick.zwarning)
thdulist = fits.HDUList([prihdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/flags-%s.fits' % plate, clobber=True)
output = io.WriteRedmonster(zpick, dest='/uufs/astro.utah.edu/common/home/u0814744/scratch/screens', clobber=True)
def parallel_rm(xxx_todo_changeme ):
(plate,mjd,fiberid) = xxx_todo_changeme
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
# Write chi2 file with zbase
prihdu = fits.PrimaryHDU(zssp.zchi2arr)
col1 = fits.Column(name='ZBASE', format='E', array=zssp.zbase)
cols = fits.ColDefs([col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
thdulist = fits.HDUList([prihdu,tbhdu])
thdulist.writeto('chi2arr-%s-%s.fits' % (plate, zssp.type), overwrite=True)
def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is None: fiberid = [i for i in range(1000)]
else:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except:
print('fiberid not set properly - running full plate!')
fiberid = [i for i in range(1000)]
else:
fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
if len(self.templates) == 1:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
elif len(self.templates) == 2:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1])
elif len(self.templates) == 3:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2])
elif len(self.templates) == 4:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3])
elif len(self.templates) == 5:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3],
zfindobjs[4], zfitobjs[4], flags[4])
output = None
# Write output
if self.dest is None:
output = io.WriteRedmonster(zpick, overwrite=self.overwrite)
else:
if type(self.dest) is str:
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
else:
try:
self.dest = str(self.dest)
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
except:
print(
'Could not convert dest to string - writing to default directory and NOT clobbering old files!'
)
output = io.WriteRedmonster(zpick, overwrite=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiberid()
else: output.write_plate()
''' Set plate, mjd, and fibers to be run. If fiberid is not specified
here and subsequently passed in during the next step, the default behavior
is to run on all fibers.
'''
plate = 8187
mjd = 57402
fiberid = [266,267] # fiberid must be a list, not a numpy array
''' Read spPlate file. specs.flux, specs.ivar, specs.loglambda, are
[nfibers, npix] arrays containing flux, inverse variances, and
log-wavelength, respectively. This step also flags sky fibers and
masks pixels with unreasonable S/N.
'''
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
#specs = donna_spec.Spec('/uufs/astro.utah.edu/common/home/u0814744/test.fits')
#skyfibers = n.where(specs.ebt1 == 0)[0][0:2]
#specs.flux = specs.flux[skyfibers]
#specs.ivar = specs.ivar[skyfibers]
#specs.dof = specs.dof[skyfibers]
''' Instantiate zfinder object that will do z-finding for the entire
plate using a single template. Here, fname is the template filename,
assumed to be in $REDMONSTER_DIR/templates/ . npoly specifies number
of polynomial terms to be used in finding, zmin and zmax are upper
and lower bounds of the redshift range to be explored. Optionally,
npixstep can specify the width of pixel steps in doing the
cross-correlation. If left blank, it defaults to 1.
'''
def reduce_plate_mjd(self,
plate=None,
mjd=None,
fiberid=None,
data_range=None,
chi2file=False,
platepath=None):
print "\nPlate %s MJD %s Fiber %s" % (plate, mjd, fiberid)
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is not None:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except ValueError:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except ValueError:
print('fiberid not set properly - running full plate!')
else:
fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate,
mjd=mjd,
fiberid=fiberid,
platepath=platepath)
fiberid = specs.fiberid
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i],
nproc=self.nproc))
if self.mask:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file,
linelist=__linelist__)
else:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i],
nproc=self.nproc))
if mask:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file,
linelist=__linelist__)
else:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
zpick = zpicker2.ZPicker(specs, zfindobjs, zfitobjs, flags)
output = None
# Write output
if self.dest is None:
output = io2.WriteRedmonster(zpick, overwrite=True)
else:
if type(self.dest) is str:
output = io2.WriteRedmonster(zpick,
dest=self.dest,
overwrite=True)
else:
try:
self.dest = str(self.dest)
output = io2.WriteRedmonster(zpick,
dest=self.dest,
overwrite=True)
except Exception as e:
print(
'Could not convert dest to string - writing to \
default directory and NOT clobbering old files! \
Exception: %r' % e)
output = io2.WriteRedmonster(zpick, overwrite=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiber()
else: output.write_plate()
def find_comp_purity(this_thresh, args):
purity = []
completeness = []
run = 1
for iarg in args:
print('Running plate %s of 8' % (run))
run += 1
plate = iarg[0]
mjd = iarg[1]
fiberid = iarg[2]
zperson = iarg[3]
origlen = float(len(fiberid))
badfibs = n.where(n.asarray(iarg[5]) != 'v5_4_9 ok')[0]
usefiberid = n.delete(n.asarray(fiberid), badfibs).tolist()
usezperson = n.delete(n.asarray(zperson), badfibs).tolist()
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=usefiberid)
hdu = fits.open(
'/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-ssp_galaxy_glob.fits'
% plate)
#hdu = fits.open('/Users/boltonlab3/scratch/chi2arr-%s-ssp_galaxy_glob.fits' % plate)
sspchi2arr = hdu[0].data
zbasessp = hdu[1].data.ZBASE
hdu = fits.open(
'/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-spEigenStar.fits'
% plate)
#hdu = fits.open('/Users/boltonlab3/scratch/chi2arr-%s-spEigenStar.fits' % plate)
starchi2arr = hdu[0].data
zbasestar = hdu[1].data.ZBASE
zfit_ssp = zfitter.ZFitter(sspchi2arr, zbasessp)
zfit_ssp.z_refine(threshold=this_thresh)
zfit_star = zfitter.ZFitter(starchi2arr, zbasestar)
zfit_star.z_refine(threshold=this_thresh)
ssp_flags = misc.comb_flags_2(specs, zfit_ssp.zwarning)
star_flags = misc.comb_flags_2(specs, zfit_star.zwarning)
zpick = Hacked_zpicker(specs, usefiberid, sspchi2arr, zfit_ssp,
ssp_flags, starchi2arr, zfit_star, star_flags)
#completeness.append( (len(n.where(zpick.zwarning == 0)[0]))/float(len(fiberid)) )
completeness.append(
(len(n.where((zpick.zwarning.astype(int) & 4) == 0)[0])) /
float(len(usefiberid)))
purity_set = zpick.z[n.where((zpick.zwarning.astype(int) & 4) == 0)[0],
0]
purity_zperson = n.asarray(usezperson)[n.where(
(zpick.zwarning.astype(int) & 4) == 0)[0]]
#purity.append( (len(n.where(abs(zpick.z[:,0]-zperson) <= .0005)[0]))/float(len(fiberid)) )
purity.append(
(len(n.where(abs(purity_set - purity_zperson) <= .0005)[0])) /
float(len(purity_set)))
this_comp = n.mean(completeness)
this_pur = n.mean(purity)
return this_comp, this_pur