def parallel_rm(xxx_todo_changeme ):
(plate,mjd,fiberid) = xxx_todo_changeme
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
# Write chi2 file with zbase
prihdu = fits.PrimaryHDU(zssp.zchi2arr)
col1 = fits.Column(name='ZBASE', format='E', array=zssp.zbase)
cols = fits.ColDefs([col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
thdulist = fits.HDUList([prihdu,tbhdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-%s.fits' % (plate, zssp.type), clobber=True)
# ----
zstar = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', npoly=4, zmin=-.005, zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
# Write chi2 file with zbase
prihdu = fits.PrimaryHDU(zstar.zchi2arr)
col1 = fits.Column(name='ZBASE', format='E', array=zstar.zbase)
cols = fits.ColDefs([col1])
tbhdu = fits.BinTableHDU.from_columns(cols)
thdulist = fits.HDUList([prihdu,tbhdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-%s.fits' % (plate, zstar.type), clobber=True)
# ----
zfit_ssp = zfitter.ZFitter(zssp.zchi2arr, zssp.zbase)
zfit_ssp.z_refine()
zfit_star = zfitter.ZFitter(zstar.zchi2arr, zstar.zbase)
zfit_star.z_refine()
ssp_flags = misc.comb_flags(specs, zssp, zfit_ssp)
star_flags = misc.comb_flags(specs, zstar, zfit_star)
zpick = zpicker.ZPicker(specs, zssp, zfit_ssp, ssp_flags, zstar, zfit_star, star_flags)
# Write flags file
prihdu = fits.PrimaryHDU(zpick.zwarning)
thdulist = fits.HDUList([prihdu])
thdulist.writeto('/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/flags-%s.fits' % plate, clobber=True)
output = io.WriteRedmonster(zpick, dest='/uufs/astro.utah.edu/common/home/u0814744/scratch/screens', clobber=True)
def run_redmonster(this_thresh):
#global completeness
#global purity
global nfib
completeness = []
purity = []
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
zstar = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', npoly=4, zmin=-.005, zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
zfit_ssp = zfitter.ZFitter(zssp.zchi2arr, zssp.zbase)
zfit_ssp.z_refine(threshold=this_thresh)
zfit_star = zfitter.ZFitter(zstar.zchi2arr, zstar.zbase)
zfit_star.z_refine(threshold=this_thresh)
zpick = zpicker.ZPicker(specs, zssp, zfit_ssp, zstar, zfit_star)
ssp_flags = n.zeros(len(fiberid))
star_flags = n.zeros(len(fiberid))
for ifiber in range(len(fiberid)):
ssp_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zssp.zwarning[ifiber])) | int(zfit_ssp.zwarning[ifiber])
star_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zstar.zwarning[ifiber])) | int(zfit_star.zwarning[ifiber])
#purity.append( (len(n.where(ssp_flags == 0)))/float(nfib) )
#completeness.append( (len(n.where(abs(zpick.z[:,0]-zperson) <= .0001)))/float(nfib) )
purity = (len(n.where(ssp_flags == 0)))/float(nfib)
completeness = (len(n.where(abs(zpick.z[:,0]-zperson) <= .0001)))/float(nfib)
return completeness, purity
def reduce_plate_mjd(self, plate, mjd, fiberid=None, chi2file=False):
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is None: fiberid = [i for i in range(1000)]
else:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except:
print('fiberid not set properly - running full plate!')
fiberid = [i for i in range(1000)]
else:
fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i]))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
if len(self.templates) == 1:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0])
elif len(self.templates) == 2:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1])
elif len(self.templates) == 3:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2])
elif len(self.templates) == 4:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3])
elif len(self.templates) == 5:
zpick = zpicker.ZPicker(specs, zfindobjs[0], zfitobjs[0], flags[0],
zfindobjs[1], zfitobjs[1], flags[1],
zfindobjs[2], zfitobjs[2], flags[2],
zfindobjs[3], zfitobjs[3], flags[3],
zfindobjs[4], zfitobjs[4], flags[4])
output = None
# Write output
if self.dest is None:
output = io.WriteRedmonster(zpick, overwrite=self.overwrite)
else:
if type(self.dest) is str:
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
else:
try:
self.dest = str(self.dest)
output = io.WriteRedmonster(zpick,
dest=self.dest,
overwrite=self.overwrite)
except:
print(
'Could not convert dest to string - writing to default directory and NOT clobbering old files!'
)
output = io.WriteRedmonster(zpick, overwrite=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiberid()
else: output.write_plate()
#zssp2 = zfinder.ZFinder(fname='ndArch-ssp_galaxy_emit-v002.fits',
#npoly=4, zmin=-0.01, zmax=1.2)
#zssp2.zchi2(specs.flux, specs.loglambda, specs.ivar, npixstep=2)
zstar = zfinder.ZFinder(fname='ndArch-all-CAP-grids.fits', npoly=4, zmin=-.005,
zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
zqso = zfinder.ZFinder(fname='ndArch-QSO-V003.fits', npoly=4, zmin=.4, zmax=3.5)
zqso.zchi2(specs.flux, specs.loglambda, specs.ivar, npixstep=4)
''' Instantiate ZFitter to do subgrid fitting. zchi2_ssp is chi^2 array
from zfinder object above, and zssp.zbase is redshift-pixel baseline
over the range explored by zfinder.
'''
zfit_ssp1 = zfitter.ZFitter(zssp1.zchi2arr, zssp1.zbase)
''' Do actual subgrid refinement and fitting. Best-fit and
second-best-fit redshifts will be in [nfibers,2] shaped array in
zfit_*****.z , and associated errors are in zfit_*****.z_err .
This routine also flags for small delta chi^2 and if the global
minimum chi^2 is on an edge of the input chi^2 array. z_refine()
includes an optional parameter, 'width', which specifies half the
width of a window around the global chi2 minimum in which local minima
are ignored during calculation of second-best redshift and flagging
small delta-chi2. If not specified, 'width' defaults to 15.
'''
zfit_ssp1.z_refine2()
''' Same as above for second template.'''
completeness = []
purity = []
#-------------------------------------------------------------------------------------------------------
# Loop over various dchi2 thresholds
for inc in range(2):
this_thresh = 41 + inc
threshold_vals.append(this_thresh)
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=fiberid)
zssp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits', type='GALAXY', npoly=4, zmin=-0.01, zmax=1.2)
zssp.zchi2(specs.flux, specs.loglambda, specs.ivar)
zstar = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', type='STAR', npoly=4, zmin=-.005, zmax=.005)
zstar.zchi2(specs.flux, specs.loglambda, specs.ivar)
zfit_ssp = zfitter.ZFitter(zssp.zchi2arr, zssp.zbase)
zfit_ssp.z_refine(threshold=this_thresh)
zfit_star = zfitter.ZFitter(zstar.zchi2arr, zstar.zbase)
zfit_star.z_refine(threshold=this_thresh)
zpick = zpicker.ZPicker(specs, zssp, zfit_ssp, zstar, zfit_star)
ssp_flags = n.zeros(len(fiberid))
star_flags = n.zeros(len(fiberid))
for ifiber in range(len(fiberid)):
ssp_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zssp.zwarning[ifiber])) | int(zfit_ssp.zwarning[ifiber])
star_flags[ifiber] = (int(specs.zwarning[ifiber]) | int(zstar.zwarning[ifiber])) | int(zfit_star.zwarning[ifiber])
purity.append( (len(n.where(ssp_flags == 0)))/float(nfib) )
completeness.append( (len(n.where(abs(zpick.z[:,0]-zperson) <= .0001)))/float(nfib) )
#-------------------------------------------------------------------------------------------------------
def reduce_gen(self, filepath=None, data_range=None, chi2file=False):
self.chi2file = False
if filepath is None: print("Invalid file path.")
else:
specs = io2.SpecGen(filepath=filepath)
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i],
nproc=self.nproc))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr,
zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i],
nproc=self.nproc))
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr,
zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
zpick = zpicker2.ZPicker(specs, zfindobjs, zfitobjs, flags)
output = None
# Write output
if self.dest is None:
output = io2.WriteRedmonster(zpick,
dest=filepath,
overwrite=True)
if output:
output.write_gen()
def reduce_plate_mjd(self,
plate=None,
mjd=None,
fiberid=None,
data_range=None,
chi2file=False,
platepath=None):
print "\nPlate %s MJD %s Fiber %s" % (plate, mjd, fiberid)
self.chi2file = chi2file
# Check types and try to convert to proper types if necessary
if fiberid is not None:
if type(fiberid) is not list:
try:
fiberid = [fiberid]
fiberid = list(map(int, fiberid))
except ValueError:
try:
fiberid = fiberid.tolist()
fiberid = list(map(int, fiberid))
except ValueError:
print('fiberid not set properly - running full plate!')
else:
fiberid = list(map(int, fiberid))
# Spec
specs = spec.Spec(plate=plate,
mjd=mjd,
fiberid=fiberid,
platepath=platepath)
fiberid = specs.fiberid
# ZFinder, ZFitter
zfindobjs = []
zfitobjs = []
if (self.zmin is not None) & (self.zmax is not None):
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
zmin=self.zmin[i],
zmax=self.zmax[i],
nproc=self.nproc))
if self.mask:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file,
linelist=__linelist__)
else:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
else:
for i in range(len(self.templates)):
zfindobjs.append(
zfinder.ZFinder(fname=self.templates[i],
group=self.group[i],
npoly=self.npoly[i],
npixstep=self.npixstep[i],
nproc=self.nproc))
if mask:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file,
linelist=__linelist__)
else:
zfindobjs[i].zchi2(specs.flux,
specs.loglambda,
specs.ivar,
npixstep=self.npixstep[i],
plate=plate,
mjd=mjd,
fiberid=fiberid[0],
chi2file=self.chi2file)
zfitobjs.append(
zfitter.ZFitter(zfindobjs[i].zchi2arr, zfindobjs[i].zbase))
zfitobjs[i].z_refine2()
# Flags
flags = []
for i in range(len(zfindobjs)):
flags.append(misc.comb_flags(specs, zfindobjs[i], zfitobjs[i]))
# ZPicker
zpick = zpicker2.ZPicker(specs, zfindobjs, zfitobjs, flags)
output = None
# Write output
if self.dest is None:
output = io2.WriteRedmonster(zpick, overwrite=True)
else:
if type(self.dest) is str:
output = io2.WriteRedmonster(zpick,
dest=self.dest,
overwrite=True)
else:
try:
self.dest = str(self.dest)
output = io2.WriteRedmonster(zpick,
dest=self.dest,
overwrite=True)
except Exception as e:
print(
'Could not convert dest to string - writing to \
default directory and NOT clobbering old files! \
Exception: %r' % e)
output = io2.WriteRedmonster(zpick, overwrite=True)
if output:
if len(zpick.fiberid) == 1: output.write_fiber()
else: output.write_plate()
def find_comp_purity(this_thresh, args):
purity = []
completeness = []
run = 1
for iarg in args:
print('Running plate %s of 8' % (run))
run += 1
plate = iarg[0]
mjd = iarg[1]
fiberid = iarg[2]
zperson = iarg[3]
origlen = float(len(fiberid))
badfibs = n.where(n.asarray(iarg[5]) != 'v5_4_9 ok')[0]
usefiberid = n.delete(n.asarray(fiberid), badfibs).tolist()
usezperson = n.delete(n.asarray(zperson), badfibs).tolist()
specs = spec.Spec(plate=plate, mjd=mjd, fiberid=usefiberid)
hdu = fits.open(
'/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-ssp_galaxy_glob.fits'
% plate)
#hdu = fits.open('/Users/boltonlab3/scratch/chi2arr-%s-ssp_galaxy_glob.fits' % plate)
sspchi2arr = hdu[0].data
zbasessp = hdu[1].data.ZBASE
hdu = fits.open(
'/uufs/astro.utah.edu/common/home/u0814744/scratch/screens/chi2arr-%s-spEigenStar.fits'
% plate)
#hdu = fits.open('/Users/boltonlab3/scratch/chi2arr-%s-spEigenStar.fits' % plate)
starchi2arr = hdu[0].data
zbasestar = hdu[1].data.ZBASE
zfit_ssp = zfitter.ZFitter(sspchi2arr, zbasessp)
zfit_ssp.z_refine(threshold=this_thresh)
zfit_star = zfitter.ZFitter(starchi2arr, zbasestar)
zfit_star.z_refine(threshold=this_thresh)
ssp_flags = misc.comb_flags_2(specs, zfit_ssp.zwarning)
star_flags = misc.comb_flags_2(specs, zfit_star.zwarning)
zpick = Hacked_zpicker(specs, usefiberid, sspchi2arr, zfit_ssp,
ssp_flags, starchi2arr, zfit_star, star_flags)
#completeness.append( (len(n.where(zpick.zwarning == 0)[0]))/float(len(fiberid)) )
completeness.append(
(len(n.where((zpick.zwarning.astype(int) & 4) == 0)[0])) /
float(len(usefiberid)))
purity_set = zpick.z[n.where((zpick.zwarning.astype(int) & 4) == 0)[0],
0]
purity_zperson = n.asarray(usezperson)[n.where(
(zpick.zwarning.astype(int) & 4) == 0)[0]]
#purity.append( (len(n.where(abs(zpick.z[:,0]-zperson) <= .0005)[0]))/float(len(fiberid)) )
purity.append(
(len(n.where(abs(purity_set - purity_zperson) <= .0005)[0])) /
float(len(purity_set)))
this_comp = n.mean(completeness)
this_pur = n.mean(purity)
return this_comp, this_pur
ztemp = zfinder.ZFinder(fname='ndArch-ssp_em_galaxy-v000.fits',
npoly=4,
zmin=-0.01,
zmax=1.2)
# Use Carlos' stellar templates
#ztemp = zfinder.ZFinder(fname='ndArch-all-CAP-grids.fits', npoly=4,
#zmin=-.005, zmax=.005)
# Use spEigenstars from IDL pipeline
#ztemp = zfinder.ZFinder(fname='ndArch-spEigenStar-55734.fits', npoly=4,
#zmin=-.005, zmax=.005)
# Use Nao's quasars
#ztemp = zfinder.ZFinder(fname='ndArch-QSO-V003.fits', npoly=4, zmin=.4,
#zmax=3.5)
ztemp.zchi2(specs.flux, specs.loglambda, specs.ivar, npixstep=1)
zfit_temp = zfitter.ZFitter(ztemp.zchi2arr, ztemp.zbase)
zfit_temp.z_refine()
#temp_flags = misc.comb_flags(specs, ztemp, zfit_temp)
#zpick = zpicker.ZPicker(specs, ztemp, zfit_temp)
# Solve for parameters, create model
import pdb
pdb.set_trace()
minloc = n.unravel_index(ztemp.zchi2arr.argmin(), ztemp.zchi2arr.shape)
pmat = n.zeros((specs.flux.shape[-1], ztemp.npoly + 1))
this_temp = ztemp.templates[minloc[1:-1]]
pmat[:, 0] = this_temp[(minloc[-1] * ztemp.npixstep) +
ztemp.pixoffset:(minloc[-1] * ztemp.npixstep) +
ztemp.pixoffset + specs.flux.shape[-1]]
polyarr = poly_array(ztemp.npoly, specs.flux.shape[-1])
pmat[:, 1:] = n.transpose(polyarr)