def getSpectrum(self, grism, extconf, path):
fluxs, fvars, waves, chi2s = [], [], [], []
with h5table.H5Table(grism.dataset, 'ddt', path=path) as h5:
for detname, detimg in grism:
# open the detector
h5det = h5[detname]
detconf = extconf[detname]
# read the images
sci, scihdr = grism.readfits(detconf.sciext, detconf.extver)
unc, unchdr = grism.readfits(detconf.uncext, detconf.extver)
dqa, dqahdr = grism.readfits(detconf.dqaext, detconf.extver)
# load the beamsy
for beam, beamconf in detconf:
h5beam = h5det[beam]
beamconf = detconf[beam]
# read the table
ddt = h5table.DDT(self.segid)
ddt.readH5(h5beam)
# apply flat, pam, etc. to sci/unc
# fit the sky
sky, box = self.fitSky(sci, unc, dqa, ddt)
# clean the sky
cln = self.cleanSky(sky)
sci[box] -= cln
del cln, sky
# fit the object
vals = self.fitSource(sci, unc, dqa, ddt, beamconf)
# clean up
del sci, unc, dqa
# output the results
fluxs.extend(vals[0])
fvars.extend(vals[1])
waves.extend(vals[2])
chi2s.extend(vals[3])
del vals # more cleaning
return fluxs, fvars, waves, chi2s
def maskBeams(self,flt,mskconf,path):
masks={}
if len(mskconf.beams)!=0:
print("[info]Making beam mask for: {}".format(flt.filename))
with h5table.H5Table(flt.dataset,path=path,suffix='omt') as h5:
# loop over detectors
for detname,detimg in flt:
h5det=h5[detname] # get the group
detconf=mskconf[detname] # grism config
mask=np.ones(detimg.naxis,dtype=np.bool)
for beam,beamconf in detconf:
h5beam=h5det[beam]
for segid in h5beam:
xyg=h5beam[segid][:]
xg,yg=indices.one2two(xyg,detimg.naxis)
mask[yg,xg]=False
masks[detname]=mask
return masks
def makeODTs(grism,sources,grismconf,path,remake,nsub):
# create the table
tab=h5table.H5Table(grism.dataset,TTYPE,path=path)
# remake the table?
#if not (remake or not os.path.isfile(tab.filename)):
# return tab.filename
if remake:
if os.path.isfile(tab.filename):
os.remove(tab.filename)
else:
return tab.filename
# pixel based ------------------------------
dx=np.array([0,0,1,1]) # HARDCODE
dy=np.array([0,1,1,0]) # HARDCODE
#-------------------------------------------
with tab as h5:
# add some stuff to that header
h5utils.writeAttr(h5,'segmap',sources.segmap)
h5utils.writeAttr(h5,'nsource',np.uint16(len(sources)))
h5utils.writeAttr(h5,'detimage',sources.obsdata.detImage)
h5utils.writeAttr(h5,'detband',sources.obsdata.detName)
h5utils.writeAttr(h5,'maglimit',np.float32(sources.maglimit))
# process each detector
for det,detconf in grismconf:
#detgrp=detGroup(h5,det,detconf)
detgrp=h5.require_group(det)
# get the center of the detector
xc,yc=detconf.naxis/2.
# get this grism image
thisGrism=grism[det]
# the pixel area of this detector
detpixelarea=thisGrism.pixelarea
for beam,beamconf in detconf:
beamgrp=detgrp.require_group(beam)
if remake:
sourcesDone=[]
else:
sourcesDone=list(beamgrp.keys())
#if beam in detgrp:
# beamgrp=detgrp[beam]
# sourcesDone=list(beamgrp.keys())
#else:
# beamgrp=detgrp.create_group(beam)
# sourcesDone=[]
# compute the set difference
#segids=[src.name for src in sources]
#done=list(beamgrp.keys())
#toDo=segids-done
# get the ODT wavelenegths. NOTE: This are *NOT* the
# same as the extraction wavelengths due to NSUB.
# here using center of detector. Could improve this by
# putting inside loop on sources and take (xc,yc)
# from the source. This is just faster and doesn't
# seem to be a problem just yet
wav=beamconf.wavelengths(xc,yc,nsub)
dwav=wav[1]-wav[0]
# put some stuff in the table
h5utils.writeAttr(beamgrp,'wav0',np.float32(wav[0]))
h5utils.writeAttr(beamgrp,'wav1',np.float32(wav[-1]))
h5utils.writeAttr(beamgrp,'dwav',np.float32(dwav))
# process each source
for segid,src in sources:
if src.name not in sourcesDone: # only process new sources
# compute ratio of pixel area between
# the FLT and the source
pixrat=detpixelarea/src.pixelarea
# make an ODT
odt=h5table.ODT(src.segid,wav=wav)
# process each pixel in the source
for xd,yd,wd in src:
# convert the corners of the direct image to the
# corresponding grism image
xg,yg=src.xy2xy(xd+dx,yd+dy,thisGrism)
# disperse those corners
xyg,lam,val=beamconf.specDrizzle(xg,yg,wav)
if len(xyg)!=0:
# create the PDT
pix=(int(xd-src.ltv[0]),int(yd-src.ltv[1]))
pdt=h5table.PDT(pix)
pdt.extend(xyg,lam,val)
# scale the PDT by:
# 1. direct image weight (wd),
# 2. ratio of pixel areas between seg & FLT
# 3. wavelength sampling (trapezoidal rule)
pdt*=(wd*pixrat*dwav)
# append the PDT
odt.extend(pdt)
# if ODT is valid, then write it!
if len(odt)!=0:
xyc=np.float32(src.xyc-src.ltv)
if TTYPE=='ODT':
odt.writeH5(beamgrp,RA=src.adc[0],\
Dec=src.adc[1],\
xc=xyc[0],yc=xyc[1],\
mag=np.float32(src.mag),\
area=np.float32(src.area),\
npix=np.uint32(src.npix))
elif TTYPE=='DDT':
ddt=odt.decimate(thisGrism.npix)
ddt.writeH5(beamgrp,RA=src.adc[0],\
Dec=src.adc[1],\
xc=xyc[0],yc=xyc[1],\
mag=np.float32(src.mag),\
area=np.float32(src.area),\
npix=np.uint32(src.npix))
else:
raise NotImplementedError("invalid table type")
return tab.filename
def makeOMTs(flt,sources,grismconf,path,remake,nsub):
print("[info]Making the OMTs")
# create the table
tab=h5table.H5Table(flt.dataset,'omt',path=path)
# remake the table?
if remake and os.path.isfile(tab.filename):
os.remove(tab.filename)
with tab as h5:
# add some stuff to that header
h5utils.writeAttr(h5,'segmap',sources.segmap)
h5utils.writeAttr(h5,'nsource',np.uint16(len(sources)))
h5utils.writeAttr(h5,'detimage',sources.obsdata.detImage)
h5utils.writeAttr(h5,'detband',sources.obsdata.detName)
h5utils.writeAttr(h5,'maglimit',np.float32(sources.maglimit))
for det,detconf in grismconf:
detgrp=h5.require_group(det)
#if det in h5:
# detgrp=h5[det]
#else:
# detgrp=h5.create_group(det)
# get the center of the detector
xc,yc=detconf.naxis/2.
thisGrism=flt[det]
for beam,beamconf in detconf:
beamgrp=detgrp.require_group(beam)
if remake:
sourcesDone=[]
else:
sourcesDone=list(beamgrp.keys())
#if beam in detgrp:
# beamgrp=detgrp[beam]
# sourcesDone=list(beamgrp.keys())
#else:
# beamgrp=detgrp.create_group(beam)
# sourcesDone=[]
wav=beamconf.wavelengths(xc,yc,1) # force nsub=1
# add some stuff to table
h5utils.writeAttr(beamgrp,'wav0',np.float32(wav[0]))
h5utils.writeAttr(beamgrp,'wav1',np.float32(wav[-1]))
h5utils.writeAttr(beamgrp,'dwav',np.float32(wav[1]-wav[0]))
# process each source
for segid,src in sources:
if src.name not in sourcesDone:
xd,yd=src.convexHull
xg,yg=src.xy2xy(xd,yd,thisGrism)
xyg,lam,val=beamconf.specDrizzle(xg,yg,wav)
if len(xyg)!=0:
omt=h5table.OMT(segid)
xyg=indices.unique(np.array(xyg))
omt.extend(xyg)
omt.writeH5(beamgrp,RA=src.adc[0],\
Dec=src.adc[1],\
xc=xyc[0],yc=xyc[1],\
mag=np.float32(src.mag),\
area=np.float32(src.area),\
npix=np.uint32(src.npix))
#omt.writeH5(beamgrp)
return tab.filename
def loadFLT(self,flt,sources,extconf,mskconf,grismFF,pb,path):
# output stuff
i = []
j = []
aij = []
# make mask for this FLT
masks=self.maskBeams(flt,mskconf,path)
import pickle,os,psutil
pid = os.getpid()
py = psutil.Process(pid)
# open the H5Table
with h5table.H5Table(flt.dataset,self.TTYPE,path=path) as h5:
#if __RAM__:
# print("start loadFLT:",py.memory_info()[0]/1024/1024/1024)
# loop over detectors
for detname,detimg in flt:
h5det=h5[detname] # get the group
detconf=extconf[detname] # grism config
# save this for easy access later
self.npix=detimg.npix
# read the images
sci,scihdr=flt.readfits(detconf.sciext,detconf.extver)
unc,unchdr=flt.readfits(detconf.uncext,detconf.extver)
dqa,dqahdr=flt.readfits(detconf.dqaext,detconf.extver)
xyg=[] # a container
# make a good pixel mask
gpx=(dqa == 0) & (unc > 0)
if len(masks)!=0:
gpx &= masks[detname]
del dqa,dqahdr,unchdr # don't need these anymore
#if __RAM__:
# print("calling loadBeams:",py.memory_info()[0]/1024/1024/1024)
# call a load beam
data=self.loadBeams(h5det,detconf,detimg,unc,gpx,sources,\
grismFF)
self.imgindex+=1
#if __RAM__:
# print("back from loadBeams:",py.memory_info()[0]/1024/1024/1024)
# collect the results
if len(data[3])!=0:
# collect the matrix terms
# i.extend(data[0])
# j.extend(data[1])
# aij.extend(data[2])
#i = np.hstack((i,data[0]))
#j = np.hstack((j,data[1]))
#aij = np.hstack((aij,data[2]))
i.append(data[0])
j.append(data[1])
aij.append(data[2])
# compute pixel (x,y) pairs
xyg=indices.unique(np.array(data[3]))
# the following line was encapsulated in unqiify
# (written by R Ryan), but needs to be explicitly
# put in for the differences with the way unique was
# implemented (could put sort flag in indices.unique)
xyg=np.sort(xyg)
xg,yg=indices.one2two(xyg,detimg.naxis)
xg=xg.astype(int)
yg=yg.astype(int)
bi=sci[yg,xg]/unc[yg,xg]
del xg,yg # clean up memory usage
# check for bad values in bi
g=np.where(np.isinf(bi))[0]
if len(g)!=0:
print('[warn]Infinite values in bi; is UNC image ok?')
print(bi[g])
raise RuntimeError("Infinite values. aborting.")
# like IDL's push
#self.bi.extend(bi)
self.bi = np.hstack((self.bi,bi))
del bi # again, every little bit helps
# save the memory usage
del data
i = np.hstack(i)
j = np.hstack(j)
aij = np.hstack(aij)
#if __RAM__:
# print("done with loadBeams:",py.memory_info()[0]/1024/1024/1024)
return i,j,aij
def simulateWorker(flt, conf, grismconf, grismflat, sources, overwrite=True):
''' helper function to facilitate multiprocessing '''
path = conf['tables']['path']
# make the output fits file
hdul = fits.HDUList()
# get the primary header from the FLT
#hdr=flt.phdu
hdr = fits.Header()
# make a timestamp
now = datetime.datetime.now()
# update the PHDU for the output image
hdr.append(('ORIGIN', 'pyLINEAR', 'how the file was created'), end=True)
hdr.append(('VERSION', __version__, 'pyLINEAR version'), end=True)
hdr.append(('DATE',now.strftime("%Y-%m-%d"),\
'date this file was written (yyyy-mm-dd)'),end=True)
hdr.add_blank(value='', after='DATE')
hdr.add_blank(value='/ Observational Properties')
hdr.add_blank(value='')
hdr.append(('TELESCOP',grismconf.telescope,\
'telescope used to "acquire" data'),end=True)
hdr.append(('INSTRUME',grismconf.camera,\
'instrument used to "acquire" data'),end=True)
hdr.append(('DETECTOR', grismconf.instrument, 'detector in use'), end=True)
hdr.append(('ROOTNAME',flt.dataset,'rootname of the observation set'),\
end=True)
hdr.append(('OBSTYPE','SPECTROSCOPIC',\
'observation type - imaging or spectroscopic'),end=True)
hdr.append(('FILTER',grismconf.grism,\
'element selected from filter wheel'),end=True)
hdr.add_blank(value='', after='FILTER')
hdr.add_blank(value='/ Simulation Properties')
hdr.add_blank(value='')
hdr.append(('NSOURCE', len(sources), 'number of simulated sources'),
end=True)
hdr.append(('SEGMAP', sources.segmap, 'segmentation map'), end=True)
hdr.append(('DETIMG', sources.obsdata.detImage, 'detection image'),
end=True)
hdr.add_blank(value='', after='DETIMG')
hdr.add_blank(value='/ Noise Properties')
hdr.add_blank(value='')
hdr.append(('NOISE', conf['noise']['perform'], 'is noise added?'),
end=True)
if conf['noise']['perform']:
hdr.append(('SKYRATE',conf['noise']['skyrate'],\
'sky count rate [e-/s]'),end=True)
hdr.append(('EXPTIME',conf['noise']['exptime'],\
'exposure time [s]'),end=True)
after = 'EXPTIME'
else:
after = 'NOISE'
hdr.add_blank(value='', after=after)
hdr.add_blank(value='/ YAML Input')
hdr.add_blank(value='')
hdr.add_blank(value='')
for value in conf:
hdr.add_comment(value=value)
# put this in the FITS FILE
hdul.append(fits.PrimaryHDU(header=hdr))
# open the H5table
with h5table.H5Table(flt.dataset, path=path, suffix=TTYPE) as h5:
# loop over detectors within an FLT
for detname, det in flt:
detgrp = h5[detname]
detconf = grismconf[detname]
# get the EXTVER, which describes which detector this is
extver = detconf.extver
# create an empty array
sci = np.zeros(np.flip(det.naxis, 0), dtype=SCITYPE)
for beam, beamconf in detconf:
beamgrp = detgrp[beam]
for segid, src in sources:
if TTYPE == 'odt':
odt = h5table.ODT(segid)
odt.readH5(beamgrp)
ddt = odt.decimate()
del odt
elif TTYPE == 'ddt':
ddt = h5table.DDT(segid)
ddt.readH5(beamgrp)
else:
raise NotImplementedError("Invalid TTYPE")
if len(ddt) != 0:
# compute the (x,y) pair for each val in the DDT
xg, yg = indices.one2two(ddt.xyg, det.naxis)
# get scaling terms
s = beamconf.sensitivity(ddt.wav)
f = src.sed.interpolate(ddt.wav)
p = det.pixelArea(xg, yg)
ff = grismflat(xg, yg, ddt.wav, detname)
# scale the DDT
ddt *= (s * f * p * ff)
del s, f, p, ff
# sum over pixels
val, xyu = indices.decimate(ddt.xyg, ddt.val)
del ddt
# get unique coordinates
xg, yg = indices.one2two(xyu, det.naxis)
del xyu
# put flux in the image
sci[yg, xg] += val
del val
# update the SCI image for noise and make an UNC image
sci, unc = addNoise(conf['noise'], sci)
# create a DQA image (set to all 0)
dqa = np.full_like(sci, 0, dtype=DQATYPE)
# the SCI image
hdr = det.mkhdr(SCITYPE, extname=detconf.sciext, extver=extver)
hdul.append(fits.ImageHDU(data=sci, header=hdr))
# the UNC image
hdr = det.mkhdr(UNCTYPE, extname=detconf.uncext, extver=extver)
hdul.append(fits.ImageHDU(data=unc, header=hdr))
# the DQA image
hdr = det.mkhdr(DQATYPE, extname=detconf.dqaext, extver=extver)
hdul.append(fits.ImageHDU(data=dqa, header=hdr))
# output the file
outfile = flt.filename
print('writing simulated image {}'.format(outfile))
hdul.writeto(outfile, overwrite=overwrite)
# do we gzip?
if conf['gzip']:
gzip.gzip(outfile)
outfile += '.gz'
return outfile
def groupFLT(flt, sources, extconf, path, minarea=0.1):
#print('loading the polygons for {}'.format(flt.dataset))
# get the polygons for this FLT:
with h5table.H5Table(flt.dataset, TTYPE, path=path) as h5:
for detname, detimg in flt:
h5det = h5[detname]
detconf = extconf[detname]
for beam, beamconf in detconf:
h5beam = h5det[beam]
ids = []
polys = []
for segid, src in sources:
# read the DDT
ddt = h5table.DDT(src.segid)
ddt.readH5(h5beam)
if len(ddt) != 0:
# collect the points accordingly
xyg = ddt.xyg.to_numpy
xyg = indices.unique(xyg)
x, y = indices.one2two(xyg, detimg.naxis)
del xyg
# get the vertices
xy = convexhull.vertices(x, y)
# reform to (x,y) pairs
xy = list(zip(*xy))
# make into a polygon from Shapely
poly = Polygon(xy)
# save the results
ids.append([segid])
polys.append(poly)
# At this point, we've made shapely.Polygons out of a given DDT
#print('grouping the polygons for {}'.format(flt.dataset))
# group those sources with Shapely math
data = list(zip(ids, polys))
nnew = ndata = len(ids)
if nnew == 0:
#print('[warn]No objects to group for {}'.format(flt.dataset))
return []
while nnew != 0:
groups = []
while len(data) != 0:
thisid, thispoly = data.pop(0)
#ids=thisid
for i, (testid, testpoly) in enumerate(data):
inter = thispoly.intersection(testpoly)
#inter=inter.area # intersection area
r1 = inter.area / testpoly.area
r2 = inter.area / thispoly.area
if r1 > minarea and r2 > minarea:
#if area>minarea:
data.pop(i) # it was grouped, so remove it from the list
#print(r1,r2)
#print(inter.area,testpoly.area,thispoly.area)
#fig,ax=plt.subplots(1,1)
#ax.plot(*thispoly.exterior.xy, color='#6699cc', alpha=0.7,
# linewidth=3, solid_capstyle='round', zorder=2)
#ax.plot(*testpoly.exterior.xy, color='#cccccc', alpha=0.7,
# linewidth=3, solid_capstyle='round', zorder=2)
#ax.set_title('Polygon')
#plt.show()
# update the this
thispoly = thispoly.union(testpoly)
thisid.extend(testid)
#print(i,area,thisid)
groups.append((thisid, thispoly))
data = groups
nnew = ndata - len(data)
ndata = len(data)
# get just the IDs
groups = list(zip(*groups))[0]
# return a list of sets
ids = [set(group) for group in groups]
#print(len(ids))
return ids