def movie(band,skypos,tranges,skyrange,framesz=0,width=False,height=False,
verbose=0,memlight=False,coadd=False,response=False,hdu=False,retries=20,
detsize=1.1):
"""Generate a movie (mov)."""
# Not defining stepsz creates a single full depth image.
print tranges
if coadd or (len(tranges)==1 and not framesz) or (not len(tranges)):
if verbose>2:
print 'Coadding across '+str(tranges)
mv = integrate_map(band,skypos,tranges,skyrange,width=width,
height=height,verbose=verbose,memlight=memlight,hdu=hdu,
retries=retries,response=response,detsize=detsize)
else:
for trange in tranges:
stepsz = framesz if framesz else trange[1]-trange[0]
steps = np.ceil((trange[1]-trange[0])/stepsz)
for i,t0 in enumerate(np.arange(trange[0],trange[1],stepsz)):
if verbose>1:
mc.print_inline('Movie frame '+str(i+1)+' of '+
str(int(steps)))
t1 = trange[1] if i==steps else t0+stepsz
img = integrate_map(band,skypos,[[t0,t1]],skyrange,
width=width,height=height,verbose=verbose,
memlight=memlight,hdu=hdu,retries=retries,
response=response,detsize=detsize)
if img.min() == 0 and img.max() == 0:
if verbose>1:
print 'No data in frame {i}. Skipping...'.format(i=i)
continue
try:
mv.append(img)
except:
mv = [img]
return np.array(mv)
def pullphotons(band, ra0, dec0, tranges, radius, events={}, verbose=0,
tscale=1000., calpath='../cal/', chunksz=10e6):
"""Retrieve photons within an aperture from the database."""
events = {'t':[],'ra':[],'dec':[],'xi':[],'eta':[]}
if verbose:
print "Retrieving photons at ["+str(ra0)+", "+str(dec0)+"] within a radius of "+str(radius)
for trange in tranges:
if verbose:
mc.print_inline(" and between "+str(trange[0])+" and "+
str(trange[1])+".")
stream = gQuery.getArray(
gQuery.allphotons(band, ra0, dec0, trange[0], trange[1], radius),
verbose=verbose,retries=100)
if not stream:
continue
events['t'] = events['t']+np.array(np.array(stream,
dtype='float64')[:,0]/tscale).tolist()
# The float64 precision _is_ significant for RA / Dec.
events['ra'] = events['ra']+np.array(np.array(stream,
dtype='float64')[:,1]).tolist()
events['dec'] = events['dec']+np.array(np.array(stream,
dtype='float64')[:,2]).tolist()
events['xi'] = events['xi']+np.array(np.array(stream,
dtype='float32')[:,3]).tolist()
events['eta'] = events['eta']+np.array(np.array(stream,
dtype='float32')[:,4]).tolist()
events['t'] = np.array(events['t'],dtype='float64')
events['ra'] = np.array(events['ra'],dtype='float64')
events['dec'] = np.array(events['dec'],dtype='float64')
events['xi'] = np.array(events['xi'],dtype='float32')
events['eta'] = np.array(events['eta'],dtype='float32')
events = hashresponse(band, events, calpath=calpath, verbose=verbose)
return events
def integrate_map(band,skypos,tranges,skyrange,width=False,height=False,
verbose=0,memlight=False,hdu=False,retries=20,response=False,
detsize=1.1):
""" Integrate an image over some number of time ranges. Use a reduced
memory optimization (at the expense of more web queries) if requested.
"""
imsz = gxt.deg2pix(skypos,skyrange)
img = np.zeros(imsz)
for trange in tranges:
# If memlight is requested, break the integration into
# smaller chunks.
# FIXME: memlight gives slightly wrong answers right now
# This is probably due to a quirk of SQL, per issue #140.
# Deprecating memlight until this can be resolved.
step = memlight if memlight else trange[1]-trange[0]
for i in np.arange(trange[0],trange[1],step):
t0,t1=i,i+step
if verbose:
mc.print_inline('Coadding '+str(t0)+' to '+str(t1))
img += makemap(band,skypos,[t0,t1],skyrange,response=response,
verbose=verbose,detsize=detsize)
if response: # This is an intensity map.
img /= dbt.compute_exptime(band,trange,skypos=skypos,
verbose=verbose)
return img
def fits_header(band,skypos,tranges,skyrange,width=False,height=False,
verbose=0,hdu=False,retries=20):
"""Populate a FITS header."""
if verbose:
mc.print_inline('Populating FITS header.')
hdu = hdu if hdu else pyfits.PrimaryHDU()
wcs = define_wcs(skypos,skyrange,width=width,height=height)
hdu.header['CDELT1'],hdu.header['CDELT2'] = wcs.wcs.cdelt
hdu.header['CTYPE1'],hdu.header['CTYPE2'] = wcs.wcs.ctype
hdu.header['CRPIX1'],hdu.header['CRPIX2'] = wcs.wcs.crpix
hdu.header['CRVAL1'],hdu.header['CRVAL2'] = wcs.wcs.crval
#hdu.header['RA_CENT'],hdu.header['DEC_CENT'] = wcs.wcs.crval # Dupe.
hdu.header['EQUINOX'],hdu.header['EPOCH'] = 2000., 2000.
hdu.header['BAND'] = 1 if band=='NUV' else 2
hdu.header['VERSION'] = 'v{v}'.format(v=__version__)
# Do we want to set the following?
#hdu.header['OW'] = 1
#hdu.header['DIRECT'] = 1
#hdu.header['GRISM'] = 0
#hdu.header['OPAQUE'] = 0
# Put the total exposure time into the primary header
hdu.header['EXPTIME'] = 0.
for trange in tranges:
hdu.header['EXPTIME'] += dbt.compute_exptime(band,trange,
verbose=verbose,retries=retries)
if len(tranges)==1:
# Put the time range into the primary header for a single frame image
hdu.header['EXPSTART'],hdu.header['EXPEND'] = tranges[0]
# These are the proper keywords for this:
hdu.header['TIME-OBS'],hdu.header['TIME-END'] = tranges[0]
return hdu
def movie(band,skypos,tranges,skyrange,framesz=0,width=False,height=False,
verbose=0,memlight=False,coadd=False,response=False,hdu=False,retries=20):
"""Generate a movie (mov)."""
# Not defining stepsz creates a single full depth image.
if coadd or (len(tranges)==1 and not framesz):
if verbose>2:
print 'Coadding across '+str(tranges)
mv = integrate_map(band,skypos,tranges,skyrange,width=width,
height=height,verbose=verbose,memlight=memlight,hdu=hdu,
retries=retries,response=response)
#rr.append(rrhr(band,skypos,tranges,skyrange,response=response,width=width,height=height,stepsz=1.,verbose=verbose,hdu=hdu,retries=retries)) if response else rr.append(np.ones(np.shape(mv)[1:]))
else:
for trange in tranges:
stepsz = framesz if framesz else trange[1]-trange[0]
steps = np.ceil((trange[1]-trange[0])/stepsz)
for i,t0 in enumerate(np.arange(trange[0],trange[1],stepsz)):
if verbose>1:
mc.print_inline('Movie frame '+str(i+1)+' of '+
str(int(steps)))
t1 = trange[1] if i==steps else t0+stepsz
img = integrate_map(band,skypos,[[t0,t1]],skyrange,
width=width,height=height,verbose=verbose,
memlight=memlight,hdu=hdu,retries=retries,
response=response)
try:
mv.append(img)
except:
mv = [img]
# # FIXME: This should not create an rr unless it's requested...
# rr.append(rrhr(band,skypos,[[t0,t1]],skyrange,response=response,width=width,height=height,stepsz=1.,verbose=verbose,retries=retries)) if response else rr.append(np.ones(np.shape(mv)[1:]))
return np.array(mv)
def movie(band,skypos,tranges,skyrange,framesz=0,width=False,height=False,
verbose=0,memlight=False,coadd=False,
response=False,hdu=False,retries=20):
"""Generate a movie (mov)."""
# Not defining stepsz effectively creates a count map.
mv = []
rr = []
if coadd:
if verbose>2:
print 'Coadding across '+str(tranges)
mv.append(countmap(band,skypos,tranges,skyrange,width=width,
height=height,verbose=verbose,memlight=memlight,
hdu=hdu,retries=retries))
rr.append(rrhr(band,skypos,tranges,skyrange,response=response,width=width,height=height,stepsz=1.,verbose=verbose,hdu=hdu,retries=retries)) if response else rr.append(np.ones(np.shape(mv)[1:]))
else:
for trange in tranges:
stepsz = framesz if framesz else trange[1]-trange[0]
steps = np.ceil((trange[1]-trange[0])/stepsz)
for i,t0 in enumerate(np.arange(trange[0],trange[1],stepsz)):
if verbose>1:
mc.print_inline('Movie frame '+str(i+1)+' of '+
str(int(steps)))
t1 = trange[1] if i==steps else t0+stepsz
mv.append(countmap(band,skypos,[[t0,t1]],skyrange,width=width,
height=height,verbose=verbose,
memlight=memlight,hdu=hdu,retries=retries))
# FIXME: This should not create an rr unless it's requested...
rr.append(rrhr(band,skypos,[[t0,t1]],skyrange,response=response,width=width,height=height,stepsz=1.,verbose=verbose,retries=retries)) if response else rr.append(np.ones(np.shape(mv)[1:]))
return np.array(mv),np.array(rr)
def sigmaclip_bg(data,radius,annulus,skypos,maxiter=10,sigmaclip=3.,
gausslim=50.,verbose=0,pixsz=0.000416666666666667):
"""Produce an estimate of background counts within an aperture (radius)
using a sigma clipping method for extracting the background from an
annulus.
This attempts to reproduce the calcuations of the backcalc() function in
mapaps/poissonbg.c of the mission pipeline. (Probably written by Ted Wyder.)
"""
# FIXME: Does not apply response!
# This cut is now handled by the ugly loop below, which barely dodges a
# conceptula issue about fractional pixels...
#ix = np.where((d>annulus[0]) & (d<annulus[1]))
imsz=gxt.deg2pix(skypos,[annulus[1]*2,annulus[1]*2])
wcs=define_wcs(skypos,[annulus[1]*2,annulus[1]*2])
foc_ra,foc_dec=wcs.sip_pix2foc(wcs.wcs_world2pix(data['ra'],data['dec'],1),1)
H,xedges,yedges=np.histogram2d(foc_ra-0.5,foc_dec-0.5,bins=imsz,
range=([ [0,imsz[0]],[0,imsz[1]]]))
# Convert Gaussian sigma to a probability
problim = 0.5*scipy.special.erfc(sigmaclip/np.sqrt(2.0))
# Mask out non-annulus regions... there's probalby a more pythonic way
bgimg=np.copy(H)
for i in range(H.shape[0]):
for j in range(H.shape[1]):
# Add a little buffer to account for pixel widths?
# FIXME? including everything within the annulus...
# if (mc.distance(H.shape[0]/2.,H.shape[1]/2.,i,j)<annulus[0]/pixsz or
if mc.distance(H.shape[0]/2.,H.shape[1]/2.,i,j)>annulus[1]/pixsz:#):
bgimg[i,j]=-1
ix=np.where(bgimg>=0)
m,s=bgimg[ix].mean(),bgimg[ix].std()
d = 1.
for i in range(maxiter):
if d<=10e-5 or m<2:
continue
if m>=gausslim:
# Mask anything outside of 3 sigma from the mean (of unmasked data)
klim=m+sigmaclip*np.sqrt(m)#s
klo=m-sigmaclip*np.sqrt(m)#s
if verbose:
print 'Gaussian cut: {klo} to {klim}'.format(klo=klo,klim=klim)
else:
klim = scipy.special.gammainccinv(m,problim)
klo = -1 # None
if verbose:
print 'Poisson cut: {klo} to {klim}'.format(klo=klo,klim=klim)
ix = np.where((bgimg>=klim) | (bgimg<=klo))
bgimg[ix]=-1
ix=np.where(bgimg>=0)
d = np.abs((bgimg[ix].mean()-m)/m)# - 1)
m,s=bgimg[ix].mean(),bgimg[ix].std()
ix = np.where(bgimg>=0)
return mc.area(radius)*bgimg[ix].mean()/mc.area(pixsz)
def error(data,band,radius,annulus):
N_a = 1
N_b0 = (mc.area(annulus[1])-mc.area(annulus[0]))/mc.area(radius)
N_b = data[band]['bg_eff_area']/mc.area(radius)
B0 = data[band]['bg']
B = data[band]['bg_cheese']
S = gt.mag2counts(data[band]['mag'],band)*data[band]['t_eff']
s2 = {'bg_cheese_err':(S-B)+(N_a+(N_a**2.)/N_b),
'bg_err':(S-B0)+(N_a+(N_a**2.)/N_b0)}
return s2
def define_wcs(skypos,skyrange,width=False,height=False,verbose=0,
pixsz=0.000416666666666667):
"""Define the world coordinate system (WCS)."""
if verbose:
mc.print_inline('Defining World Coordinate System (WCS).')
wcs = pywcs.WCS(naxis=2) # NAXIS = 2
imsz = gxt.deg2pix(skypos,skyrange)
wcs.wcs.cdelt = np.array([-pixsz,pixsz])
wcs.wcs.ctype = ['RA---TAN','DEC--TAN']
wcs.wcs.crpix = [(imsz[1]/2.)+0.5,(imsz[0]/2.)+0.5]
wcs.wcs.crval = skypos
return wcs
def rrhr(band,skypos,tranges,skyrange,width=False,height=False,stepsz=1.,
verbose=0,response=True,hdu=False,retries=20):
"""Generate a high resolution relative response (rrhr) map."""
imsz = gxt.deg2pix(skypos,skyrange)
# TODO the if width / height
flat, flatinfo = cal.flat(band)
npixx,npixy = flat.shape
fltsz = flat.shape
pixsz = flatinfo['CDELT2']
detsize = 1.25
# Rotate the flat into the correct orientation to start.
flat = np.flipud(np.rot90(flat))
# NOTE: This upsample interpolation is done _last_ in the canonical
# pipeline as part of the poissonbg.c routine.
# The interpolation function is "congrid" in the same file.
# TODO: Should this be first order interpolation? (i.e. bilinear)
hrflat = scipy.ndimage.interpolation.zoom(flat,4.,order=0,prefilter=False)
img = np.zeros(hrflat.shape)[
hrflat.shape[0]/2.-imsz[0]/2.:hrflat.shape[0]/2.+imsz[0]/2.,
hrflat.shape[1]/2.-imsz[1]/2.:hrflat.shape[1]/2+imsz[1]/2.]
for trange in tranges:
t0,t1=trange
entries = gQuery.getArray(gQuery.aspect(t0,t1),retries=retries)
n = len(entries)
asptime = np.float64(np.array(entries)[:,2])/tscale
aspra = np.float32(np.array(entries)[:,3])
aspdec = np.float32(np.array(entries)[:,4])
asptwist= np.float32(np.array(entries)[:,5])
aspflags= np.float32(np.array(entries)[:,6])
asptwist= np.float32(np.array(entries)[:,9])
aspra0 = np.zeros(n)+skypos[0]
aspdec0 = np.zeros(n)+skypos[1]
xi_vec, eta_vec = gnomonic.gnomfwd_simple(
aspra,aspdec,aspra0,aspdec0,-asptwist,1.0/36000.,0.)
col = 4.*( ((( xi_vec/36000.)/(detsize/2.)*(detsize/(fltsz[0]*pixsz)) + 1.)/2. * fltsz[0]) - (fltsz[0]/2.) )
row = 4.*( (((eta_vec/36000.)/(detsize/2.)*(detsize/(fltsz[1]*pixsz)) + 1.)/2. * fltsz[1]) - (fltsz[1]/2.) )
vectors = mc.rotvec(np.array([col,row]),-asptwist)
for i in range(n):
if verbose>1:
mc.print_inline('Stamping '+str(asptime[i]))
# FIXME: Clean this mess up a little just for clarity.
img += scipy.ndimage.interpolation.shift(scipy.ndimage.interpolation.rotate(hrflat,-asptwist[i],reshape=False,order=0,prefilter=False),[vectors[1,i],vectors[0,i]],order=0,prefilter=False)[hrflat.shape[0]/2.-imsz[0]/2.:hrflat.shape[0]/2.+imsz[0]/2.,hrflat.shape[1]/2.-imsz[1]/2.:hrflat.shape[1]/2+imsz[1]/2.]*dbt.compute_exptime(band,[asptime[i],asptime[i]+1],verbose=verbose,retries=retries)*gxt.compute_flat_scale(asptime[i]+0.5,band,verbose=0)
return img
def hashresponse(band,events,verbose=0):
"""Given detector xi, eta, return the response at each position."""
# Hash out the response correction
if verbose:
mc.print_inline("Applying the response correction.")
flat, _ = cal.flat(band)
events['col'], events['row'] = xieta2colrow(
events['xi'], events['eta'], band)
events['flat'] = flat[np.array(events['col'], dtype='int16'),
np.array(events['row'], dtype='int16')]
events['scale'] = gxt.compute_flat_scale(events['t'], band)
# TODO: Separately do the binlinearly interpolated response
events['response'] = (events['flat']*events['scale'])
return events
def getcurve(band, ra0, dec0, radius, annulus=None, stepsz=None, lcurve={},
trange=None, tranges=None, verbose=0, coadd=False, minexp=1.,
maxgap=1., maskdepth=20, maskradius=1.5,
photonfile=None, detsize=1.1):
skyrange = [np.array(annulus).max().tolist() if annulus else radius,
np.array(annulus).max().tolist() if annulus else radius,]
if verbose:
mc.print_inline("Getting exposure ranges.")
if tranges is None:
tranges = dbt.fGetTimeRanges(band, [ra0, dec0], trange=trange,
maxgap=maxgap, minexp=minexp, verbose=verbose, detsize=detsize)
elif not np.array(tranges).shape:
print "No exposure time at this location: [{ra},{dec}]".format(
ra=ra0,dec=dec0)
# FIXME: Everything goes to hell if no exposure time is available...
# TODO: Add an ability to specify or exclude specific time ranges
if verbose:
mc.print_inline("Moving to photon level operations.")
# FIXME: This error handling is hideous.
try:
lcurve = quickmag(band, ra0, dec0, tranges, radius, annulus=annulus,
stepsz=stepsz, verbose=verbose, coadd=coadd,
maskdepth=maskdepth,
maskradius=maskradius,photonfile=photonfile)
lcurve['cps'] = lcurve['sources']/lcurve['exptime']
lcurve['cps_bgsub'] = (lcurve['sources']-
lcurve['bg']['simple'])/lcurve['exptime']
lcurve['cps_bgsub_cheese'] = (lcurve['sources']-
lcurve['bg']['cheese'])/lcurve['exptime']
lcurve['mag'] = gxt.counts2mag(lcurve['cps'],band)
lcurve['mag_bgsub'] = gxt.counts2mag(lcurve['cps_bgsub'],band)
lcurve['mag_bgsub_cheese'] = gxt.counts2mag(
lcurve['cps_bgsub_cheese'],band)
lcurve['flux'] = gxt.counts2flux(lcurve['cps'],band)
lcurve['flux_bgsub'] = gxt.counts2flux(lcurve['cps_bgsub'],band)
lcurve['flux_bgsub_cheese'] = gxt.counts2flux(
lcurve['cps_bgsub_cheese'],band)
lcurve['detrad'] = mc.distance(lcurve['detxs'],lcurve['detys'],400,400)
except ValueError:
lcurve['cps']=[]
lcurve['cps_bgsub']=[]
lcurve['cps_bgsub_cheese']=[]
lcurve['mag']=[]
lcurve['mag_bgsub']=[]
lcurve['mag_bgsub_cheese']=[]
lcurve['flux']=[]
lcurve['flux_bgsub']=[]
lcurve['flux_bgsub_cheese']=[]
lcurve['detrad']=[]
if verbose:
mc.print_inline("Done.")
mc.print_inline("")
return lcurve
def bg_mask_sources(band,ra0,dec0,ras,decs,responses,sources,maskradius=1.5):
# At present, masks to 1.5 sigma where FWHM = 2.3548*sigma
for i in range(len(sources['ra'])):
ix = np.where(mc.angularSeparation(sources['ra'][i], sources['dec'][i],
ras,decs)>=(maskradius/2.3548)*np.median(sources['fwhm'][i,:]))
ras, decs, responses = ras[ix], decs[ix], responses[ix]
return ras,decs,responses
def makemap(band,skypos,trange,skyrange,response=False,verbose=0,detsize=1.1):
imsz = gxt.deg2pix(skypos,skyrange)
photons = np.array(gQuery.getArray(gQuery.skyrect(band,
skypos[0],skypos[1],trange[0],trange[1],skyrange[0],skyrange[1]),
verbose=verbose),dtype='float64')
try:
events = {'t':photons[:,0 ]/tscale,'ra':photons[:,1],'dec':photons[:,2],
'xi':photons[:,3],'eta':photons[:,4],
'x':photons[:,5], 'y':photons[:,6]}
except IndexError:
if verbose>2:
print 'No events found at {s} +/- {r} in {t}.'.format(
s=skypos,r=skyrange,t=trange)
return np.zeros(imsz)
# Trim the data on detsize
col, row = ct.xieta2colrow(events['xi'],events['eta'],band)
ix = np.where((1.25/800.)*mc.distance(col,row,400,400)<=detsize)
n = len(ix[0])
m = len(col)
#print 'With detsize {d} using {n} of {m} data.'.format(d=detsize,n=n,m=m)
if n == 0:
return np.zeros(imsz)
for k in events.keys():
events[k] = events[k][ix]
events = ct.hashresponse(band,events)
wcs = define_wcs(skypos,skyrange,width=False,height=False)
coo = zip(events['ra'],events['dec'])
foc = wcs.sip_pix2foc(wcs.wcs_world2pix(coo,1),1)
weights = 1./events['response'] if response else None
H,xedges,yedges=np.histogram2d(foc[:,1]-0.5,foc[:,0]-0.5,bins=imsz,
range=([ [0,imsz[0]],[0,imsz[1]] ]),weights=weights)
return H
def construct_row(i, band, objid, mcat, data):
# Note: mcat['skybg'] is in counts per second per square arcseconds
# where as gPhoton is reporting cps over the aperture area.
return (
objid,
data["t0"][0],
data["t1"][0],
mcat[band]["expt"][i],
data["exptime"][0],
mcat["ra"][i],
mcat["dec"][i],
data["racent"][0],
data["deccent"][0],
mcat[band][4]["mag"][i],
mcat[band][4]["err"][i],
data["mag_bgsub_cheese"][0],
data["mag_bgsub"][0],
data["mag"][0],
mc.distance(data["detxs"], data["detys"], 400, 400)[0],
data["responses"][0],
mcat[band]["skybg"][i],
data["bg"]["simple"][0],
data["bg"]["cheese"][0],
data["bg"]["eff_area"],
)
def fits_header(band,skypos,tranges,skyrange,width=False,height=False,
verbose=0,hdu=False,retries=20):
"""Populate a FITS header."""
if verbose:
mc.print_inline('Populating FITS header.')
hdu = hdu if hdu else pyfits.PrimaryHDU()
wcs = define_wcs(skypos,skyrange,width=width,height=height)
hdu.header['CDELT1'],hdu.header['CDELT2'] = wcs.wcs.cdelt
hdu.header['CTYPE1'],hdu.header['CTYPE2'] = wcs.wcs.ctype
hdu.header['CRPIX1'],hdu.header['CRPIX2'] = wcs.wcs.crpix
hdu.header['CRVAL1'],hdu.header['CRVAL2'] = wcs.wcs.crval
hdu.header['EQUINOX'],hdu.header['EPOCH'] = 2000., 2000.
hdu.header['BAND'] = 1 if band=='NUV' else 2
hdu.header['VERSION'] = 'v{v}'.format(v=__version__)
return hdu
def cheese_bg_area(band,ra0,dec0,annulus,sources,nsamples=10e5,ntests=10):
# This is just a really naive Monte Carlo
ratios = np.zeros(ntests)
for i in range(ntests):
ann_events = bg_mask_annulus(band,ra0,dec0,annulus,
np.random.uniform(ra0-annulus[1],ra0+annulus[1],int(nsamples)),
np.random.uniform(dec0-annulus[1],dec0+annulus[1],int(nsamples)),
np.ones(nsamples))
mask_events= bg_mask_sources(band,ra0,dec0,
ann_events[0],ann_events[1],ann_events[2],sources)
try:
ratios[i] = float(mask_events[2].sum())/float(ann_events[2].sum())
except ZeroDivisionError:
ratios[i] = 0.
return (mc.area(annulus[1])-mc.area(annulus[0]))*ratios.mean()
def xieta2colrow(xi, eta, calfile, detsize=1.25):
"""Convert detector xi, eta into col, row."""
flat = mc.get_fits_data(calfile)
flatinfo = mc.get_fits_header(calfile)
# should be able to get npix from the header...
npixx = flat.shape[0]
npixy = flat.shape[1]
pixsz = flatinfo['CDELT2']
flatfill = detsize/(npixx*pixsz)
col = ((( xi/36000.)/(detsize/2.)*flatfill + 1.)/2. * npixx)
row = (((eta/36000.)/(detsize/2.)*flatfill + 1.)/2. * npixy)
# You could theoretically drop a cut on detector position / detsize here...
# Also, is this cut absolutely necessary? I think it's already been taken
# care of by the flag==0 assertion in the SQL query.
#cut = ((col > 0.) & (col < flat.shape[0]-1) &
# (row > 0.) & (row < flat.shape[1]-1))
#cut = np.where(ix == True)
return col, row
def countmap(band,skypos,tranges,skyrange,width=False,height=False,
verbose=0,memlight=False,hdu=False,retries=20):
"""Create a count (cnt) map."""
imsz = gxt.deg2pix(skypos,skyrange)
count = np.zeros(imsz)
for trange in tranges:
# If memlight is requested, break the integration into
# smaller chunks.
step = memlight if memlight else trange[1]-trange[0]
for i in np.arange(trange[0],trange[1],step):
t0,t1=i,i+step
if verbose:
mc.print_inline('Coadding '+str(t0)+' to '+str(t1))
events = gQuery.getArray(gQuery.rect(band,skypos[0],skypos[1],t0,t1,
skyrange[0],skyrange[1]),
verbose=verbose,retries=retries)
# Check that there is actually data here.
if not events:
if verbose>1:
print "No data in "+str([t0,t1])
continue
times = np.array(events,dtype='float64')[:,0 ]/tscale
coo = np.array(events,dtype='float64')[:,1:]
# If there's no data, return a blank image.
if len(coo)==0:
if verbose:
print 'No data in this frame: '+str([t0,t1])
continue
# Define World Coordinate System (WCS)
wcs = define_wcs(skypos,skyrange,width=False,height=False)
# Map the sky coordinates onto the focal plane
foc = wcs.sip_pix2foc(wcs.wcs_world2pix(coo,1),1)
# Bin the events into actual image pixels
H,xedges,yedges=np.histogram2d(foc[:,1]-0.5,foc[:,0]-0.5,
bins=imsz,range=([ [0,imsz[0]],[0,imsz[1]] ]))
count += H
return count
def shift_and_add(filename,ix=None,verbose=1):
"""Performs a _shift and add_ style coadd on a sequence of images of a
planet. Recenters the center of brightness to the center of the image.
The file indicated by filename is assumed to be .ser format. The ix keyword
takes a list of frame numbers to use.
"""
if verbose>1:
print 'Reading: {f}'.format(f=filename)
header = ser.readheader(filename)
cnt = header['FrameCount']
img = np.zeros([header['ImageHeight'],header['ImageWidth']])
for i in (ix if ix else range(cnt-1)):
frame = ser.readframe(filename,i,header=header)
w,h=find_cob(frame)
vec = [header['ImageHeight']/2.-h,header['ImageWidth']/2.-w]
if verbose:
mc.print_inline('Shift and adding {i} by {vec}'.format(i=i,vec=vec))
img+=scipy.ndimage.interpolation.shift(frame,vec)/cnt
return img
def gphot_params(band,skypos,radius,annulus=None,
verbose=0.,detsize=1.25,stepsz=None,
trange=None,maskdepth=None,maskradius=None):
"""Populate a dict() with parameters that are constant over all bins."""
return {'band':band,'ra0':skypos[0],'dec0':skypos[1],'skypos':skypos,
'trange':trange,'radius':radius,'annulus':annulus,
'stepsz':stepsz,'verbose':verbose,
'maskdepth':maskdepth,'maskradius':maskradius,'detsize':detsize,
'apcorrect1':gxt.apcorrect1(radius,band),
'apcorrect2':gxt.apcorrect2(radius,band),
'detbg':gxt.detbg(mc.area(radius),band)}
def query_photons(band,ra0,dec0,tranges,radius,verbose=0):
"""Retrieve photons within an aperture from the database."""
stream = []
if verbose:
print "Retrieving photons within {rad} degrees of [{r}, {d}]".format(
rad=radius,r=ra0,d=dec0)
for trange in tranges:
if verbose:
mc.print_inline(" and between "+str(trange[0])+" and "+
str(trange[1])+".")
thisstream = gQuery.getArray(
gQuery.allphotons(band, ra0, dec0, trange[0], trange[1], radius),
verbose=verbose,retries=100)
stream.extend(thisstream)
stream = np.array(stream, 'f8').T
colnames = ['t', 'ra', 'dec', 'xi', 'eta', 'x', 'y']
dtypes = ['f8', 'f8', 'f8', 'f4', 'f4', 'f4', 'f4']
cols = map(np.asarray, stream, dtypes)
events = dict(zip(colnames, cols))
events['t']/=tscale # Adjust the timestamp by tscale
return events
def movie_tbl(band,tranges,verbose=0,framesz=0,retries=20):
"""Initialize a FITS table to contain movie frame information."""
if verbose:
mc.print_inline('Populating exposure time table.')
tstarts,tstops,exptimes=[],[],[]
for trange in tranges:
stepsz = framesz if framesz else trange[1]-trange[0]
steps = np.ceil((trange[1]-trange[0])/stepsz)
for i,t0 in enumerate(np.arange(trange[0],trange[1],stepsz)):
t1 = trange[1] if i==steps else t0+stepsz
tstarts.append(t0)
tstops.append(t1)
exptimes.append(dbt.compute_exptime(band,[t0,t1],
verbose=verbose,retries=retries))
col1 = pyfits.Column(name='tstart',format='E',array=np.array(tstarts))
col2 = pyfits.Column(name='tstop',format='E',array=np.array(tstops))
col3 = pyfits.Column(name='exptime',format='E',array=np.array(exptimes))
cols = pyfits.ColDefs([col1,col2,col3])
tbl = pyfits.BinTableHDU.from_columns(cols)
return tbl
def cheese_bg(band,ra0,dec0,radius,annulus,ras,decs,responses,maskdepth=20.,
maskradius=1.5,eff_area=False,sources=False):
""" Returns the estimate number of counts (not count rate) within the
aperture based upon a masked background annulus.
"""
if not sources:
sources = bg_sources(band,ra0,dec0,annulus[1],maskdepth=maskdepth)
bg_counts = bg_mask(band,ra0,dec0,annulus,ras,decs,responses,
sources)[2].sum()
if not eff_area:
eff_area = cheese_bg_area(band,ra0,dec0,annulus,sources)
return mc.area(radius)*bg_counts/eff_area if eff_area else 0.
def construct_row(i,band,objid,mcat,data):
# Note: mcat['skybg'] is in counts per second per square arcseconds
# where as gPhoton is reporting cps over the aperture area.
return (objid, data['t0'][0], data['t1'][0],
mcat[band]['expt'][i], data['exptime'][0],
mcat['ra'][i], mcat['dec'][i],
data['racent'][0], data['deccent'][0],
mcat[band][4]['mag'][i], mcat[band][4]['err'][i],
data['mag_bgsub_cheese'][0],
data['mag_bgsub'][0], data['mag'][0],
mc.distance(data['detxs'],data['detys'],400,400)[0],
data['responses'][0], mcat[band]['skybg'][i],
data['bg']['simple'][0], data['bg']['cheese'][0],
data['bg']['eff_area'])
def read_photons(photonfile,ra0,dec0,tranges,radius,verbose=0,
colnames=['t','x','y','xa','ya','q','xi','eta','ra','dec','flags']):
"""Read a photon list file and return a python dict() with the expected
format.
"""
if verbose:
print 'Reading photon list file: {f}'.format(f=photonfile)
data = pd.io.parsers.read_csv(photonfile,names=colnames)
ra,dec = np.array(data['ra']),np.array(data['dec'])
angsep = mc.angularSeparation(ra0,dec0,ra,dec)
ix = np.array([])
for trange in tranges:
if verbose:
print trange
cut = np.where((angsep<=radius) & (np.isfinite(angsep)))[0]
ix = np.concatenate((ix,cut),axis=0)
events = {'t':np.array(data['t'][ix])/tscale,
'ra':np.array(data['ra'][ix]),
'dec':np.array(data['dec'][ix]),
'xi':np.array(data['xi'][ix]),
'eta':np.array(data['eta'][ix]),
'x':np.array(data['x'][ix]),
'y':np.array(data['y'][ix])}
return events
def quickmag(band, ra0, dec0, tranges, radius, annulus=None, data={},
stepsz=None, verbose=0, maskdepth=20.0,
maskradius=1.5,detsize=1.25,coadd=False, photonfile=None):
if verbose:
mc.print_inline("Retrieving all of the target events.")
trange = [np.array(tranges).min(),np.array(tranges).max()]
try:
searchradius = annulus[1]
except TypeError:
searchradius = radius
data = pullphotons(band, ra0, dec0, tranges, searchradius,
verbose=verbose, photonfile=photonfile)
if verbose:
mc.print_inline("Isolating source from background.")
angSep = mc.angularSeparation(ra0, dec0, data['ra'], data['dec'])
if verbose:
mc.print_inline("Binning data according to requested depth.")
# Multiple ways of defining bins
if coadd:
bins = np.array(trange)
elif stepsz:
bins = np.append(np.arange(min(trange), max(trange), stepsz),
max(trange))
else:
bins = np.unique(np.array(tranges).flatten())
# This is equivalent in function to np.digitize(data['t'],bins) except
# that it's much, much faster. See numpy issue #2656.
ix = np.searchsorted(bins,data['t'],"right")
# Initialize histogrammed arrays
# FIXME: allocate these from a dict of constructors
lcurve_cols = ['counts', 'sources', 'bg_counts','responses',
'detxs', 'detys', 't0_data', 't1_data', 't_mean', 'racent',
'deccent']
lcurve = {'params':gphot_params(band,[ra0,dec0],radius,annulus=annulus,
verbose=verbose,
detsize=detsize,stepsz=stepsz,
trange=trange,maskdepth=maskdepth,
maskradius=maskradius)}
for col in lcurve_cols:
lcurve[col] = np.zeros(len(bins)-1)
# FIXME: Bottleneck. There's probably a way to do this without looping.
# Don't bother looping through anything with no data.
lcurve['bg'] = {'simple':np.zeros(len(bins)-1),
'cheese':np.zeros(len(bins)-1)}
if annulus is not None:
lcurve['bg']['sources'] = bg_sources(band,ra0,dec0,annulus[1],
maskdepth=maskdepth)
lcurve['bg']['eff_area'] = cheese_bg_area(band,ra0,dec0,annulus,
lcurve['bg']['sources'])
else:
lcurve['bg']['sources'] = None
lcurve['bg']['eff_area'] = 0.
if verbose:
mc.print_inline("Populating histograms.")
for cnt,i in enumerate(np.unique(ix)):
# Exclude data outside of the bins in searchsorted.
if i-1<0 or i==len(bins):
continue
if verbose:
mc.print_inline('Binning {i} of {l}.'.format(
i=cnt,l=len(np.unique(ix))))
t_ix = np.where(ix==i)
# TODO: Optionally limit data to specific parts of detector.
rad_ix = np.where((angSep <= radius) & (ix == i))
# NOTE: This checks for the dim edge case where you have photons in
# the annulus but not in the aperture.
if not len(rad_ix[0]):
continue
lcurve['t0_data'][i-1] = data['t'][rad_ix].min()
lcurve['t1_data'][i-1] = data['t'][rad_ix].max()
lcurve['t_mean'][i-1] = data['t'][rad_ix].mean()
lcurve['counts'][i-1] = len(rad_ix[0])
lcurve['sources'][i-1] = (1./data['response'][rad_ix]).sum()
lcurve['responses'][i-1] = data['response'][rad_ix].mean()
lcurve['detxs'][i-1] = data['col'][rad_ix].mean()
lcurve['detys'][i-1] = data['row'][rad_ix].mean()
lcurve['racent'][i-1] = data['ra'][rad_ix].mean()
lcurve['deccent'][i-1] = data['dec'][rad_ix].mean()
if annulus is not None:
ann_ix = np.where((angSep > annulus[0]) &
(angSep <= annulus[1]) & (ix == i))
lcurve['bg_counts'][i-1] = len(ann_ix[0])
# Background is reported as counts within the aperture
lcurve['bg']['simple'][i-1] = (mc.area(radius) *
(1./data['response'][ann_ix]).sum() /
(mc.area(annulus[1])-mc.area(annulus[0])))
lcurve['bg']['cheese'][i-1] = cheese_bg(band, ra0, dec0, radius,
annulus, data['ra'][t_ix], data['dec'][t_ix],
data['response'][t_ix], maskdepth=maskdepth,
eff_area=lcurve['bg']['eff_area'],
sources=lcurve['bg']['sources'])
else:
lcurve['bg_counts'][i-1]=0.
lcurve['bg']['simple'][i-1]=0.
lcurve['bg']['cheese'][i-1]=0.
# Only return bins that contain data.
ix = np.where((np.isfinite(lcurve['sources'])) &
(np.array(lcurve['sources']) > 0))
lcurve['t0'] = bins[ix]
lcurve['t1'] = bins[ix[0]+1]
for col in lcurve_cols:
lcurve[col] = lcurve[col][ix]
if annulus is not None:
lcurve['bg']['simple']=lcurve['bg']['simple'][ix]
lcurve['bg']['cheese']=lcurve['bg']['cheese'][ix]
else:
lcurve['bg']['simple']=0.
lcurve['bg']['cheese']=0.
lcurve['exptime'] = np.array(
[dbt.compute_exptime(band,trange,skypos=[ra0,dec0],
verbose=verbose,coadd=coadd)
for trange in zip(lcurve['t0_data'],lcurve['t1_data'])])
if verbose:
mc.print_inline("Returning curve data.")
lcurve['photons'] = data
return lcurve
def test_angularSeparation(self):
self.assertAlmostEqual(
mc.angularSeparation(10,20,10.1,20.1),0.13720278279273748)
data[band] = pd.read_csv('{base}{band}.csv'.format(
base=base,band=band))
print '{band} sources: {cnt}'.format(
band=band,cnt=data[band]['objid'].shape[0])
"""dMag vs. Mag"""
for band in bands:
dmag = {'gphot_cheese':(lambda band:
data[band]['aper4']-data[band]['mag_bgsub_cheese']),
'gphot_nomask':(lambda band:
data[band]['aper4']-data[band]['mag_bgsub']),
'gphot_sigma':(lambda band:
data[band]['aper4']-data[band]['mag_bgsub_sigmaclip']),
'mcat':lambda band: data[band]['aper4']-
gt.counts2mag(gt.mag2counts(data[band]['mag'],band)-
data[band]['skybg']*3600**2*mc.area(gt.aper2deg(4)),
band)}
bgmodekeys={'gphot_cheese':'mag_bgsub_cheese',
'gphot_nomask':'mag_bgsub',
'gphot_sigma':'mag_bgsub_sigmaclip',
'mcat':'skybg'}
for bgmode in dmag.keys():
for band in bands:
fig = plt.figure(figsize=(8*scl,4*scl))
fig.subplots_adjust(left=0.12,right=0.95,wspace=0.02,
bottom=0.15,top=0.9)
dmag_err=gu.dmag_errors(100.,band,sigma=1.41)
# Make a cut on crazy outliers in the MCAT. Also on det radius and expt.
ix = ((data[band]['aper4']>0) & (data[band]['aper4']<30) &
(data[band]['distance']<300) & (data[band]['t_eff']<300) &
(np.isfinite(np.array(data[band][bgmodekeys[bgmode]]))))
def bg_mask_annulus(band,ra0,dec0,annulus,ras,decs,responses):
ix = np.where((mc.angularSeparation(ra0,dec0,ras,decs)>=annulus[0]) &
(mc.angularSeparation(ra0,dec0,ras,decs)<=annulus[1]))
return ras[ix],decs[ix],responses[ix]
