def read_atlas(**keys):
import sdsspy
imd=sdsspy.read('fpAtlas', **keys)
calib=sdsspy.read('calibPhotom', lower=True, **keys)
if calib is None:
print 'calibPhotom not found: using default rouch calib'
nmgypercount=_nmgypercount
else:
w,=where(calib['field']==keys['field'])
if w.size==0:
raise ValueError("calib doesn't have field %d" % keys['field'])
nmgypercount=calib['nmgypercount'][w[0],:]
if not imd:
raise RuntimeError("no images returned")
sky=imd['SOFT_BIAS']
for i in xrange(5):
im = imd['images'][i].astype('f4')-sky
im *= nmgypercount[i]*_scales[i]
imd['images'][i] = im
return imd
def extract_runcamcol(self, w, columns):
run = self.run[w[0]]
camcol = self.camcol[w[0]]
gal = sdsspy.read('calibobj.gal', run, camcol, lower=True,verbose=True)
star = sdsspy.read('calibobj.star', run, camcol, lower=True, verbose=True)
gid = sdsspy.photoid(gal)
sid = sdsspy.photoid(star)
data = []
m,mg = eu.numpy_util.match(self.photoid[w], gid)
if m.size > 0:
gal = gal[mg]
gdata = eu.numpy_util.extract_fields(gal, columns, strict=True)
data.append(gdata)
m,ms = eu.numpy_util.match(self.photoid[w], sid)
if m.size > 0:
star = star[ms]
sdata = eu.numpy_util.extract_fields(star, columns, strict=True)
data.append(sdata)
data = eu.numpy_util.combine_arrlist(data)
if data.size != w.size:
mess = "Some objects did not match: %i/%i" % (data.size,w.size)
if not self.allow_nomatch:
raise mess
else:
print(mess)
return data
def _read_psfield(self, field):
psfield = sdsspy.read("psfield", run=self["run"], camcol=self["camcol"], field=field)
psf_kl = []
for filter in sdsspy.FILTERCHARS:
kl = sdsspy.read("psField", run=self["run"], camcol=self["camcol"], field=field, filter=filter)
psf_kl.append(kl)
return psfield, psf_kl
def test_em_sdss(ngauss=2,
run=756,
field=45,
camcol=1,
filter='r',
row=123.1,
col=724.8,
cocenter=False,
show=False):
import sdsspy
fnum=sdsspy.FILTERNUM[filter]
psfield=sdsspy.read('psField', run=run, camcol=camcol, field=field,
lower=True)
psfkl=sdsspy.read('psField', run=run, camcol=camcol, field=field,
filter=filter)
if psfkl is None:
print 'no such field'
return
im_nonoise=psfkl.rec(row, col, trim=True)
im0,skysig=add_noise_matched(im_nonoise, 1.e8)
ivar=1./skysig**2
dims=im0.shape
cen=[(dims[0]-1.)/2., (dims[1]-1.)/2.]
fwhm=psfield['psf_width'][0,fnum]
sigma_guess=fimage.fwhm2sigma(fwhm)
sigma_guess /= 0.4 # pixels
print 'guess fwhm:',fwhm
gm=gmix_image.gmix_em.GMixEMBoot(im0, ngauss, cen,
sigma_guess=sigma_guess,
ivar=ivar,
cocenter=cocenter)
res=gm.get_result()
gmx=gm.get_gmix()
print 'numiter:',res['numiter']
if show and have_images:
import biggles
biggles.configure('screen','width',1000)
biggles.configure('screen','height',1000)
mod=gmix2image(gmx,im0.shape)
counts=im0.sum()
mod *= counts/mod.sum()
if cocenter:
title='cocenter ngauss: %d' % ngauss
else:
title='free centers: %d' % ngauss
images.compare_images(im0, mod, label1='image',label2='model',
title=title)
print gmx
return res
def cache(self, run, camcol, field, verbose=True):
key = '%06i-%i-%04i' % (run,camcol,field)
if key != self.psf_key:
self.psfield = sdsspy.read('psfield',run,camcol,field, verbose=verbose)
self.psf_kls = []
for filter in sdsspy.FILTERCHARS:
kl = sdsspy.read('psField',run,camcol,field,filter=filter, verbose=verbose)
self.psf_kls.append(kl)
self.psf_key = key
def make_frame_jpg(**keys):
from PIL import Image
from numpy import array, flipud
import sdsspy
import esutil as eu
import images
#import biggles
g,gh=sdsspy.read('frame', filter='g', header=True, **keys)
r,rh=sdsspy.read('frame', filter='r', header=True, **keys)
i,ih=sdsspy.read('frame', filter='i', header=True, **keys)
gwcs=eu.wcsutil.WCS(gh)
rwcs=eu.wcsutil.WCS(rh)
iwcs=eu.wcsutil.WCS(ih)
# convert r-band center to ra,dec
rowcen_r=r.shape[0]/2.
colcen_r=r.shape[1]/2.
racen_r,deccen_r = rwcs.image2sky(colcen_r, rowcen_r)
# now see where that falls in the g and i images
gcol,grow=gwcs.sky2image(racen_r, deccen_r)
icol,irow=iwcs.sky2image(racen_r, deccen_r)
gshift=[rowcen_r-grow, colcen_r-gcol]
ishift=[rowcen_r-irow, colcen_r-icol]
g = shift_image(g, gshift)
i = shift_image(i, ishift)
rescale=0.4
scales= rescale*array([5., 6.5, 9.])
nonlinear=0.15
colorim=images.get_color_image(i, r, g,
scales=scales,
nonlinear=nonlinear,
satval=30.)
fname=get_jpg_path(**keys)
print 'writing:',fname
dname=os.path.dirname(fname)
if not os.path.exists(dname):
print 'making dirs:',dname
os.makedirs(dname)
#colorim = (colorim*255).astype('u1')
colorim = images.bytescale(colorim)
colorim=flipud(colorim)
pim=Image.fromarray(colorim)
pim.save(fname, quality=90)
def test_sweep_selector(run=756, camcol=3, type='gal'):
sweep=sdsspy.read('calibobj.'+type, run=run, camcol=camcol, lower=True)
fields=numpy.unique(sweep['field'])
for field in fields:
pobj=sdsspy.read('photoObj', run=run, camcol=camcol, field=field,
lower=True)
sel=SweepSelector(pobj, type)
ind=sel.get_indices()
w,=numpy.where(sweep['field'] == field)
ndiff=w.size-ind.size
print('field:',field,'sweep count:',w.size,'photoObj trimmed count:',ind.size,'diff:',ndiff)
def get_family(**keys):
"""
Given the input ids, find the family.
Returns a Family object. Reads the photoobj files to get
the data and find the family.
You must enter as keywords objid or photoid or run,rerun,camcol,field,id
"""
from .util import get_id_info, get_photoid
data=sdsspy.read('photoObj',lower=True, **keys)
if data is None:
raise ValueError("failed to read any photoObj files for this id")
ids=get_id_info(**keys)
pids=get_photoid(data)
index,=where(pids == ids['photoid'])
if index.size == 0:
raise ValueError("result doesn't contain object?")
fam=Family(data, index[0])
return fam
def get_family(**keys):
"""
Given the input ids, find the family.
Returns a Family object. Reads the photoobj files to get
the data and find the family.
You must enter as keywords objid or photoid or run,rerun,camcol,field,id
"""
from .util import get_id_info, get_photoid
data = sdsspy.read('photoObj', lower=True, **keys)
if data is None:
raise ValueError("failed to read any photoObj files for this id")
ids = get_id_info(**keys)
pids = get_photoid(data)
index, = where(pids == ids['photoid'])
if index.size == 0:
raise ValueError("result doesn't contain object?")
fam = Family(data, index[0])
return fam
def load_kl(self, i):
import sdsspy
self.kl = sdsspy.read('psField',
self.flist['run'][i],
self.flist['camcol'][i],
self.flist['field'][i],
filter=self['filter'],
verbose=self.verbose)
def load_meta(self, i):
import sdsspy
meta = sdsspy.read('psField',
self.flist['run'][i],
self.flist['camcol'][i],
self.flist['field'][i],
lower=True,
verbose=self.verbose)
self.meta = meta[0]
def cache_column(self, type,run,camcol, **keys):
import sdsspy
key = '%s-%06i-%d' % (type,run,camcol)
if self.key != key:
data = sdsspy.read('calibobj.%s' % type,
run,camcol, lower=True, verbose=self.verbose,
**keys)
self.key = key
self.data = data
def run(self, runs=None):
"""
Collate the collated files from the run into a columns directory.
Multi-band columns are split to, e.g., flux_r, flux_i, ...
"""
c = self.open_columns()
print("Will write in coldir:", c.dir)
if c.dir_exists():
raise ValueError("coldir already exist, start from scratch")
c.create()
# create the meta column from input data
meta = self.make_meta()
c.write_column('meta', meta)
if runs is None:
win = sdsspy.window.Window()
runs, reruns = win.runlist(self.minscore)
nrun=len(runs)
print("Processing",nrun,"runs\n")
for i in xrange(nrun):
run=runs[i]
print("\nProcessing run: %s %s/%s" % (run,i+1,nrun))
print("-"*70)
for camcol in [1,2,3,4,5,6]:
print(" %06i-%i" % (run,camcol))
tmp = sdsspy.read('calibobj.'+self.sweep_type, run, camcol,
lower=True, ensure_native=True)
if len(tmp) == 0:
raise ValueError("sweep is empty")
print(" Found",tmp.size)
if self.type in ['primgal','primstar']:
print(" Selecting survey_primary")
w=self.get_primary_indices(tmp)
else:
print(" Selecting run_primary")
w=self.get_primary_indices(tmp, run_primary=True)
print(" Found",w.size)
if w.size > 0:
tmp = tmp[w]
print(" Getting output")
out_dict = self.create_output(tmp)
print(" Writing columns")
for name in out_dict:
c.write_column(name, out_dict[name])
print("Done")
def cache_psf(self, run, camcol, field):
"""
This will cache reads so it can be used for multiple filter
processings. So process all objects from a field in order to be most
efficient.
"""
import sdsspy
key = '%06i-%i-%04i' % (run,camcol,field)
if self.field_key != key:
self.psfield = sdsspy.read('psfield',run,camcol,field,
verbose=self.verbose)
self.psfKLs = []
for filter in sdsspy.FILTERCHARS:
kl = sdsspy.read('psField',run,camcol,field,filter=filter,
verbose=self.verbose)
self.psfKLs.append(kl)
self.field_key = key
def cache_atlas(self, run, camcol, field, id):
"""
This will cache reads so it can be used for multiple
filter processings
"""
import sdsspy
key = '%06i-%i-%04i-%05i' % (run,camcol,field,id)
if self.atls_key != key:
self.atls = sdsspy.read('fpAtlas',run,camcol,field,id,
trim=True,
verbose=self.verbose)
self.atls_key = key
def get_test_data():
from numpy import log10
import sdsspy
from . import select
t=sdsspy.read('calibobj.gal', 756, 3, 300, rerun='301', lower=True)
sel=select.Selector(t)
mag=22.5-2.5*log10( t['modelflux'][:,2].clip(0.001, t['modelflux'][:,2].max()))
fl=sel.flag_logic()
w,=where(fl & (mag < 15) & (mag > 14))
return t,w
def gather_epochs(self, data, epochs_index_current):
photoid = sdsspy.photoid(data)
fh, frev=eu.stat.histogram(data['field'], binsize=1, rev=True)
epochlist = []
epochs_indices = numpy.zeros(data.size)
epochs_count = numpy.zeros(data.size)
epochs_index = epochs_index_current
for i in xrange(fh.size):
if frev[i] != frev[i+1]:
wfield = frev[ frev[i]:frev[i+1] ]
field = data['field'][wfield[0]]
epochs = sdsspy.read('photoEpochs',
data['run'][0],
data['camcol'][0],
field=field,
verbose=False,
lower=True)
# first extract those that were actually used according
# to the criteria for selecting the primaries above
w=where1(epochs['cmodel_used'][:,2] == 1)
if w.size == 0:
raise ValueError("found none that were used!")
epochs = epochs[w]
# extract those that match by photoid
for j in xrange(wfield.size):
pid = photoid[wfield[j]]
w=where1(epochs['primary_photoid'] == pid)
if w.size == 0:
raise ValueError("no matches for pid",pid)
epochs_indices[wfield[j]] = epochs_index
epochs_count[wfield[j]] = w.size
epochs_index += w.size
epochlist.append(epochs[w])
if len(epochlist) == 0:
raise ValueError("Expected to find some epochs!")
epochs = eu.numpy_util.combine_arrlist(epochlist)
if (epochs_index-epochs_index_current) != epochs.size:
raise ValueError("epoch index not incremented right amount:"
"%i vs %i" % (epochs_index-epochs_index_current,epochs.size))
return epochs, epochs_indices, epochs_count
def _read_atlas(self, obj):
"""
Read atlas images as sky-subtracted, 'f8'
"""
from sdsspy.atlas import NoAtlasImageError
field = obj["field"]
id = obj["id"]
try:
atlas = sdsspy.read("fpAtlas", run=self["run"], camcol=self["camcol"], field=field, id=id)
except NoAtlasImageError:
print "object has no atlas image"
atlas = None
return atlas
def read_family(**keys):
import sdsspy
# try to read the data
cat=sdsspy.read('photoObj', lower=True, **keys)
if cat is None:
raise ValueError("Could not read photoObj for your object")
idinfo=get_id_info(**keys)
index,=where(cat['id'] == idinfo['id'])
if index.size==0:
raise ValueError("problem finding id in catalog")
index=index[0]
fam=sdsspy.util.Family(cat, index)
return fam,cat,index
def _load_data(self):
self.objs = None
objs0 = sdsspy.read(
"photoObj", run=self["run"], camcol=self["camcol"], field=self["field"], lower=True, verbose=True
)
if objs0 is None:
print "no data read"
return
w, mags = self._select(objs0)
if w.size == 0:
print "no objects passed cuts"
return
objs0 = objs0[w]
objs = eu.numpy_util.add_fields(objs0, [("rmag", "f8")])
objs["rmag"][:] = mags
self.objs = objs
self.psfield, self.psf_kl = self._read_psfield(self["field"])
def admom_atlas(type, run, camcol, field, id, filter,
objs=None,
show=False, showmin=0, **keys):
"""
Objs must be for this camcol!
"""
import admom
import sdsspy
c = sdsspy.FILTERNUM[filter]
if objs is None:
sc=SweepCache()
obj = sc.readobj(type, run, camcol, field, id)
else:
w=where1( (objs['field'] == field) & (objs['id'] == id) )
if w.size == 0:
raise ValueError("Object not found objs: "
"%06i-%i-%04i" % (run,camcol,id))
obj = objs[w[0]]
atls = sdsspy.read('fpAtlas',run,camcol,field,id, trim=True)
im = numpy.array( atls['images'][c], dtype='f8') - atls['SOFT_BIAS']
psfield = sdsspy.read('psField', run, camcol, field, lower=True)
rowc=obj['rowc'][c]
colc=obj['colc'][c]
row=rowc - atls['row0'][c] - 0.5
col=colc - atls['col0'][c] - 0.5
sigsky = psfield['skysig'][0,c]
Tguess_psf=admom.fwhm2mom( obj['psf_fwhm'][c], pixscale=0.4)
if obj['m_rr_cc'][c] > Tguess_psf:
Tguess=obj['m_rr_cc'][c]
else:
Tguess=Tguess_psf
out = admom.admom(im,
row,
col,
sky=0.0,
sigsky=sigsky,
Tguess=Tguess)
if show:
import fimage
import biggles
plt=images.view(im, min=showmin, show=False, **keys)
if out['whyflag'] == 0:
levels=7
wrow = out['wrow']
wcol = out['wcol']
model = fimage.model_image('gauss',
im.shape,
[wrow,wcol],
[out['Irr'],out['Irc'],out['Icc']],
counts=im.sum())
cmod = biggles.Contours(model.transpose(), color='grey')
cmod.levels = levels
plt.add(cmod)
plt.show()
return out
def collate(self):
"""
Keep objects if have nrunmin matches in both model and psf flux, in
*any* one of g,r,i
"""
runs = stripe82_run_list()
#runs=runs[where1(runs == 259)]
pcoldir = self.coldir('primary82')
ecoldir = self.coldir('epochs82')
if os.path.exists(pcoldir):
raise ValueError("primary coldir exists: '%s'\n Start Fresh" % pcoldir)
if os.path.exists(ecoldir):
raise ValueError("epochs coldir exists: '%s'\n Start Fresh" % ecoldir)
pcols = columns.Columns(pcoldir)
pcols.create()
ecols = columns.Columns(ecoldir)
ecols.create()
# count in the epochs columns
epochs_index = 0
for run in runs:
for camcol in [1,2,3,4,5,6]:
fname=sdsspy.filename('photoCombineCamcol', run, camcol)
if os.path.exists(fname):
data=sdsspy.read('photoCombineCamcol', run, camcol,
verbose=True,lower=True)
# first select by number of runs matched, then do flag
# selection and such
print(" selecting r-band nrunmin >",self.nrunmin)
model_nuse = data['cmodel_nuse'][:,2]
psf_nuse = data['psf_nuse'][:,2]
w=where1( (psf_nuse >= self.nrunmin) & (model_nuse >= self.nrunmin) )
print(" kept:",w.size)
if w.size > 0:
data = data[w]
print(" getting sweep matches for flags")
data = self.match_sweeps(data)
print(" found",data.size,"sweep matches")
if data.size > 0:
print(" selecting flags and rflux >",self.rlim)
w = where1( get_select_logic(data, self.rlim) )
print(" kept:",w.size)
if w.size > 0:
data = data[w]
print(" making primary output")
poutput = self.make_primary_output(data)
print(" gathering epochs")
epochs, epochs_indices, epochs_count = \
self.gather_epochs(data, epochs_index)
poutput['epochs_index'] = epochs_indices
poutput['epochs_count'] = epochs_count
nexpected = self.calc_expected_nepoch(data)
if epochs.size != nexpected:
#if nexpected-epochs.size == 1:
#print("FOUND ERROR nexpected greater than epochs by 1. primary bug?")
#else:
# raise ValueError("Expected",nexpected,"epochs, but found",epochs.size)
print("ERROR: Expected",nexpected,"epochs, but found",epochs.size)
print(" found:",epochs.size)
print(" making epochs output")
eoutput = self.make_epochs_output(epochs)
pcols.write_columns(poutput)
ecols.write_columns(eoutput)
# keep track of index in epochs database
epochs_index += epochs.size
def match_sweeps(self, data):
gal=sdsspy.read('calibobj.gal',data['run'][0], data['camcol'][0],
lower=True)
star=sdsspy.read('calibobj.star',data['run'][0], data['camcol'][0],
lower=True)
photoid = sdsspy.photoid(data)
gphotoid = sdsspy.photoid(gal)
sphotoid = sdsspy.photoid(star)
mdata_g, mg = eu.numpy_util.match(photoid, gphotoid)
print(" gal matches:",mg.size)
mdata_s, ms = eu.numpy_util.match(photoid, sphotoid)
print(" star matches:",ms.size)
nmatch = mdata_g.size + mdata_s.size
if nmatch == 0:
return numpy.array([])
newdt = [('photoid','i8'),
('objc_flags','i4'),
('flags','i4',5),
('flags2','i4',5),
('calib_status','i4',5),
('modelflux','f4',5),
('modelflux_ivar','f4',5),
('psfflux','f4',5),
('psfflux_ivar','f4',5)]
dtype = data.dtype.descr + newdt
out = numpy.zeros(nmatch, dtype=dtype)
#data_keep = numpy.zeros(nmatch, dtype=data.dtype)
if mg.size > 0:
gal = gal[mg]
out['photoid'][0:mg.size] = gphotoid[mg]
out['flags'][0:mg.size] = gal['flags']
out['flags2'][0:mg.size] = gal['flags2']
out['objc_flags'][0:mg.size] = gal['objc_flags']
out['calib_status'][0:mg.size] = gal['calib_status']
out['modelflux'][0:mg.size] = gal['modelflux']
out['modelflux_ivar'][0:mg.size] = gal['modelflux_ivar']
out['psfflux'][0:mg.size] = gal['psfflux']
out['psfflux_ivar'][0:mg.size] = gal['psfflux_ivar']
for n in data.dtype.names:
out[n][0:mg.size] = data[n][mdata_g]
if ms.size > 0:
star=star[ms]
out['photoid'][mg.size:] = sphotoid[ms]
out['flags'][mg.size:] = star['flags']
out['flags2'][mg.size:] = star['flags2']
out['objc_flags'][mg.size:] = star['objc_flags']
out['calib_status'][mg.size:] = star['calib_status']
out['modelflux'][mg.size:] = star['modelflux']
out['modelflux_ivar'][mg.size:] = star['modelflux_ivar']
out['psfflux'][mg.size:] = star['psfflux']
out['psfflux_ivar'][mg.size:] = star['psfflux_ivar']
for n in data.dtype.names:
out[n][mg.size:] = data[n][mdata_s]
return out