def extract_runcamcol(self, w, columns):
run = self.run[w[0]]
camcol = self.camcol[w[0]]
gal = sdsspy.read('calibobj.gal', run, camcol, lower=True,verbose=True)
star = sdsspy.read('calibobj.star', run, camcol, lower=True, verbose=True)
gid = sdsspy.photoid(gal)
sid = sdsspy.photoid(star)
data = []
m,mg = eu.numpy_util.match(self.photoid[w], gid)
if m.size > 0:
gal = gal[mg]
gdata = eu.numpy_util.extract_fields(gal, columns, strict=True)
data.append(gdata)
m,ms = eu.numpy_util.match(self.photoid[w], sid)
if m.size > 0:
star = star[ms]
sdata = eu.numpy_util.extract_fields(star, columns, strict=True)
data.append(sdata)
data = eu.numpy_util.combine_arrlist(data)
if data.size != w.size:
mess = "Some objects did not match: %i/%i" % (data.size,w.size)
if not self.allow_nomatch:
raise mess
else:
print(mess)
return data
def extract_fields(self, objs):
"""
Extract tags from the input structure and add some if needed.
"""
dtype=self.keep_dtype()
new = numpy.zeros(objs.size, dtype=dtype)
esutil.numpy_util.copy_fields(objs, new)
# calculate new fields
new['photoid'] = sdsspy.photoid(new)
flux,ivar=new['modelflux'],new['modelflux_ivar']
ext=new['extinction']
flux_dered,ivar_dered = sdsspy.dered_fluxes(ext, flux, ivar=ivar)
mag_dered,err_dered = sdsspy.util.nmgy2mag(flux_dered, ivar=ivar_dered)
new['modelmag_dered'] = mag_dered
new['modelmag_dered_err'] = err_dered
cmag_dered,cmagerr_dered = sdsspy.make_cmodelmag(objs, dered=True)
new['cmodelmag_dered'] = cmag_dered
new['cmodelmag_dered_err'] = cmagerr_dered
return new
def gather_epochs(self, data, epochs_index_current):
photoid = sdsspy.photoid(data)
fh, frev=eu.stat.histogram(data['field'], binsize=1, rev=True)
epochlist = []
epochs_indices = numpy.zeros(data.size)
epochs_count = numpy.zeros(data.size)
epochs_index = epochs_index_current
for i in xrange(fh.size):
if frev[i] != frev[i+1]:
wfield = frev[ frev[i]:frev[i+1] ]
field = data['field'][wfield[0]]
epochs = sdsspy.read('photoEpochs',
data['run'][0],
data['camcol'][0],
field=field,
verbose=False,
lower=True)
# first extract those that were actually used according
# to the criteria for selecting the primaries above
w=where1(epochs['cmodel_used'][:,2] == 1)
if w.size == 0:
raise ValueError("found none that were used!")
epochs = epochs[w]
# extract those that match by photoid
for j in xrange(wfield.size):
pid = photoid[wfield[j]]
w=where1(epochs['primary_photoid'] == pid)
if w.size == 0:
raise ValueError("no matches for pid",pid)
epochs_indices[wfield[j]] = epochs_index
epochs_count[wfield[j]] = w.size
epochs_index += w.size
epochlist.append(epochs[w])
if len(epochlist) == 0:
raise ValueError("Expected to find some epochs!")
epochs = eu.numpy_util.combine_arrlist(epochlist)
if (epochs_index-epochs_index_current) != epochs.size:
raise ValueError("epoch index not incremented right amount:"
"%i vs %i" % (epochs_index-epochs_index_current,epochs.size))
return epochs, epochs_indices, epochs_count
def create_output(self, st):
bands = ['u','g','r','i','z']
out_dict = {}
for d in st.dtype.descr:
name = str( d[0] )
if len(d) == 3:
if d[2] == 5:
# ignoring the higher dim stuff
for bandi in xrange(5):
fname = name+'_'+bands[bandi]
out_dict[fname] = st[name][:, bandi]
else:
out_dict[name] = st[name]
if 'devflux' in st.dtype.names:
#cmodelflux, cmodelflux_ivar = sdsspy.make_cmodelflux(st, doivar=True)
cmodelmag_dered, cmodelmag_dered_err = sdsspy.make_cmodelmag(st, dered=True)
modelflux, modelflux_ivar = sdsspy.dered_fluxes(st['extinction'],
st['modelflux'],
st['modelflux_ivar'])
modelmag_dered, modelmag_dered_err = sdsspy.nmgy2mag(modelflux, modelflux_ivar)
for f in sdsspy.FILTERCHARS:
fnum = sdsspy.FILTERNUM[f]
if 'devflux' in st.dtype.names:
out_dict['cmodelflux_'+f] = cmodelmag_dered[:,fnum]
out_dict['cmodelflux_ivar_'+f] = cmodelmag_dered_err[:,fnum]
out_dict['cmodelmag_dered_'+f] = cmodelmag_dered[:,fnum]
out_dict['cmodelmag_dered_err_'+f] = cmodelmag_dered_err[:,fnum]
out_dict['modelmag_dered_'+f] = modelmag_dered[:,fnum]
out_dict['modelmag_dered_err_'+f] = modelmag_dered_err[:,fnum]
out_dict['photoid'] = sdsspy.photoid(st)
self.set_maskflags(out_dict)
# we will add an "survey_primary" column if we are not explicitly
# selecting survey_primary
if self.type not in ['primgal','primstar']:
survey_primary = numpy.zeros(st.size, dtype='i1')
w=self.get_primary_indices(st)
if w.size > 0:
survey_primary[w] = 1
out_dict['survey_primary'] = survey_primary
return out_dict
def make_epochs_output(self, data):
dtype = self.epochs_dtype()
out = numpy.zeros(data.size, dtype=dtype)
out['primary_photoid'] = data['primary_photoid']
out['photoid'] = sdsspy.photoid(data)
out['objc_type'] = data['objc_type']
for ftype in ['psf','cmodel']:
for filter in ['g','r','i']:
fnum=sdsspy.FILTERNUM[filter]
out[ftype+'flux_'+filter] = data[ftype+'flux'][:,fnum]
out[ftype+'flux_ivar_'+filter] = data[ftype+'flux_ivar'][:,fnum]
out[ftype+'_used_'+filter] = data[ftype+'_used'][:,fnum]
return out
def collate_psf_fwhm(self):
"""
Collate the results from matching epochs to sweeps to get psf_fwhm.
"""
print("opening columns")
cols = self.open_columns('epochs82')
print("getting photoid")
pid = cols['photoid'][:]
sgdir = self.dir()
extract_file = path_join(sgdir, 'epochs82-psf-fwhm.rec')
print("reading psf_fwhm from:",extract_file)
fwhm_data = eu.io.read(extract_file)
print("matching")
fwhm_pid = sdsspy.photoid(fwhm_data)
m,ms = eu.numpy_util.match(pid, fwhm_pid)
print(" matched %i/%i" % (m.size,pid.size))
fwhm = numpy.zeros(pid.size, dtype='f4')
print("writing psf_fwhm_g")
fwhm[:] = -9999.
fwhm[m] = fwhm_data['psf_fwhm'][ms,1]
cols.write_column('psf_fwhm_g', fwhm, create=True)
print("writing psf_fwhm_r")
fwhm[:] = -9999.
fwhm[m] = fwhm_data['psf_fwhm'][ms,2]
cols.write_column('psf_fwhm_r', fwhm, create=True)
print("writing psf_fwhm_i")
fwhm[:] = -9999.
fwhm[m] = fwhm_data['psf_fwhm'][ms,3]
cols.write_column('psf_fwhm_i', fwhm, create=True)
def get_subset_and_derived(self,data):
"""
Add columns such as photoid, htmid, mask flags and dereddened model
mags
"""
# create the output with the tags we want
new = numpy.zeros(data.size, dtype=self.get_dtype())
esutil.numpy_util.copy_fields(data, new)
new['photoid'] = sdsspy.photoid(data)
primflag = sdsspy.flags.flagval('resolve_status','survey_primary')
w,=where((new['resolve_status'] & primflag) != 0)
if w.size > 0:
new['isprimary'][w] = 1
flux,ivar = self.dered_fluxes(data['modelflux'],
data['modelflux_ivar'],
data['extinction'])
mag,err = sdsspy.util.nmgy2mag(flux, ivar=ivar)
new['modelmag_dered'] = mag
new['modelmag_dered_err'] = err
self.load_stomp_maps()
self.load_htm()
new['htmid10'] = SweepStuffer._htm.lookup_id(data['ra'], data['dec'])
new['basic_maskflags'] = SweepStuffer._basic_map.Contains(data['ra'],data['dec'],'eq')
new['tycho_maskflags'] = SweepStuffer._tycho_map.Contains(data['ra'],data['dec'],'eq')
return new
def make_primary_output(self, data):
dtype = self.prim_dtype()
out = numpy.zeros(data.size, dtype=dtype)
out['photoid'] = sdsspy.photoid(data)
out['objc_type'] = data['objc_type']
for ftype in ['psf','cmodel']:
for filter in ['g','r','i']:
fnum=sdsspy.FILTERNUM[filter]
if ftype == 'cmodel':
# we couldn't get cmodel because stars don't have dev,exp
out['modelflux_'+filter] = data['modelflux'][:,fnum]
out['modelflux_ivar_'+filter] = data['modelflux_ivar'][:,fnum]
else:
out[ftype+'flux_'+filter] = data[ftype+'flux'][:,fnum]
out[ftype+'flux_ivar_'+filter] = data[ftype+'flux_ivar'][:,fnum]
out[ftype+'_nuse_'+filter] = data[ftype+'_nuse'][:,fnum]
out[ftype+'flux_mean_'+filter] = data[ftype+'flux_mean'][:,fnum]
out[ftype+'flux_mean_ivar_'+filter] = data[ftype+'flux_mean_ivar'][:,fnum]
return out
def match_sweeps(self, data):
gal=sdsspy.read('calibobj.gal',data['run'][0], data['camcol'][0],
lower=True)
star=sdsspy.read('calibobj.star',data['run'][0], data['camcol'][0],
lower=True)
photoid = sdsspy.photoid(data)
gphotoid = sdsspy.photoid(gal)
sphotoid = sdsspy.photoid(star)
mdata_g, mg = eu.numpy_util.match(photoid, gphotoid)
print(" gal matches:",mg.size)
mdata_s, ms = eu.numpy_util.match(photoid, sphotoid)
print(" star matches:",ms.size)
nmatch = mdata_g.size + mdata_s.size
if nmatch == 0:
return numpy.array([])
newdt = [('photoid','i8'),
('objc_flags','i4'),
('flags','i4',5),
('flags2','i4',5),
('calib_status','i4',5),
('modelflux','f4',5),
('modelflux_ivar','f4',5),
('psfflux','f4',5),
('psfflux_ivar','f4',5)]
dtype = data.dtype.descr + newdt
out = numpy.zeros(nmatch, dtype=dtype)
#data_keep = numpy.zeros(nmatch, dtype=data.dtype)
if mg.size > 0:
gal = gal[mg]
out['photoid'][0:mg.size] = gphotoid[mg]
out['flags'][0:mg.size] = gal['flags']
out['flags2'][0:mg.size] = gal['flags2']
out['objc_flags'][0:mg.size] = gal['objc_flags']
out['calib_status'][0:mg.size] = gal['calib_status']
out['modelflux'][0:mg.size] = gal['modelflux']
out['modelflux_ivar'][0:mg.size] = gal['modelflux_ivar']
out['psfflux'][0:mg.size] = gal['psfflux']
out['psfflux_ivar'][0:mg.size] = gal['psfflux_ivar']
for n in data.dtype.names:
out[n][0:mg.size] = data[n][mdata_g]
if ms.size > 0:
star=star[ms]
out['photoid'][mg.size:] = sphotoid[ms]
out['flags'][mg.size:] = star['flags']
out['flags2'][mg.size:] = star['flags2']
out['objc_flags'][mg.size:] = star['objc_flags']
out['calib_status'][mg.size:] = star['calib_status']
out['modelflux'][mg.size:] = star['modelflux']
out['modelflux_ivar'][mg.size:] = star['modelflux_ivar']
out['psfflux'][mg.size:] = star['psfflux']
out['psfflux_ivar'][mg.size:] = star['psfflux_ivar']
for n in data.dtype.names:
out[n][mg.size:] = data[n][mdata_s]
return out
def make_columns(self):
"""
Make a columns database from all available data
"""
from glob import glob
import columns
import sdsspy
coldir = self.columns_dir()
stdout.write("Will create columns: %s\n" % coldir)
if os.path.exists(coldir):
raise ValueError("coldir exists, please start from scratch")
c=columns.Columns(coldir, verbose=False)
dir=self.output_dir()
pattern = self.scinv_file_pattern()
pattern=path_join(dir, pattern)
flist = glob(pattern)
# z values in p(z)
zvals = list( self.dr7pofz_zvals() )
pofz_meta = {'zvals': zvals}
i=1
ntot=len(flist)
for scinv_file in sorted(flist):
stdout.write('-'*70+'\n')
stdout.write('%s/%s\n' % (i,ntot))
raw_file = scinv_file.replace('-scinv.rec','.dat')
# read both scinv and raw, should line up
stdout.write("Reading scinv data from: %s\n" % scinv_file)
scinv, scinv_meta = esutil.sfile.read(scinv_file, header=True)
scinv_meta = {'zlvals': scinv_meta['zlvals']}
raw = self.read_raw(raw_file)
if scinv.size != raw.size:
raise ValueError("scinv and raw file are different sizes")
photoid = sdsspy.photoid(raw)
# write data from the raw file
c.write_column('photoid',photoid, type='rec')
c.write_column('casid',raw['casid'], type='rec')
c.write_column('run',raw['run'], type='rec')
c.write_column('rerun',raw['rerun'], type='rec')
c.write_column('camcol',raw['camcol'], type='rec')
c.write_column('field',raw['field'], type='rec')
c.write_column('id',raw['id'], type='rec')
c.write_column('ra',raw['ra'], type='rec')
c.write_column('dec',raw['dec'], type='rec')
c.write_column('pofz',raw['pz'], type='rec',
meta=pofz_meta)
# now the scinv data
c.write_column('mean_scinv',scinv['mean_scinv'], type='rec',
meta=scinv_meta)
i+=1
def create_output(self, st):
import sdsspy
bands = ['u','g','r','i','z']
out_dict = {}
for d in st.dtype.descr:
name = str( d[0] )
if len(d) == 3:
if isinstance(d[2],tuple):
sz=d[2][0]
else:
sz=d[2]
if sz == 5:
for bandi in xrange(5):
fname = name+'_'+bands[bandi]
out_dict[fname] = st[name][:, bandi]
else:
out_dict[name] = st[name]
# match up to sweeps and get some info
print(" matching to primary sweeps")
w=self.sweep_cols['thing_id'].match(st['thing_id'])
#print(" found %s/%s" % (w.size,st.size))
if w.size != st.size:
raise ValueError("Did not match all")
out_dict['primgal_id'] = w
print(" Getting psf ellip")
Irr_psf = st['Irr_psf']
Irc_psf = st['Irc_psf']
Icc_psf = st['Icc_psf']
T_psf = st['Irr_psf'] + st['Icc_psf']
if 'e1' not in st.dtype.names:
e1,e2 = self.make_e1e2(st)
print(" Copying ellip,mags")
for f in sdsspy.FILTERCHARS:
fnum = sdsspy.FILTERNUM[f]
out_dict['e1_psf_'+f] = (Icc_psf[:,fnum]-Irr_psf[:,fnum])/T_psf[:,fnum]
out_dict['e2_psf_'+f] = 2*Irc_psf[:,fnum]/T_psf[:,fnum]
out_dict['modelmag_dered_'+f] = self.sweep_cols['modelmag_dered_'+f][w]
ext = self.sweep_cols['extinction_'+f][w]
out_dict['extinction_'+f] = ext
cmodelname = 'cmodelmag_dered_'+f
if cmodelname in self.sweep_cols:
out_dict[cmodelname] = self.sweep_cols[cmodelname][w]
if self.sweeptype == 'star':
flux = self.sweep_cols['psfflux_'+f][w]
ivar = self.sweep_cols['psfflux_ivar_'+f][w]
flux_dered = sdsspy.dered_fluxes(ext, flux)
mag_dered = sdsspy.nmgy2mag(flux_dered)
out_dict['psfflux_'+f] = flux
out_dict['psfflux_ivar_'+f] = ivar
out_dict['psfmag_dered_'+f] = mag_dered
if 'e1' not in st.dtype.names:
out_dict['e1_'+f] = e1[:,fnum]
out_dict['e2_'+f] = e2[:,fnum]
out_dict['photoid'] = sdsspy.photoid(st)
return out_dict