def same_same_fdict(self, type):
dir = self.same_same_dir()
fdict={}
fdict['dir'] = dir
origfile = self.wt.filename(type)
fdict['origfile'] = origfile
fdict['nnear1'] = 5
fdict['nnear2'] = 100
trainfile = os.path.basename(origfile).replace('.dat','-train.dat')
validfile = os.path.basename(origfile).replace('.dat','-valid.dat')
fdict['trainfile']=path_join(dir,trainfile)
fdict['photofile']=path_join(dir,validfile)
fdict['wfile1'] = fdict['trainfile'].replace('.dat','-weights-5.dat')
fdict['wfile_nozero1'] = \
fdict['trainfile'].replace('.dat','-weights-nozero-5.dat')
fdict['wfile2'] = fdict['trainfile'].replace('.dat','-weights-100.dat')
fdict['numfile1'] = fdict['photofile'].replace('.dat','-num-5.dat')
fdict['numfile2'] = fdict['photofile'].replace('.dat','-num-100.dat')
fdict['wscript']=path_join(dir,type+'-doweights.sh')
fdict['pofzscript']=path_join(dir,type+'-dopofz.sh')
fdict['zhistfile'] = fdict['photofile'].replace('.dat','-zhist.eps')
return fdict
def read_raw(self):
dir = lensing.files.catalog_dir()
dir = path_join(dir, self['catalog'])
f='%s.fits' % self['catalog']
infile = path_join(dir, f)
stdout.write("Reading raw ra,dec: %s\n" % infile)
data = eu.io.read(infile, lower=True, ensure_native=True)
def __init__(self):
pzdir=zphot.photoz_dir()
self.dir = path_join(pzdir,'training','original-2010-09-19')
self.infile = path_join(self.dir,'primus.zerod.10oct29.fits.gz')
self.outfile = path_join(self.dir,'primus.zerod.10oct29.zconf4.rec')
self.data = eu.io.read(self.infile,lower=True,verbose=True)
eu.numpy_util.to_native(self.data, inplace=True)
def plotdir(self):
d=os.environ['LENSDIR']
d=path_join(d,'regauss-tests',self.procrun)
if self.run != 'any':
d = path_join(d,'%06i' % self.run)
#if self.camcol != 'any':
# d = path_join(d,'bycamcol')
return d
def columns_dir(self):
self.photo_sweep=os.environ['PHOTO_SWEEP']
dirname=os.path.basename(self.photo_sweep)
#dirname += '_new'
dir=os.environ['SWEEP_REDUCE']
dir = path_join(dir,dirname)
if not os.path.exists(dir):
os.makedirs(dir)
dir=path_join(dir ,self.type+'.cols')
return dir
def make_dc5b_data_html():
d=os.environ['DESFILES_DIR']
f = path_join(d,'dc5b','dc5b-runfiles-stat.json')
runstat = eu.io.read(f)
desdir = os.environ['DESDATA']
www_dir=path_join(desdir,'www','dc5b')
fhtml = path_join(www_dir, 'dc5bstat.html')
fobj = open(fhtml, 'w')
fobj.write('<html>\n')
fobj.write('<body>\n')
fobj.write('<head><link rel="stylesheet" href="../table.css" type="text/css"></head>\n')
fobj.write('<h1>DC5b remote files</h1>\n')
fobj.write("""<pre>
The following table summarizes the location of data for DC5b runs.
Only the desar machine is currently visible to BNL machines, so
only desar1,desar2,desardata locations included.
</pre>""")
fobj.write('<table class=simple>\n')
fobj.write(' <tr><th>run</th><th>ftype</th><th>num</th><th>size (TB)</th><th>original<br>location</th><th>BNL<br>Has</th></tr>\n')
# single epoch
for ri in runstat['info']:
run = ri['run']
for ftype in ['red','red_cat']:
num = ri[ftype]['num']
sizetb = ri[ftype]['sizetb']
host = ri[ftype]['host']
have = ri[ftype]['bnlhas']
fobj.write(' <tr><td>')
data = [run, ftype, str(num), str(sizetb), host, have]
data = '</td><td>'.join(data)
fobj.write(data)
fobj.write('</td></tr>\n')
fobj.write('</table>\n')
fobj.write('</html>\n')
fobj.write('</body>\n')
fobj.close()
def plotfile(self, field, rmag_max, plot_type='meane'):
camcol=self.camcol
d=self.plotdir()
f = 'rg-%(run)s%(band)s-%(type)s-vs-%(field)s-%(rmag_max)0.2f'
f = f % {'run':self.procrun,'band':self.band,'type':plot_type,
'field':field,'rmag_max':rmag_max}
if self.run != 'any':
f += '-%06i' % self.run
if camcol != 'any':
d = path_join(d,'bycamcol')
f += '-%i' % camcol
f += '.eps'
f=path_join(d,f)
return f
def fname(self, scinv=False):
d = self.dir()
f='%s-sources.fits' % self['catalog']
if scinv:
f=f.replace('sources','sources-scinv')
infile = path_join(d, f)
return infile
def combine_scinv(self):
from glob import glob
outfile=self.combined_file()
stdout.write("Will write to file: %s\n" % outfile)
dir=self.output_dir()
pattern = self.scinv_file_pattern()
pattern=path_join(dir, pattern)
flist = glob(pattern)
datalist = []
idmin = 0
for f in flist:
print f
tdata = esutil.io.read(f)
data = numpy_util.add_fields(tdata, [('zid','i4')])
data['zid'] = idmin + numpy.arange(data.size)
idmin += data.size
print data['zid'].min(), data['zid'].max()
datalist.append(data)
hdr = esutil.sfile.read_header(flist[0])
print 'combining data'
data = numpy_util.combine_arrlist(datalist)
print 'writing file: %s' % outfile
esutil.sfile.write(data, outfile, header=hdr)
def __init__(self, photo_sample, nchunk=None):
"""
E.g. procrun='prim04'
"""
self.prefix='zinput'
self.photo_sample = photo_sample
self.nchunk = nchunk
self.data = None
self.keep_indices=None
pzdir = zphot.photoz_dir()
self.dir = path_join(pzdir,'inputs',self.photo_sample)
print("photo dir: ",self.dir)
self.conf = zphot.read_config('zinput',photo_sample)
if self.conf['photo_sample'] != photo_sample:
raise ValueError("photo_sample in config is not '%s'" \
% photo_sample)
pprint.pprint(self.conf)
if self.conf['weighting']:
self.conf['filetype'] = 'dat'
source = self.conf.get('source','dr8_final')
if source != 'dr8_final':
raise ValueError("only set up for dr8_final source")
def catalog_file(type):
if type == "public-v1":
name = "maxbcg_public_catalog.fit"
else:
raise ValueError("don't support catalog type '%s' yet" % type)
d = catalog_dir()
f = path_join(d, name)
return f
def config_file(type, id, ext="yaml"):
"""
Want to move over to yaml files
"""
dir = config_dir()
fname = "%s-%s.%s" % (type, id, ext)
fname = path_join(dir, fname)
return fname
def seeing_plotfile(self, name, greyscale=False):
dir = self.plotdir()
s=self.train_sample
if greyscale:
s+='-bw'
fname = '%s-match-seeing-%s.eps' % (name,s)
fname=path_join(dir, fname)
return fname
def plotfile(self, field, rmag_max):
f = 'chrm-meane-ri-vs-%s-rmag%0.2f' % (field, rmag_max)
if self.run != 'any':
f += '-%06i' % self.run
f += '.eps'
d=self.plotdir()
f=path_join(d,f)
return f
def R_polyfile(self, camcol, rmag_max):
f = 'rg-%(run)s%(band)s-meane-vs-R-%(rmag_max)0.2f-%(camcol)s-polyfit.yaml'
f = f % {'run':self.procrun,'band':self.band,
'rmag_max':rmag_max,'camcol':camcol}
d=self.plotdir()
f=path_join(d,'bycamcol',f)
return f
def plotfile(self, type, extra=None):
dir = self.plotdir()
fname = self.fname_matched(type)
fname = os.path.basename(fname)
fname = fname.replace('.rec','.eps')
if extra is not None:
fname = fname.replace('.eps','-'+extra+'.eps')
fname = path_join(dir,fname)
return fname
def input_coldir(name, version):
"""
version should be e.g. dr8-v2
"""
basedir = os.environ["CLUSTERS_INPUT"]
coldir = "%s-input-%s.cols" % (name, version)
coldir = path_join(basedir, coldir)
return coldir
def dir(self):
dir=os.environ.get('REGAUSSIM_DIR',None)
if dir is None:
raise ValueError("REGAUSSIM_DIR must be set")
dir=path_join(dir,'admom-covar-meas-vs-input')
if not os.path.exists(dir):
os.makedirs(dir)
return dir
def init(self, train_sample):
self.types = {'primus':'primus.zerod.10oct29.zconf4',
'2slaq':'2slaq',
'cfrs':'cfrs',
'cnoc2':'cnoc2.cut',
'deep2':'deep2.form.fix',
'sdss':'sdssobjids',
'tkrs':'tkrs-fix',
'vvds':'vvds',
'zcosmos':'zcosmos'}
self.dir_matched = path_join(self.basedir,
'matched',
train_sample)
if self.no_photo_cuts:
self.dir_matched = path_join(self.dir_matched,'no_photo_cuts')
self.conf = zphot.read_config('train',self.train_sample)
self.photo_conf = zphot.read_config('zinput',self.conf['photo_sample'])
pprint.pprint(self.conf)
def columns_dir(self):
if 'SWEEP_REDUCE' not in os.environ:
raise ValueError("SWEEP_REDUCE is not set")
dir = os.environ['SWEEP_REDUCE']
if 'PHOTO_SWEEP' not in os.environ:
raise ValueError("PHOTO_SWEEP is not set")
sweep=os.environ['PHOTO_SWEEP']
sweep = os.path.basename(sweep)
dir = path_join(dir, sweep, self.name()+'.cols')
return dir
def read_exposure_data(dir,exposurename):
out_dtype=[('ccd','i1'),
('x','f4'),('y','f4'),
('psf_flags','i4'),
('e1','f4'),('e2','f4'),
('e1interp','f4'),('e2interp','f4')]
datalist=[]
for ccd in xrange(1,1+62):
psf_file = path_join(dir,'%s_%02d_psf.fits' % (exposurename,ccd))
psf=esutil.io.read(psf_file)
psfe1 = psf['shapelets'][:,3]*sqrt(2)
psfe2 = -psf['shapelets'][:,4]*sqrt(2)
shear_file = path_join(dir,'%s_%02d_shear.fits' % (exposurename,ccd))
shear=esutil.io.read(shear_file)
e1interp = shear['interp_psf_coeffs'][:,3]*sqrt(2)
e2interp = -shear['interp_psf_coeffs'][:,4]*sqrt(2)
mpsf,mshear=numpy_util.match(psf['id'], shear['id'])
tdata = numpy.zeros(mpsf.size, dtype=out_dtype)
tdata['ccd'] = ccd
tdata['x'] = psf['x'][mpsf]
tdata['y'] = psf['y'][mpsf]
tdata['e1'] = psfe1[mpsf]
tdata['e2'] = psfe2[mpsf]
tdata['psf_flags'] = psf['psf_flags'][mpsf]
tdata['e1interp'] = e1interp[mshear]
tdata['e2interp'] = e2interp[mshear]
datalist.append(tdata)
data = numpy_util.combine_arrlist(datalist)
return data
def __init__(self, train_sample=None, no_photo_cuts=False):
self.train_sample = train_sample
self.dir_matched = None
self.no_photo_cuts = no_photo_cuts
pzdir = zphot.photoz_dir()
self.basedir = path_join(pzdir, 'training')
self.prefix = 'train'
self.conf=None
if self.train_sample is not None:
self.init(self.train_sample)
def combine_matches(self):
"""
Combine all the caches into one big file.
"""
dir = self.match_dir()
pattern = path_join(dir,'match-regauss-*-*-*.fits')
files = glob.glob(pattern)
files.sort()
print("Found",len(files),"files")
data = eu.io.read(files, combine=True)
outfile=self.match_file('all')
eu.io.write(outfile,data,verbose=True)
def match_file(self, run):
"""
run might be 'all'
"""
dir=self.match_dir()
if not os.path.exists(dir):
os.makedirs(dir)
if run == 'all':
f='match-regauss-%s-%s.fits' % (self.procrun,self.band)
else:
f='match-regauss-%s-%s-%06d.fits' % (self.procrun,self.band,run)
f = path_join(dir,f)
return f
def __init__(self, origin='uchicago', type='dr7pofz'):
self.good_origins = ['uchicago']
if origin not in self.good_origins:
raise ValueError("origin should be in: %s" % self.good_origins)
self.good_types = ['dr7pofz']
if type not in self.good_types:
raise ValueError("type should be in: %s" % self.good_types)
self.origin = origin
self.type=type
self.basedir = path_join('~esheldon','photoz',origin,type)
self.basedir = os.path.expanduser(self.basedir)
def collate_se_pointing(serun, expname):
"""
Collate the output data from
$DESDATA/wlbnl/$SERUN/pointing
With the input images and catalogs. All I do is look at the .json
files to get the images/catalogs and copy all into a single
directory under
$DESDATA/wlbnl/$SERUN/collated/pointings
"""
import shutil
dir = deswl.files.wlse_collated_dir(serun)
dir = path_join(dir, 'pointings', expname)
if not os.path.exists(dir):
os.mkdir(dir)
# now read all the json files and copy data over
for ccd in range(1,62+1):
statfile=deswl.files.wlse_path(expname, ccd, 'stat', serun=serun)
stdout.write("\nReading stat file: %s\n" % statfile)
stat = esutil.io.read(statfile)
for type in ['imfile','catfile','psf','fitpsf','shear','stars']:
fullpath=stat[type]
if fullpath.find('scratch') != -1:
fullpath = \
fullpath.replace(stat['rootdir'],deswl.files.des_rootdir())
bname=os.path.basename(fullpath)
outfile = path_join(dir, bname)
stdout.write("Copying from %s to %s\n" % (fullpath,outfile))
shutil.copy2(fullpath, outfile)
def all_other_fdict(self,type):
"""
Trying to recover a given training set using
the everything set
"""
dir = self.all_other_dir()
fdict={}
fdict['dir'] = dir
fdict['nnear1'] = 5
fdict['nnear2'] = 100
fdict['trainfile'] = self.wt.filename('all')
# this is the original file, it has the z values
origfile = self.wt.filename(type)
fdict['origfile'] = origfile
# we need to write out origfile in the photofile format
photofile = os.path.basename(origfile).replace('.dat','-asphoto.dat')
photofile = path_join(dir, photofile)
fdict['photofile'] = photofile
tbase = os.path.basename(fdict['trainfile'])
pbase = os.path.basename(fdict['photofile'])
fdict['wfile1'] = \
path_join(dir,tbase.replace('.dat','-%s-weights-5.dat' % type))
fdict['wfile_nozero1'] = \
path_join(dir,tbase.replace('.dat','-%s-weights-nozero-5.dat' % type))
fdict['wfile2'] = \
path_join(dir,tbase.replace('.dat','-%s-weights-100.dat' % type))
fdict['numfile1'] = path_join(dir,pbase.replace('.dat','-num-5.dat'))
fdict['numfile2'] = path_join(dir,pbase.replace('.dat','-num-100.dat'))
fdict['wscript']=path_join(dir,type+'-doweights.sh')
fdict['pofzscript']=path_join(dir,type+'-dopofz.sh')
fdict['zhistfile'] = path_join(dir, pbase.replace('.dat','-zhist.eps') )
return fdict
def get_exposure_wcs_example():
from esutil.ostools import getenv_check, path_join
import cPickle as pickle
expname='decam--24--15-i-6'
d=getenv_check('DESFILES_DIR')
pickle_path=path_join(d, 'wcsexample', 'wcs-'+expname+'.pickle')
if not os.path.exists(pickle_path):
stdout.write("Creating pickled version: %s\n" % pickle_path)
tmpwcs = get_exposure_wcs(expname)
pickle.dump(tmpwcs, open(pickle_path,'w'))
stdout.write("Don't forget to svn add!\n")
stdout.write("Loading wcs example: %s\n" % pickle_path)
wcs = pickle.load(open(pickle_path))
return wcs
def extract_psf_fwhm(self):
"""
This could have been avoided had I added psf_fwhm to the epochs
Not all will match the sweeps, we will have to collate the results
"""
import es_sdsspy
sgdir = self.dir()
extract_file = path_join(sgdir, 'epochs82-psf-fwhm.rec')
cols = self.open_columns('epochs82')
extract_cols = ['run','rerun','camcol','field','id','psf_fwhm']
pid = cols['photoid'][:]
se=es_sdsspy.sweeps.SweepExtractor(pid, allow_nomatch=True)
se.extract(extract_cols, extract_file)
def plotfile(self, run, objmodel, psfmodel, type, alt=None,
s2=None, Rmin=None, dotitle=False, yrange=None):
dir = get_plotdir(run)
f = 'rgsim-%sobj-%spsf-%s' % (objmodel, psfmodel, type)
if alt is not None:
f += '-alt'+alt
if s2 is not None:
f += '-s2-%0.3f' % s2
if Rmin is not None:
f += '-Rmin%0.3f' % Rmin
if dotitle:
f += '-tit'
if yrange is not None:
f += '-yr%0.3f-%0.3f' % tuple(yrange)
f += '.eps'
f = path_join(dir, f)
return f
def rotdir(type='eq'):
d = os.environ['LENSDIR']
d = path_join(d, 'sdss-shape-rot', type)
return d
def rotfile(run, type='eq'):
d = rotdir(type)
f = '%srot-%06i-301.fits' % (type, run)
f = path_join(d, f)
return f
