def same_same_fdict(self, type): dir = self.same_same_dir() fdict={} fdict['dir'] = dir origfile = self.wt.filename(type) fdict['origfile'] = origfile fdict['nnear1'] = 5 fdict['nnear2'] = 100 trainfile = os.path.basename(origfile).replace('.dat','-train.dat') validfile = os.path.basename(origfile).replace('.dat','-valid.dat') fdict['trainfile']=path_join(dir,trainfile) fdict['photofile']=path_join(dir,validfile) fdict['wfile1'] = fdict['trainfile'].replace('.dat','-weights-5.dat') fdict['wfile_nozero1'] = \ fdict['trainfile'].replace('.dat','-weights-nozero-5.dat') fdict['wfile2'] = fdict['trainfile'].replace('.dat','-weights-100.dat') fdict['numfile1'] = fdict['photofile'].replace('.dat','-num-5.dat') fdict['numfile2'] = fdict['photofile'].replace('.dat','-num-100.dat') fdict['wscript']=path_join(dir,type+'-doweights.sh') fdict['pofzscript']=path_join(dir,type+'-dopofz.sh') fdict['zhistfile'] = fdict['photofile'].replace('.dat','-zhist.eps') return fdict
def read_raw(self): dir = lensing.files.catalog_dir() dir = path_join(dir, self['catalog']) f='%s.fits' % self['catalog'] infile = path_join(dir, f) stdout.write("Reading raw ra,dec: %s\n" % infile) data = eu.io.read(infile, lower=True, ensure_native=True)
def __init__(self): pzdir=zphot.photoz_dir() self.dir = path_join(pzdir,'training','original-2010-09-19') self.infile = path_join(self.dir,'primus.zerod.10oct29.fits.gz') self.outfile = path_join(self.dir,'primus.zerod.10oct29.zconf4.rec') self.data = eu.io.read(self.infile,lower=True,verbose=True) eu.numpy_util.to_native(self.data, inplace=True)
def plotdir(self): d=os.environ['LENSDIR'] d=path_join(d,'regauss-tests',self.procrun) if self.run != 'any': d = path_join(d,'%06i' % self.run) #if self.camcol != 'any': # d = path_join(d,'bycamcol') return d
def columns_dir(self): self.photo_sweep=os.environ['PHOTO_SWEEP'] dirname=os.path.basename(self.photo_sweep) #dirname += '_new' dir=os.environ['SWEEP_REDUCE'] dir = path_join(dir,dirname) if not os.path.exists(dir): os.makedirs(dir) dir=path_join(dir ,self.type+'.cols') return dir
def make_dc5b_data_html(): d=os.environ['DESFILES_DIR'] f = path_join(d,'dc5b','dc5b-runfiles-stat.json') runstat = eu.io.read(f) desdir = os.environ['DESDATA'] www_dir=path_join(desdir,'www','dc5b') fhtml = path_join(www_dir, 'dc5bstat.html') fobj = open(fhtml, 'w') fobj.write('<html>\n') fobj.write('<body>\n') fobj.write('<head><link rel="stylesheet" href="../table.css" type="text/css"></head>\n') fobj.write('<h1>DC5b remote files</h1>\n') fobj.write("""<pre> The following table summarizes the location of data for DC5b runs. Only the desar machine is currently visible to BNL machines, so only desar1,desar2,desardata locations included. </pre>""") fobj.write('<table class=simple>\n') fobj.write(' <tr><th>run</th><th>ftype</th><th>num</th><th>size (TB)</th><th>original<br>location</th><th>BNL<br>Has</th></tr>\n') # single epoch for ri in runstat['info']: run = ri['run'] for ftype in ['red','red_cat']: num = ri[ftype]['num'] sizetb = ri[ftype]['sizetb'] host = ri[ftype]['host'] have = ri[ftype]['bnlhas'] fobj.write(' <tr><td>') data = [run, ftype, str(num), str(sizetb), host, have] data = '</td><td>'.join(data) fobj.write(data) fobj.write('</td></tr>\n') fobj.write('</table>\n') fobj.write('</html>\n') fobj.write('</body>\n') fobj.close()
def plotfile(self, field, rmag_max, plot_type='meane'): camcol=self.camcol d=self.plotdir() f = 'rg-%(run)s%(band)s-%(type)s-vs-%(field)s-%(rmag_max)0.2f' f = f % {'run':self.procrun,'band':self.band,'type':plot_type, 'field':field,'rmag_max':rmag_max} if self.run != 'any': f += '-%06i' % self.run if camcol != 'any': d = path_join(d,'bycamcol') f += '-%i' % camcol f += '.eps' f=path_join(d,f) return f
def fname(self, scinv=False): d = self.dir() f='%s-sources.fits' % self['catalog'] if scinv: f=f.replace('sources','sources-scinv') infile = path_join(d, f) return infile
def combine_scinv(self): from glob import glob outfile=self.combined_file() stdout.write("Will write to file: %s\n" % outfile) dir=self.output_dir() pattern = self.scinv_file_pattern() pattern=path_join(dir, pattern) flist = glob(pattern) datalist = [] idmin = 0 for f in flist: print f tdata = esutil.io.read(f) data = numpy_util.add_fields(tdata, [('zid','i4')]) data['zid'] = idmin + numpy.arange(data.size) idmin += data.size print data['zid'].min(), data['zid'].max() datalist.append(data) hdr = esutil.sfile.read_header(flist[0]) print 'combining data' data = numpy_util.combine_arrlist(datalist) print 'writing file: %s' % outfile esutil.sfile.write(data, outfile, header=hdr)
def __init__(self, photo_sample, nchunk=None): """ E.g. procrun='prim04' """ self.prefix='zinput' self.photo_sample = photo_sample self.nchunk = nchunk self.data = None self.keep_indices=None pzdir = zphot.photoz_dir() self.dir = path_join(pzdir,'inputs',self.photo_sample) print("photo dir: ",self.dir) self.conf = zphot.read_config('zinput',photo_sample) if self.conf['photo_sample'] != photo_sample: raise ValueError("photo_sample in config is not '%s'" \ % photo_sample) pprint.pprint(self.conf) if self.conf['weighting']: self.conf['filetype'] = 'dat' source = self.conf.get('source','dr8_final') if source != 'dr8_final': raise ValueError("only set up for dr8_final source")
def catalog_file(type): if type == "public-v1": name = "maxbcg_public_catalog.fit" else: raise ValueError("don't support catalog type '%s' yet" % type) d = catalog_dir() f = path_join(d, name) return f
def config_file(type, id, ext="yaml"): """ Want to move over to yaml files """ dir = config_dir() fname = "%s-%s.%s" % (type, id, ext) fname = path_join(dir, fname) return fname
def seeing_plotfile(self, name, greyscale=False): dir = self.plotdir() s=self.train_sample if greyscale: s+='-bw' fname = '%s-match-seeing-%s.eps' % (name,s) fname=path_join(dir, fname) return fname
def plotfile(self, field, rmag_max): f = 'chrm-meane-ri-vs-%s-rmag%0.2f' % (field, rmag_max) if self.run != 'any': f += '-%06i' % self.run f += '.eps' d=self.plotdir() f=path_join(d,f) return f
def R_polyfile(self, camcol, rmag_max): f = 'rg-%(run)s%(band)s-meane-vs-R-%(rmag_max)0.2f-%(camcol)s-polyfit.yaml' f = f % {'run':self.procrun,'band':self.band, 'rmag_max':rmag_max,'camcol':camcol} d=self.plotdir() f=path_join(d,'bycamcol',f) return f
def plotfile(self, type, extra=None): dir = self.plotdir() fname = self.fname_matched(type) fname = os.path.basename(fname) fname = fname.replace('.rec','.eps') if extra is not None: fname = fname.replace('.eps','-'+extra+'.eps') fname = path_join(dir,fname) return fname
def input_coldir(name, version): """ version should be e.g. dr8-v2 """ basedir = os.environ["CLUSTERS_INPUT"] coldir = "%s-input-%s.cols" % (name, version) coldir = path_join(basedir, coldir) return coldir
def dir(self): dir=os.environ.get('REGAUSSIM_DIR',None) if dir is None: raise ValueError("REGAUSSIM_DIR must be set") dir=path_join(dir,'admom-covar-meas-vs-input') if not os.path.exists(dir): os.makedirs(dir) return dir
def init(self, train_sample): self.types = {'primus':'primus.zerod.10oct29.zconf4', '2slaq':'2slaq', 'cfrs':'cfrs', 'cnoc2':'cnoc2.cut', 'deep2':'deep2.form.fix', 'sdss':'sdssobjids', 'tkrs':'tkrs-fix', 'vvds':'vvds', 'zcosmos':'zcosmos'} self.dir_matched = path_join(self.basedir, 'matched', train_sample) if self.no_photo_cuts: self.dir_matched = path_join(self.dir_matched,'no_photo_cuts') self.conf = zphot.read_config('train',self.train_sample) self.photo_conf = zphot.read_config('zinput',self.conf['photo_sample']) pprint.pprint(self.conf)
def columns_dir(self): if 'SWEEP_REDUCE' not in os.environ: raise ValueError("SWEEP_REDUCE is not set") dir = os.environ['SWEEP_REDUCE'] if 'PHOTO_SWEEP' not in os.environ: raise ValueError("PHOTO_SWEEP is not set") sweep=os.environ['PHOTO_SWEEP'] sweep = os.path.basename(sweep) dir = path_join(dir, sweep, self.name()+'.cols') return dir
def read_exposure_data(dir,exposurename): out_dtype=[('ccd','i1'), ('x','f4'),('y','f4'), ('psf_flags','i4'), ('e1','f4'),('e2','f4'), ('e1interp','f4'),('e2interp','f4')] datalist=[] for ccd in xrange(1,1+62): psf_file = path_join(dir,'%s_%02d_psf.fits' % (exposurename,ccd)) psf=esutil.io.read(psf_file) psfe1 = psf['shapelets'][:,3]*sqrt(2) psfe2 = -psf['shapelets'][:,4]*sqrt(2) shear_file = path_join(dir,'%s_%02d_shear.fits' % (exposurename,ccd)) shear=esutil.io.read(shear_file) e1interp = shear['interp_psf_coeffs'][:,3]*sqrt(2) e2interp = -shear['interp_psf_coeffs'][:,4]*sqrt(2) mpsf,mshear=numpy_util.match(psf['id'], shear['id']) tdata = numpy.zeros(mpsf.size, dtype=out_dtype) tdata['ccd'] = ccd tdata['x'] = psf['x'][mpsf] tdata['y'] = psf['y'][mpsf] tdata['e1'] = psfe1[mpsf] tdata['e2'] = psfe2[mpsf] tdata['psf_flags'] = psf['psf_flags'][mpsf] tdata['e1interp'] = e1interp[mshear] tdata['e2interp'] = e2interp[mshear] datalist.append(tdata) data = numpy_util.combine_arrlist(datalist) return data
def __init__(self, train_sample=None, no_photo_cuts=False): self.train_sample = train_sample self.dir_matched = None self.no_photo_cuts = no_photo_cuts pzdir = zphot.photoz_dir() self.basedir = path_join(pzdir, 'training') self.prefix = 'train' self.conf=None if self.train_sample is not None: self.init(self.train_sample)
def combine_matches(self): """ Combine all the caches into one big file. """ dir = self.match_dir() pattern = path_join(dir,'match-regauss-*-*-*.fits') files = glob.glob(pattern) files.sort() print("Found",len(files),"files") data = eu.io.read(files, combine=True) outfile=self.match_file('all') eu.io.write(outfile,data,verbose=True)
def match_file(self, run): """ run might be 'all' """ dir=self.match_dir() if not os.path.exists(dir): os.makedirs(dir) if run == 'all': f='match-regauss-%s-%s.fits' % (self.procrun,self.band) else: f='match-regauss-%s-%s-%06d.fits' % (self.procrun,self.band,run) f = path_join(dir,f) return f
def __init__(self, origin='uchicago', type='dr7pofz'): self.good_origins = ['uchicago'] if origin not in self.good_origins: raise ValueError("origin should be in: %s" % self.good_origins) self.good_types = ['dr7pofz'] if type not in self.good_types: raise ValueError("type should be in: %s" % self.good_types) self.origin = origin self.type=type self.basedir = path_join('~esheldon','photoz',origin,type) self.basedir = os.path.expanduser(self.basedir)
def collate_se_pointing(serun, expname): """ Collate the output data from $DESDATA/wlbnl/$SERUN/pointing With the input images and catalogs. All I do is look at the .json files to get the images/catalogs and copy all into a single directory under $DESDATA/wlbnl/$SERUN/collated/pointings """ import shutil dir = deswl.files.wlse_collated_dir(serun) dir = path_join(dir, 'pointings', expname) if not os.path.exists(dir): os.mkdir(dir) # now read all the json files and copy data over for ccd in range(1,62+1): statfile=deswl.files.wlse_path(expname, ccd, 'stat', serun=serun) stdout.write("\nReading stat file: %s\n" % statfile) stat = esutil.io.read(statfile) for type in ['imfile','catfile','psf','fitpsf','shear','stars']: fullpath=stat[type] if fullpath.find('scratch') != -1: fullpath = \ fullpath.replace(stat['rootdir'],deswl.files.des_rootdir()) bname=os.path.basename(fullpath) outfile = path_join(dir, bname) stdout.write("Copying from %s to %s\n" % (fullpath,outfile)) shutil.copy2(fullpath, outfile)
def all_other_fdict(self,type): """ Trying to recover a given training set using the everything set """ dir = self.all_other_dir() fdict={} fdict['dir'] = dir fdict['nnear1'] = 5 fdict['nnear2'] = 100 fdict['trainfile'] = self.wt.filename('all') # this is the original file, it has the z values origfile = self.wt.filename(type) fdict['origfile'] = origfile # we need to write out origfile in the photofile format photofile = os.path.basename(origfile).replace('.dat','-asphoto.dat') photofile = path_join(dir, photofile) fdict['photofile'] = photofile tbase = os.path.basename(fdict['trainfile']) pbase = os.path.basename(fdict['photofile']) fdict['wfile1'] = \ path_join(dir,tbase.replace('.dat','-%s-weights-5.dat' % type)) fdict['wfile_nozero1'] = \ path_join(dir,tbase.replace('.dat','-%s-weights-nozero-5.dat' % type)) fdict['wfile2'] = \ path_join(dir,tbase.replace('.dat','-%s-weights-100.dat' % type)) fdict['numfile1'] = path_join(dir,pbase.replace('.dat','-num-5.dat')) fdict['numfile2'] = path_join(dir,pbase.replace('.dat','-num-100.dat')) fdict['wscript']=path_join(dir,type+'-doweights.sh') fdict['pofzscript']=path_join(dir,type+'-dopofz.sh') fdict['zhistfile'] = path_join(dir, pbase.replace('.dat','-zhist.eps') ) return fdict
def get_exposure_wcs_example(): from esutil.ostools import getenv_check, path_join import cPickle as pickle expname='decam--24--15-i-6' d=getenv_check('DESFILES_DIR') pickle_path=path_join(d, 'wcsexample', 'wcs-'+expname+'.pickle') if not os.path.exists(pickle_path): stdout.write("Creating pickled version: %s\n" % pickle_path) tmpwcs = get_exposure_wcs(expname) pickle.dump(tmpwcs, open(pickle_path,'w')) stdout.write("Don't forget to svn add!\n") stdout.write("Loading wcs example: %s\n" % pickle_path) wcs = pickle.load(open(pickle_path)) return wcs
def extract_psf_fwhm(self): """ This could have been avoided had I added psf_fwhm to the epochs Not all will match the sweeps, we will have to collate the results """ import es_sdsspy sgdir = self.dir() extract_file = path_join(sgdir, 'epochs82-psf-fwhm.rec') cols = self.open_columns('epochs82') extract_cols = ['run','rerun','camcol','field','id','psf_fwhm'] pid = cols['photoid'][:] se=es_sdsspy.sweeps.SweepExtractor(pid, allow_nomatch=True) se.extract(extract_cols, extract_file)
def plotfile(self, run, objmodel, psfmodel, type, alt=None, s2=None, Rmin=None, dotitle=False, yrange=None): dir = get_plotdir(run) f = 'rgsim-%sobj-%spsf-%s' % (objmodel, psfmodel, type) if alt is not None: f += '-alt'+alt if s2 is not None: f += '-s2-%0.3f' % s2 if Rmin is not None: f += '-Rmin%0.3f' % Rmin if dotitle: f += '-tit' if yrange is not None: f += '-yr%0.3f-%0.3f' % tuple(yrange) f += '.eps' f = path_join(dir, f) return f
def rotdir(type='eq'): d = os.environ['LENSDIR'] d = path_join(d, 'sdss-shape-rot', type) return d
def rotfile(run, type='eq'): d = rotdir(type) f = '%srot-%06i-301.fits' % (type, run) f = path_join(d, f) return f