def doQueryType(t, curMD, tNum, cSize=cSize, obsId=obsId): print 'NOW QUERYING:', t myQDB = queryDB.queryDB(chunksize=cSize, objtype=t) ic = myQDB.getInstanceCatalogById(obsId) while ic != None: curMD = doIC(ic, myQDB, curMD, tNum) tNum += 1 ic = myQDB.getNextChunk() return curMD, tNum, myQDB
def runDia(csize, obsidList, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False): if repodir is None: repodir = outdir meta = None opsimid = None writeJobEvent(je, 'start') cattype = "DIASOURCE" objtype = 'SSM' warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "diaBatchStarting_%i"%(obsidList[0]) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) if not os.path.exists(outBase): os.makedirs(outBase) writeJobEvent(je, 'MakeDirs', 'Made output directories %s'%(outBase)) nid = 0 for obsid in obsidList: writeJobEvent(je, 'Obshistid:%s'%(obsid), 'Doing %i out of %i total'%(nid, len(obsidList))) filename = "dia_%i.dat"%(obsid) outfile = os.path.join(outBase,filename) myqdb = queryDB.queryDB(chunksize=csize,objtype=objtype,filetypes=(cattype,)) ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent(je, 'GetChunk', 'Got chunk #%i of length %i'%(cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.metadata.validateMetadata(cattype, opsimid) ic.metadata.writeMetadata(outfile, cattype, opsimid, newfile=True, filelist=None, compress=compress) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s'%(outfile)) ic.validateData(cattype) ic.writeCatalogData(outfile, cattype, newfile = False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i'%(cnum,numRec)) ic = myqdb.getNextChunk() cnum += 1 myqdb.closeSession() nid += 1 mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')
def runCalib(csize, obsid, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False): if repodir is None: repodir = outdir meta = None opsimid = None files = [] writeJobEvent(je, 'start') cattype = "CALIB" print "Warning -- only MSSTARS in calib cats currently" #objtypes = ['MSSTARS','WDSTARS','BHBSTARS','RRLYSTARS', \ # 'GLENS','IMAGE','EBSTARS','CEPHEIDSTARS','EASTEREGGS','GALAXY_BULGE','GALAXY_DISK','AGN'] #varobj = ['MSSTARS', 'RRLYSTARS', 'AGN', 'IMAGE', 'WDSTARS', 'EBSTARS', 'CEPHEIDSTARS'] objtypes = ['MSSTARS', ] varobj = ['MSSTARS',] warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "obsid%i"%(obsid) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) subdir = "pops" popsPath = os.path.join(outBase, subdir) if not os.path.exists(popsPath): os.makedirs(popsPath) writeJobEvent(je, 'MakeDirs', 'Made output directories %s'%(popsPath)) files = [] for objtype in objtypes: writeJobEvent(je, 'Object:%s'%(objtype), 'Doing %s out of: %s'%(objtype, ",".join(objtypes))) filename = "trim_%i_%s.dat"%(obsid,objtype) outfile = os.path.join(popsPath,filename) myqdb = queryDB.queryDB(chunksize=csize,objtype=objtype) # This grabs the metadata and grabs the catalog in a circle around central pointing. # WHY COMBINED? DO WE NEED TO SPLIT TO PROVIDE ADDITIONAL METADATA EASILY? ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent(je, 'GetChunk', 'Got chunk #%i of length %i'%(cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) # Calculate focal plane mm/mm and chip x/y for each object. ic.makeXyCoords() writeJobEvent(je, 'MakeXY', 'Made xy coords for chunk #%i'%(cnum)) # Calculate calib counts for each object. ic.calcCalibCounts() writeJobEvent(je, 'calcCalibCounts', 'Generated throughput curve and calculated counts for chunk #%i' %(cnum)) if cnum == 0: if compress: files.append(os.path.join(subdir,filename)+".gz") else: files.append(os.path.join(subdir,filename)) # Okay, need to understand this a little more ... think it's mostly formatting/writing metadata. # NEED TO WRITE A NEW VERSION? mUtils.trimGeneration.derivedTrimMetadata(ic) if meta is None: meta = deepcopy(ic.metadata) else: meta.mergeMetadata(ic.metadata) if objtype in varobj: writeJobEvent(je, 'DoVar', 'Applying variability to chunk #%i of type %s'%(cnum, objtype)) ic.applyVariability() # WILL NEED SOMETHING HERE FOR CALIB CAT ic.validateData('TRIM') numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.writeCatalogData(outfile, "TRIM", newfile = True, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote first chunk of length %i'%(numRec)) else: ic.writeCatalogData(outfile, "TRIM", newfile = False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i'%(cnum,numRec)) if numRec == csize: ic = myqdb.getNextChunk() else: ic = None cnum += 1 writeJobEvent(je, 'Finished Object:%s'%(objtype), 'Finished object %s'%(objtype)) meta.validateMetadata(cattype, opsimid) metaOutfile = os.path.join(outBase,"metadata_%i.dat"%(obsid)) meta.writeMetadata(metaOutfile, cattype, opsimid, newfile=True, filelist=files, compress=False) #files.append(os.path.join("obsid%i"%(obsid),"metadata_%i.dat"%(obsid))) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s'%(metaOutfile)) mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')
#!/usr/bin/env python import copy from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox if __name__ == "__main__": csize = 10 objects = ("GALAXY", ) myqdb = queryDB.queryDB(chunksize=csize, objtype="GALAXY") ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) curMD.validateMetadata('TRIM')
#!/usr/bin/env python from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox if __name__ == "__main__": csize = 10 objects = ("GALAXY",) myqdb = queryDB.queryDB(chunksize=csize, objtype="GALAXY") ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters
#!/usr/bin/env python import numpy, math, copy, cPickle, os, sys from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox import lsst.sims.catalogs.measures.utils as mUtils csize = 10 myqdb = queryDB.queryDB(chunksize=csize, objtype=sys.argv[1]) ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.calculateUnrefractedAltAz() print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) curMD.writeMetadata('testMetadata.dat', 'TRIM', myqdb.opsim, newfile=True) cPickle.dump(ic, open('test.pkl', 'w')) #ic.makeHelio() ic.makeTrimCoords() ic.metadata.validateMetadata('TRIM', myqdb.opsim) print ic.dataArray ic.validateData('TRIM') ic.writeCatalogData('iCTest.txt', 'TRIM') ic.metadata.writeMetadata('iCTest.meta', 'TRIM', myqdb.opsim, newfile=True)
#!/usr/bin/env python import copy from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox if __name__ == "__main__": csize = 10 myqdb = queryDB.queryDB(chunksize=csize, objtype="SSM") ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) curMD.validateRequiredMetadata('TRIM', myqdb.opsim)
def doOneCatalogType(self, catalogType, queryTypes, obsHistID): #nFN = self.getNextGoodFileNum() fullTimeStart = time.time() self.executionDBManager = jobDB.JobState() t0 = self.executionDBManager.getJobId() nFN = '%s_%s' % (t0.getOwner(), t0.getId()) print 'Using job ID: %s' % nFN print 'queryTypes:', queryTypes jobNum = 0 jobTypes = []; jobNums = []; jobPickleFiles = []; useTypes = [] allOutputFiles = []; curMD = None self.metaDataManager.reset() os.system('free -m') for objectType in queryTypes: if objectType not in useTypes: useTypes.append(objectType) print 'Getting first %s instance catalog of size %i...' % ( objectType, self.chunkSize) t0 = time.time() myQDB = queryDB.queryDB( chunksize=self.chunkSize, objtype=objectType) print ' ...setting up QDB took %i sec.' % (time.time() - t0) t0 = time.time() instanceCat = myQDB.getInstanceCatalogById(obsHistID) print ' ...and getting catalog took %i sec.' % (time.time() - t0) numCats = 0 if instanceCat != None: # This code adds some needed fields to the metadata mUtils.trimGeneration.derivedTrimMetadata(instanceCat) os.system('free -m') # Deep copy so we can store this after instanceCat disappears if curMD == None: curMD = copy.deepcopy(instanceCat.metadata) else: curMD.mergeMetadata(instanceCat.metadata) while instanceCat != None: t0 = self.WorkDir + 'catData%s_%i.ja' % (nFN, jobNum) t1 = self.WorkDir + 'catData%s_%i.p' % (nFN, jobNum) print 'Now pickling query type: %s' % objectType # Store job data files in instance time0 = time.time() instanceCat.jobAllocatorDataFile = t0 allOutputFiles.append(t0) # Order is important instanceCat.jobAllocatorCatalogType = catalogType instanceCat.jobAllocatorObjectType = objectType cPickle.dump(instanceCat, open(t1, 'w')) print ' ...pickling took %i sec.' % (time.time() - time0) jobTypes.append(catalogType) jobNums.append(jobNum) jobPickleFiles.append(t1) jobNum += 1 if numCats > 0: curMD.mergeMetadata(instanceCat.metadata) # *** RRG: Free up memory somehow here for instanceCat... del(instanceCat); instanceCat = None os.system('free -m') if self.maxCats >= 0 and (numCats + 1) >= self.maxCats: instanceCat = None else: print 'Querying DB for next chunk.' t0 = time.time() instanceCat = myQDB.getNextChunk() print ' ...took %i sec.' % (time.time() - t0) if instanceCat != None: # This code adds some needed fields to the metadata mUtils.trimGeneration.derivedTrimMetadata(instanceCat) os.system('free -m') numCats += 1 # RRG: For now this must be disabled #curMD.validateMetadata(catalogType, myQDB.opsim) mFName = self.WorkDir + 'metaData%s_%s.ja' % (nFN, catalogType) curMD.writeMetadata(mFName, catalogType, myQDB.opsim, newfile=True) # Finished with queryDB; clean up nicely. myQDB.closeSession() # For debug mode, don't start the clients if self.QueryOnly == True: print 'Full time for this file: %i sec' % (time.time()-fullTimeStart) print 'DEBUG: Finished, no client processes started.' # Now fire off the jobs for i in range(len(jobNums)): jobId = '%s_%i' % (nFN, jobNums[i]) self.executionDBManager.updateState(jobId, 'JAAdded') print 'Added job to execution DB: %s' % jobId #t0 = '/astro/apps/pkg/python64/bin/python jobAllocatorRun.py %i %s %s&' % (nFN, jobId, jobPickleFiles[i]) #t0 = 'qsub ./runOneAthena.csh %i %s %s&' % (nFN, jobId, jobPickleFiles[i]) #t0 = 'ssh minerva0 "(cd $PBS_O_WORKDIR; qsub ./runOneAthena.csh %i %s %s)"' % (nFN, jobId, jobPickleFiles[i]) cwd0 = os.getcwd() f0 = open('tmpJA%s.csh' % jobId, 'w') f0.write('#!/bin/csh\n#PBS -N jA%s\n#PBS -l walltime=1:00:00\n#PBS -e jA%s.err\n#PBS -o jA%s.out\ncd %s\nsource setupAthena.csh\npython jobAllocatorRun.py %s %s %s\necho Finished.' % (jobId, jobId, jobId, cwd0, nFN, jobId, jobPickleFiles[i])) f0.close() t0 = 'ssh minerva0 "(cd %s; /opt/torque/bin/qsub tmpJA%s.csh)"' % (cwd0, jobId) print t0 os.system(t0) # Check that everything started within a certain time limit # On minerva, jobs may be queued indefinitely, so this won't work for i in range(len(jobNums)): jobId = '%s_%i' % (nFN, jobNums[i]) tryNum = 0 t0 = self.executionDBManager.queryState(jobId) while t0 != 'JAFinished': print 'Try %i: JA sees state for %s: %s' % (tryNum, jobId, t0) time.sleep(10) # Give it up to a day if tryNum > 60 * 60 * 24: raise RuntimeError, '*** Job not started: %s' % jobId tryNum += 1 t0 = self.executionDBManager.queryState(jobId) print 'Finished (Try %i): JA sees state for %s: %s' % (tryNum, jobId, t0) # Finally, merge the output trim file trimFile = self.WorkDir + 'trim%s_%s.ja' % (nFN, catalogType) t0 = 'cat %s > %s' % (mFName, trimFile) print t0 os.system(t0) for f in allOutputFiles: t0 = 'cat %s >> %s' % (f, trimFile) print t0 os.system(t0) print 'Full time for this file: %i sec' % (time.time()-fullTimeStart) print 'Finished catting trim file: ', trimFile
def runTrim(csize, obsid, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False, dodither=False, objtypes = ['SSM', 'MSSTARS','WDSTARS','BHBSTARS','RRLYSTARS',\ 'GLENS','IMAGE','EBSTARS','CEPHEIDSTARS','EASTEREGGS','GALAXY_BULGE','GALAXY_DISK','AGN'], varobj = ['MSSTARS', 'RRLYSTARS', 'AGN', 'IMAGE', 'WDSTARS', 'EBSTARS', 'CEPHEIDSTARS']): if repodir is None: repodir = outdir if dodither: opsimname = 'DITHEREDOPSIM361' else: opsimname = 'OPSIM361' meta = None opsimid = None files = [] writeJobEvent(je, 'start') cattype = "TRIM" warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "obsid%i"%(obsid) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) subdir = "pops" popsPath = os.path.join(outBase, subdir) if not os.path.exists(popsPath): os.makedirs(popsPath) writeJobEvent(je, 'MakeDirs', 'Made output directories %s'%(popsPath)) files = [] for objtype in objtypes: print "Doing object type: %s"%(objtype) writeJobEvent(je, 'Object:%s'%(objtype), 'Doing %s out of: %s'%(objtype, ",".join(objtypes))) filename = "trim_%i_%s.dat"%(obsid,objtype) outfile = os.path.join(popsPath,filename) myqdb = queryDB.queryDB(chunksize=csize,objtype=objtype,dithered=dodither) ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius, opsim=opsimname) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent(je, 'GetChunk', 'Got chunk #%i of length %i'%(cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) ic.makeTrimCoords() writeJobEvent(je, 'MakeTrim', 'Made trim coords for chunk #%i'%(cnum)) if cnum == 0: if compress: files.append(os.path.join(subdir,filename)+".gz") else: files.append(os.path.join(subdir,filename)) mUtils.trimGeneration.derivedTrimMetadata(ic) if meta is None: meta = deepcopy(ic.metadata) else: meta.mergeMetadata(ic.metadata) if objtype in varobj: writeJobEvent(je, 'DoVar', 'Applying variability to chunk #%i of type %s'%(cnum, objtype)) ic.applyVariability() ic.validateData('TRIM') numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.writeCatalogData(outfile, "TRIM", newfile = True, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote first chunk of length %i'%(numRec)) else: ic.writeCatalogData(outfile, "TRIM", newfile = False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i'%(cnum,numRec)) if numRec == csize: ic = myqdb.getNextChunk() else: ic = None cnum += 1 writeJobEvent(je, 'Finished Object:%s'%(objtype), 'Finished object %s'%(objtype)) meta.validateMetadata(cattype, opsimid) metaOutfile = os.path.join(outBase,"metadata_%i.dat"%(obsid)) meta.writeMetadata(metaOutfile, cattype, opsimid, newfile=True, filelist=files, compress=False) #files.append(os.path.join("obsid%i"%(obsid),"metadata_%i.dat"%(obsid))) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s'%(metaOutfile)) mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')
#!/usr/bin/env python import numpy import math import copy import cPickle from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox import lsst.sims.catalogs.measures.utils as mUtils csize = 10 myqdb = queryDB.queryDB(chunksize=csize, objtype="ALLSTARS") ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.calculateUnrefractedAltAz() print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) curMD.writeMetadata('testMetadata.dat', 'TRIM', myqdb.opsim, newfile=True) cPickle.dump(ic, open('test.pkl', 'w')) #ic.makeHelio() ic.makeTrimCoords() ic.metadata.validateMetadata('TRIM', myqdb.opsim) print ic.dataArray ic.validateData('TRIM') ic.writeCatalogData('iCTest.txt', 'TRIM') ic.metadata.writeMetadata('iCTest.meta','TRIM', myqdb.opsim, newfile = True)
if __name__ == "__main__": csize = 100000 nsamp = 10 magcut = 21.5 boxsizedeg = 0.2 halfboxsizedeg = boxsizedeg / 2. sqarea = (math.sin(halfboxsizedeg * math.pi / 180.) - math.sin( -halfboxsizedeg * math.pi / 180.)) * (boxsizedeg * math.pi / 180.) radius = math.acos(1 - (sqarea / (2. * math.pi))) * 180. / math.pi ra, dec = generateRaDec(nsamp) fhout = open(sys.argv[1], "w") for i in xrange(nsamp): nums = {'u': 0, 'g': 0, 'r': 0, 'i': 0, 'z': 0, 'y': 0} expmjd = 0. myqdb = queryDB.queryDB(chunksize=csize, objtype='CALIBSTARS', filetypes=('CALIB', )) ic = myqdb.getInstanceCatalogByCirc(ra[i], dec[i], radius, expmjd=expmjd) while not ic is None: for f in nums.keys(): nums[f] += len(numpy.where(ic.dataArray[f] < magcut)[0]) ic = myqdb.getNextChunk() fhout.write(",".join([str(x) for x in\ (ra[i],dec[i],sqarea,nums['u'],nums['g'],nums['r'],nums['i'],nums['z'],nums['y'])])+"\n") fhout.flush() fhout.close()
def runDia(csize, obsidList, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False): if repodir is None: repodir = outdir meta = None opsimid = None writeJobEvent(je, 'start') cattype = "DIASOURCE" objtype = 'SSM' warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "diaBatchStarting_%i" % (obsidList[0]) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) if not os.path.exists(outBase): os.makedirs(outBase) writeJobEvent(je, 'MakeDirs', 'Made output directories %s' % (outBase)) nid = 0 for obsid in obsidList: writeJobEvent(je, 'Obshistid:%s' % (obsid), 'Doing %i out of %i total' % (nid, len(obsidList))) filename = "dia_%i.dat" % (obsid) outfile = os.path.join(outBase, filename) myqdb = queryDB.queryDB(chunksize=csize, objtype=objtype, filetypes=(cattype, )) ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent( je, 'GetChunk', 'Got chunk #%i of length %i' % (cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.metadata.validateMetadata(cattype, opsimid) ic.metadata.writeMetadata(outfile, cattype, opsimid, newfile=True, filelist=None, compress=compress) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s' % (outfile)) ic.validateData(cattype) ic.writeCatalogData(outfile, cattype, newfile=False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i' % (cnum, numRec)) ic = myqdb.getNextChunk() cnum += 1 myqdb.closeSession() nid += 1 mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')
def runTrim(csize, obsid, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False, dodither=False, objtypes = ['SSM', 'MSSTARS','WDSTARS','BHBSTARS','RRLYSTARS',\ 'GLENS','IMAGE','EBSTARS','CEPHEIDSTARS','EASTEREGGS','GALAXY_BULGE','GALAXY_DISK','AGN'], varobj = ['MSSTARS', 'RRLYSTARS', 'AGN', 'IMAGE', 'WDSTARS', 'EBSTARS', 'CEPHEIDSTARS']): if repodir is None: repodir = outdir if dodither: opsimname = 'DITHEREDOPSIM361' else: opsimname = 'OPSIM361' meta = None opsimid = None files = [] writeJobEvent(je, 'start') cattype = "TRIM" warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "obsid%i" % (obsid) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) subdir = "pops" popsPath = os.path.join(outBase, subdir) if not os.path.exists(popsPath): os.makedirs(popsPath) writeJobEvent(je, 'MakeDirs', 'Made output directories %s' % (popsPath)) files = [] for objtype in objtypes: print "Doing object type: %s" % (objtype) writeJobEvent(je, 'Object:%s' % (objtype), 'Doing %s out of: %s' % (objtype, ",".join(objtypes))) filename = "trim_%i_%s.dat" % (obsid, objtype) outfile = os.path.join(popsPath, filename) myqdb = queryDB.queryDB(chunksize=csize, objtype=objtype, dithered=dodither) ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius, opsim=opsimname) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent( je, 'GetChunk', 'Got chunk #%i of length %i' % (cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) ic.makeTrimCoords() writeJobEvent(je, 'MakeTrim', 'Made trim coords for chunk #%i' % (cnum)) if cnum == 0: if compress: files.append(os.path.join(subdir, filename) + ".gz") else: files.append(os.path.join(subdir, filename)) mUtils.trimGeneration.derivedTrimMetadata(ic) if meta is None: meta = deepcopy(ic.metadata) else: meta.mergeMetadata(ic.metadata) if objtype in varobj: writeJobEvent( je, 'DoVar', 'Applying variability to chunk #%i of type %s' % (cnum, objtype)) ic.applyVariability() ic.validateData('TRIM') numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.writeCatalogData(outfile, "TRIM", newfile=True, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote first chunk of length %i' % (numRec)) else: ic.writeCatalogData(outfile, "TRIM", newfile=False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i' % (cnum, numRec)) if numRec == csize: ic = myqdb.getNextChunk() else: ic = None cnum += 1 writeJobEvent(je, 'Finished Object:%s' % (objtype), 'Finished object %s' % (objtype)) meta.validateMetadata(cattype, opsimid) metaOutfile = os.path.join(outBase, "metadata_%i.dat" % (obsid)) meta.writeMetadata(metaOutfile, cattype, opsimid, newfile=True, filelist=files, compress=False) #files.append(os.path.join("obsid%i"%(obsid),"metadata_%i.dat"%(obsid))) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s' % (metaOutfile)) mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')
def findGalaxies(ra_center, dec_center, radius, rMagMax, baMin, baMax): # CONSTANTS rad2deg = 180./math.pi csize = 100000 obj = "ASSEMBLEDGALAXY" expmjd = 0. gal_count = 0 wavelen_step = 0.25 Sixflts = ['float','float','float','float','float','float'] # BANDPASSES # Instantiate bandpasses and read into lsstbp. bpdir = os.getenv("LSST_THROUGHPUTS_BASELINE") filterlist = ('u', 'g', 'r', 'i', 'z', 'y') lsstbp = {} for f in filterlist: lsstbp[f] = Bandpass(wavelen_min=300,wavelen_max=1200, wavelen_step=0.1) lsstbp[f].readThroughput(os.path.join(bpdir, 'total_' + f + '.dat'), wavelen_step=wavelen_step) # SEDS # Read in all of the galaxy seds in the root directory: galdir = "/astro/net/lsst1/shared/data/galaxySED/" gals = {} gallist = os.listdir(galdir) for i in range(len(gallist)): if gallist[i].endswith('.gz'): gallist[i] = gallist[i][:-3] for gal in gallist: gals[gal] = Sed() gals[gal].readSED_flambda(os.path.join(galdir, gal)) # Check on resampling - want all galaxy seds to have the same wavelength range. # (although our LSST ranges are 300 - 1200 nm, the SED's are ~27 to 2290 nm). if ((gals[gallist[0]].wavelen.min() < 30) & (gals[gallist[0]].wavelen.max() > 2000)): # If true, then gals[gallist[0]] is okay to use as a template -- this ought to be true. wavelen_match = gals[gallist[0]].wavelen else: print "Had to use simple wavelength array for matching" wavelen_match = numpy.arange(30, 2200, 0.1, dtype='float') for gal in gallist: if gals[gal].needResample(wavelen_match = wavelen_match): gals[gal].resampleSED(wavelen_match = wavelen_match) # Create the galactic a, b values from CCM 89 for the source galaxy and ours. # adjust for redshift, add Milky Way dust (recalculate a and b), # normalize fluxes and calculate magnitudes: # First: calculate source galaxy a/b on wavelength range required for # source galaxy (internal)from CCM 89. a_int, b_int = gals[gallist[0]].setupCCMab() # Second: calculate milky way a/b on wavelength range required for calculating # magnitudes - i.e. 300 to 1200 nm. # Set up a Sed object that is the same for all galaxies, all chunks. # Start with a flat SED with F_AB = 3631 Jy tmpgal = Sed() tmpgal.setFlatSED(wavelen_min=300, wavelen_max=1200, wavelen_step=wavelen_step) a_mw, b_mw = tmpgal.setupCCMab() # so this is a/b on 300-1200 range. # Set up phi, the wavelength-normalized system response for each filter, for each bandpass. # sb is the system response function (throughputs). Also set up a bandpass list, for # manyMagCalc method and initiate mags w/dust. bplist = [] for f in filterlist: lsstbp[f].sbTophi() bplist.append(lsstbp[f]) phiarray, dlambda = tmpgal.setupPhiArray(bplist) objId = numpy.empty(0) ra = numpy.empty(0) dec = numpy.empty(0) diskFluxNorm = numpy.empty(0) bulgeFluxNorm = numpy.empty(0) diskSedFilename = numpy.empty(0) bulgeSedFilename = numpy.empty(0) a_disk = numpy.empty(0) b_disk = numpy.empty(0) a_bulge = numpy.empty(0) b_bulge = numpy.empty(0) pa_d = numpy.empty(0) rdshft = numpy.empty(0) Rmags = numpy.empty(0) #QUERY #myqdb = queryDB.queryDB(chunksize=csize,objtype=obj, filetypes=('REFERENCECATALOG',)) myqdb = queryDB.queryDB(chunksize=csize,objtype=obj, filetypes=('TEST',)) # Specify a circular field of view: ic = myqdb.getInstanceCatalogByCirc(ra_center, dec_center, radius, expmjd=expmjd) # Begin iteratively acquiring data gal_count = 0 while ic is not None: gal_loop = len(ic.dataArray['raJ2000']) gal_count += gal_loop objId = numpy.concatenate((objId, ic.dataArray['objId']), axis=0) ra = numpy.concatenate((ra, ic.dataArray['raJ2000']*rad2deg), axis=0) dec = numpy.concatenate((dec, ic.dataArray['decJ2000']*rad2deg), axis=0) diskFluxNorm = numpy.concatenate((diskFluxNorm, ic.dataArray['diskFluxNorm']), axis=0) bulgeFluxNorm = numpy.concatenate((bulgeFluxNorm, ic.dataArray['bulgeFluxNorm']), axis=0) diskSedFilename = numpy.concatenate((diskSedFilename, ic.dataArray['diskSedFilename']), axis=0) bulgeSedFilename = numpy.concatenate((bulgeSedFilename, ic.dataArray['bulgeSedFilename']), axis=0) a_disk = numpy.concatenate((a_disk, ic.dataArray['semiMajorDisk']), axis=0) b_disk = numpy.concatenate((b_disk, ic.dataArray['semiMinorDisk']), axis=0) a_bulge = numpy.concatenate((a_bulge, ic.dataArray['semiMajorBulge']), axis=0) b_bulge = numpy.concatenate((b_bulge, ic.dataArray['semiMinorBulge']), axis=0) pa_d = numpy.concatenate((pa_d, ic.dataArray['positionAngleDisk']), axis=0) rdshft = numpy.concatenate((rdshft, ic.dataArray['redshift']), axis=0) # Get next chunk, if it exists. ic = myqdb.getNextChunk() myqdb.closeSession() # Calculate galactic coordinates: gLon = [] gLat = [] for i in range(gal_count): gcoord = afwCoord.IcrsCoord(ra[i], dec[i]).toGalactic() gLon.append(gcoord.getL(afwCoord.DEGREES)*math.pi/180.) gLat.append(gcoord.getB(afwCoord.DEGREES)*math.pi/180.) ebv_mw = (EBV.calculateEbv(gLon, gLat, ebvMapNorth, ebvMapSouth, interp = True)) del gLon del gLat print 'gLon, gLat calculated' # Now calculate magnitudes for each galaxy. If you have a bulge, initiate its Sed # instance, multiply fnu by the bulge flux normalization and apply the bulge dust # model (currently zero). Next initiate a Sed instance for the disk (if you have # one), multiply fnu by the disk flux normalization and apply the disk dust model. # If you have bulge and disk Sed's, add them together; if not you'll just use # whichever one you have. Correct for redshift, resample the Sed since now it's # shifted in wavelength, add the Milky Way dust and calculate magnitudes using the # manyMag method. Uncorrected Magnitudes (no dust, reddening, redshifting added) # are calculated as well. uncmags = numpy.zeros(gal_count, dtype={'names':['u','g','r','i','z','y'], 'formats':Sixflts}) raEdge = numpy.zeros(len(ra)) decEdge = numpy.zeros(len(ra)) option = numpy.zeros(len(ra)) + 5 raEdge = ra - a_disk/3600.*math.sin((pa_d)*math.pi/180.) decEdge = dec + a_disk/3600.*math.cos((pa_d)*math.pi/180.) # Uncorrected Magnitudes (no dust, reddening, redshifting added) for i in range(gal_count): galdisk = diskSedFilename[i] galbulge = bulgeSedFilename[i] #raEdge = ra[i] - a_disk[i]/3600.*math.sin((pa_d[i])*math.pi/180.) #decEdge = dec[i] + a_disk[i]/3600.*math.cos((pa_d[i])*math.pi/180.) if (galbulge is not None) and (galdisk is not None): ba_disk = b_disk[i]/a_disk[i] ba_bulge = b_bulge[i]/a_bulge[i] baRatio = (diskFluxNorm[i]*ba_disk + bulgeFluxNorm[i] *ba_bulge)/(diskFluxNorm[i] + bulgeFluxNorm[i]) if baMin <= baRatio <= baMax: option[i] = 2 else: continue tmpbulge = uncMagCalc(rdshft[i], gals[galbulge].wavelen, gals[galbulge].flambda, multiFlux=bulgeFluxNorm[i]) tmpdisk = uncMagCalc(rdshft[i], gals[galdisk].wavelen, gals[galdisk].flambda, multiFlux=diskFluxNorm[i]) newgal = uncMagCalc(rdshft[i], tmpdisk.wavelen, (tmpdisk.flambda+tmpbulge.flambda), finish=1) tmpmags = newgal.manyMagCalc(phiarray, dlambda) elif galbulge is not None: baRatio = b_disk[i]/a_disk[i] if baMin <= baRatio <= baMax: option[i] = 0 else: continue tmpbulge = uncMagCalc(rdshft[i], gals[galbulge].wavelen, gals[galbulge].flambda, multiFlux=bulgeFluxNorm[i], finish=1) tmpmags = tmpbulge.manyMagCalc(phiarray, dlambda) elif galdisk is not None: baRatio = b_disk[i]/a_disk[i] if baMin <= baRatio <= baMax: option[i] = 1 else: continue tmpdisk = uncMagCalc(rdshft[i], gals[galdisk].wavelen, gals[galdisk].flambda, multiFlux=diskFluxNorm[i], finish=1) tmpmags = tmpdisk.manyMagCalc(phiarray, dlambda) j = 0 for f in filterlist: uncmags[f][i] = tmpmags[j] j += 1 del diskSedFilename, bulgeSedFilename, tmpmags del Av_d, Rv_d, Av_b, Rv_b, rdshft, uncmags, raEdge, decEdge idx = numpy.where((tmpmags[2] <= rMagMax) & (option != 5)) rmags = tmpmags[2][idx] objId = ojbId[idx] option.append(option) ra = ra[idx] dec = dec[idx] baRatio = baRatio[idx] a_disk = a_disk[idx] b_disk = b_disk[idx] a_bulge = a_bulge[idx] b_bulge = b_bulge[idx] diskFluxNorm = diskFluxNorm[idx] bulgeFluxNorm = bulgeFluxNorm[idx] pa_d = pa_d[idx] raEdge = raEdge[idx] decEdge = decEdge[idx] #print '%11.7f %11.7f %11.7f %11.7f' % (ra[i], dec[i], tmpmags[2], baRatio) print '# iterations: ', iteration + 1 print 'Total # of galaxies: %d' % (gal_count) print 'Number of qualifying Galaxies within a radius of %s deg. is %s' % (radius, len(rmags)) #print 'len(ra), len(rmags), len(baRatio), len(option): ', len(ra), len(rmags), len(baRatio), len(option) return ra, dec, rmags, baRatio, option, objId, a_disk, b_disk, a_bulge, b_bulge, diskFluxNorm, bulgeFluxNorm, pa_d, raEdge, decEdge
#!/usr/bin/env python import numpy, math, copy, cPickle, os, sys from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox import lsst.sims.catalogs.measures.utils as mUtils csize = 10 myqdb = queryDB.queryDB(chunksize=csize, objtype=sys.argv[1]) ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.calculateUnrefractedAltAz() print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) curMD.writeMetadata('testMetadata.dat', 'TRIM', myqdb.opsim, newfile=True) cPickle.dump(ic, open('test.pkl', 'w')) #ic.makeHelio() ic.makeTrimCoords() ic.metadata.validateMetadata('TRIM', myqdb.opsim) print ic.dataArray ic.validateData('TRIM') ic.writeCatalogData('iCTest.txt', 'TRIM') ic.metadata.writeMetadata('iCTest.meta','TRIM', myqdb.opsim, newfile = True)
#!/usr/bin/env python import numpy import math import copy import cPickle from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox import lsst.sims.catalogs.measures.utils as mUtils csize = 10 myqdb = queryDB.queryDB(chunksize=csize, objtype="ALLSTARS") ic = myqdb.getInstanceCatalogById(85748128) print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) mUtils.trimGeneration.derivedTrimMetadata(ic) print ic.metadata.parameters cPickle.dump(ic, open('test.pkl', 'w')) ic.makeHelio() ic.makeTrimCoords() ic.metadata.validateMetadata('TRIM', myqdb.opsim) print ic.dataArray ic.validateData('TRIM') ic.writeCatalogData('iCTest.txt', 'TRIM') ic.metadata.writeMetadata('iCTest.meta', 'TRIM', myqdb.opsim, newfile=True)
def generateRaDec(n): RA = 360.*rand.random(n) Dec = numpy.arcsin(2.*rand.random(n)-1.)*180./numpy.pi return RA, Dec if __name__ == "__main__": csize = 100000 nsamp = 10 magcut = 21.5 boxsizedeg = 0.2 halfboxsizedeg = boxsizedeg/2. sqarea = (math.sin(halfboxsizedeg*math.pi/180.) - math.sin(-halfboxsizedeg*math.pi/180.))*(boxsizedeg*math.pi/180.) radius = math.acos(1-(sqarea/(2.*math.pi)))*180./math.pi ra,dec = generateRaDec(nsamp) fhout = open(sys.argv[1], "w") for i in xrange(nsamp): nums = {'u':0,'g':0,'r':0,'i':0,'z':0,'y':0} expmjd = 0. myqdb = queryDB.queryDB(chunksize=csize,objtype='CALIBSTARS',filetypes=('CALIB',)) ic = myqdb.getInstanceCatalogByCirc(ra[i],dec[i],radius,expmjd=expmjd) while not ic is None: for f in nums.keys(): nums[f] += len(numpy.where(ic.dataArray[f] < magcut)[0]) ic = myqdb.getNextChunk() fhout.write(",".join([str(x) for x in\ (ra[i],dec[i],sqarea,nums['u'],nums['g'],nums['r'],nums['i'],nums['z'],nums['y'])])+"\n") fhout.flush() fhout.close()
#!/usr/bin/env python import numpy, math, copy, cPickle, os, sys from lsst.sims.catalogs.generation.db import queryDB from lsst.sims.catalogs.measures.astrometry import Bbox import lsst.sims.catalogs.measures.utils as mUtils csize = 10 myQDB = queryDB.queryDB(chunksize=csize, objtype='ALLSTARS') ic = myQDB.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.makeTrimCoords() print ic.metadata.parameters curMD = copy.deepcopy(ic.metadata) myQDB = queryDB.queryDB(chunksize=csize, objtype='GALAXY_DISK') ic = myQDB.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.makeTrimCoords() print ic.metadata.parameters curMD.mergeMetadata(ic.metadata) myQDB = queryDB.queryDB(chunksize=csize, objtype='GALAXY_BULGE') ic = myQDB.getInstanceCatalogById(85748128) print ic.metadata.parameters mUtils.trimGeneration.derivedTrimMetadata(ic) ic.makeTrimCoords() print ic.metadata.parameters curMD.mergeMetadata(ic.metadata)
def runCalib(csize, obsid, radius=2.1, outdir='.', repodir=None, je=None, compress=True, cleanup=False): if repodir is None: repodir = outdir meta = None opsimid = None files = [] writeJobEvent(je, 'start') cattype = "CALIB" print "Warning -- only MSSTARS in calib cats currently" #objtypes = ['MSSTARS','WDSTARS','BHBSTARS','RRLYSTARS', \ # 'GLENS','IMAGE','EBSTARS','CEPHEIDSTARS','EASTEREGGS','GALAXY_BULGE','GALAXY_DISK','AGN'] #varobj = ['MSSTARS', 'RRLYSTARS', 'AGN', 'IMAGE', 'WDSTARS', 'EBSTARS', 'CEPHEIDSTARS'] objtypes = [ 'MSSTARS', ] varobj = [ 'MSSTARS', ] warnings.simplefilter('ignore', category=exceptions.UserWarning) arcroot = "obsid%i" % (obsid) outBase = os.path.join(outdir, arcroot) if cleanup: cleanUpDirs(outBase, je) subdir = "pops" popsPath = os.path.join(outBase, subdir) if not os.path.exists(popsPath): os.makedirs(popsPath) writeJobEvent(je, 'MakeDirs', 'Made output directories %s' % (popsPath)) files = [] for objtype in objtypes: writeJobEvent(je, 'Object:%s' % (objtype), 'Doing %s out of: %s' % (objtype, ",".join(objtypes))) filename = "trim_%i_%s.dat" % (obsid, objtype) outfile = os.path.join(popsPath, filename) myqdb = queryDB.queryDB(chunksize=csize, objtype=objtype) # This grabs the metadata and grabs the catalog in a circle around central pointing. # WHY COMBINED? DO WE NEED TO SPLIT TO PROVIDE ADDITIONAL METADATA EASILY? ic = myqdb.getInstanceCatalogById(obsid, radiusdeg=radius) if opsimid is None: opsimid = myqdb.opsim cnum = 0 while ic is not None: writeJobEvent( je, 'GetChunk', 'Got chunk #%i of length %i' % (cnum, len(ic.dataArray[ic.dataArray.keys()[0]]))) # Calculate focal plane mm/mm and chip x/y for each object. ic.makeXyCoords() writeJobEvent(je, 'MakeXY', 'Made xy coords for chunk #%i' % (cnum)) # Calculate calib counts for each object. ic.calcCalibCounts() writeJobEvent( je, 'calcCalibCounts', 'Generated throughput curve and calculated counts for chunk #%i' % (cnum)) if cnum == 0: if compress: files.append(os.path.join(subdir, filename) + ".gz") else: files.append(os.path.join(subdir, filename)) # Okay, need to understand this a little more ... think it's mostly formatting/writing metadata. # NEED TO WRITE A NEW VERSION? mUtils.trimGeneration.derivedTrimMetadata(ic) if meta is None: meta = deepcopy(ic.metadata) else: meta.mergeMetadata(ic.metadata) if objtype in varobj: writeJobEvent( je, 'DoVar', 'Applying variability to chunk #%i of type %s' % (cnum, objtype)) ic.applyVariability() # WILL NEED SOMETHING HERE FOR CALIB CAT ic.validateData('TRIM') numRec = len(ic.dataArray[ic.dataArray.keys()[0]]) if cnum == 0: ic.writeCatalogData(outfile, "TRIM", newfile=True, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote first chunk of length %i' % (numRec)) else: ic.writeCatalogData(outfile, "TRIM", newfile=False, compress=compress) writeJobEvent(je, 'WriteChunk', 'Wrote chunk #%i of length %i' % (cnum, numRec)) if numRec == csize: ic = myqdb.getNextChunk() else: ic = None cnum += 1 writeJobEvent(je, 'Finished Object:%s' % (objtype), 'Finished object %s' % (objtype)) meta.validateMetadata(cattype, opsimid) metaOutfile = os.path.join(outBase, "metadata_%i.dat" % (obsid)) meta.writeMetadata(metaOutfile, cattype, opsimid, newfile=True, filelist=files, compress=False) #files.append(os.path.join("obsid%i"%(obsid),"metadata_%i.dat"%(obsid))) writeJobEvent(je, 'WriteMetadata', 'Wrote metadata to %s' % (metaOutfile)) mvFiles(repodir, outBase, arcroot, je=je) if cleanup: cleanUpDirs(outBase, je) writeJobEvent(je, 'stop')