def testDbObj(self): mystars = CatalogDBObject.from_objid('testCatalogDBObjectTeststars') mygals = CatalogDBObject.from_objid('testCatalogDBObjectTestgals') result = mystars.query_columns(obs_metadata=self.obsMd) tu.writeResult(result, "/dev/null") result = mygals.query_columns(obs_metadata=self.obsMd) tu.writeResult(result, "/dev/null")
def examplePhoSimCatalogs(): """ This method outputs several phoSim input files consisting of different types of objects(stars, galaxy bulges, galaxy disks, and AGNs) The files created are catalog_test_msstars.dat catalog_test_galaxyDisk.dat catalog_test_galaxyBulge.dat catalog_test_galaxyAgn.dat (versions are also created that end in _chunked.dat; these should have the same contents) """ obsMD = bcm.OpSim3_61DBObject() obs_metadata = obsMD.getObservationMetaData(88544919, 0.1, makeCircBounds=True) objectDict = {} objectDict['testStars'] = {'dbobj':CatalogDBObject.from_objid('msstars'), 'constraint':None, 'filetype':'phoSim_catalog_POINT', 'obsMetadata':obs_metadata} objectDict['testGalaxyBulge'] = {'dbobj':CatalogDBObject.from_objid('galaxyBulge'), 'constraint':"mass_bulge > 1. and sedname_bulge is not NULL", 'filetype':'phoSim_catalog_SERSIC2D', 'obsMetadata':obs_metadata} objectDict['testGalaxyDisk'] = {'dbobj':CatalogDBObject.from_objid('galaxyDisk'), 'constraint':"DiskLSSTg < 20. and sedname_disk is not NULL", 'filetype':'phoSim_catalog_SERSIC2D', 'obsMetadata':obs_metadata} objectDict['testGalaxyAgn'] = {'dbobj':CatalogDBObject.from_objid('galaxyAgn'), 'constraint':"sedname_agn is not NULL", 'filetype':'phoSim_catalog_ZPOINT', 'obsMetadata':obs_metadata} for objKey in objectDict.keys(): dbobj = objectDict[objKey]['dbobj'] t = dbobj.getCatalog(objectDict[objKey]['filetype'], obs_metadata=objectDict[objKey]['obsMetadata'], constraint=objectDict[objKey]['constraint']) print print "These are the required columns from the database:" print t.db_required_columns() print print "These are the columns that will be output to the file:" print t.column_outputs print filename = 'catalog_test_%s.dat'%(dbobj.objid) print "querying and writing catalog to %s:" % filename t.write_catalog(filename) filename = 'catalog_test_%s_chunked.dat'%(dbobj.objid) t.write_catalog(filename, chunk_size=10) print " - finished"
def testCanBeNull(self): """ Test to make sure that we can still write all rows to catalogs, even those with null values in key columns """ dbobj = CatalogDBObject.from_objid('cannotBeNull') cat = dbobj.getCatalog('canBeNull') fileName = 'canBeNullTestFile.txt' cat.write_catalog(fileName) dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64), ('n4',(str,40)), ('n5',(unicode,40))]) testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',') for i in range(len(self.baselineOutput)): #make sure that all of the rows in self.baselineOutput are represented in #testData for (k,xx) in enumerate(self.baselineOutput[i]): if k<4: if not numpy.isnan(xx): self.assertAlmostEqual(xx,testData[i][k], 3) else: self.assertTrue(numpy.isnan(testData[i][k])) else: msg = '%s is not %s' % (xx,testData[i][k]) self.assertEqual(xx.strip(),testData[i][k].strip(),msg=msg) self.assertEqual(i,99) self.assertEqual(len(testData), len(self.baselineOutput)) if os.path.exists(fileName): os.unlink(fileName)
def testCanBeNull(self): """ Test to make sure that we can still write all rows to catalogs, even those with null values in key columns """ baselineOutput = createCannotBeNullTestDB() dbobj = CatalogDBObject.from_objid('cannotBeNull') cat = dbobj.getCatalog('canBeNull') fileName = 'canBeNullTestFile.txt' cat.write_catalog(fileName) dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64)]) testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',') for i in range(len(baselineOutput)): if not numpy.isnan(baselineOutput['n2'][i]): for (k,xx) in enumerate(baselineOutput[i]): if not numpy.isnan(xx): self.assertAlmostEqual(xx,testData[i][k],3) else: self.assertTrue(numpy.isnan(testData[i][k])) self.assertEqual(i,99) if os.path.exists(fileName): os.unlink(fileName)
def testCanBeNull(self): """ Test to make sure that we can still write all rows to catalogs, even those with null values in key columns """ dbobj = CatalogDBObject.from_objid('cannotBeNull') cat = dbobj.getCatalog('canBeNull') fileName = 'canBeNullTestFile.txt' cat.write_catalog(fileName) dtype = numpy.dtype([('id', int), ('n1', numpy.float64), ('n2', numpy.float64), ('n3', numpy.float64), ('n4', (str, 40)), ('n5', (unicode, 40))]) testData = numpy.genfromtxt(fileName, dtype=dtype, delimiter=',') for i in range(len(self.baselineOutput)): #make sure that all of the rows in self.baselineOutput are represented in #testData for (k, xx) in enumerate(self.baselineOutput[i]): if k < 4: if not numpy.isnan(xx): self.assertAlmostEqual(xx, testData[i][k], 3) else: self.assertTrue(numpy.isnan(testData[i][k])) else: msg = '%s is not %s' % (xx, testData[i][k]) self.assertEqual(xx.strip(), testData[i][k].strip(), msg=msg) self.assertEqual(i, 99) self.assertEqual(len(testData), len(self.baselineOutput)) if os.path.exists(fileName): os.unlink(fileName)
def exampleAirmass(airmass,ra = 0.0, dec = 0.0, tol = 10.0, radiusDeg = 0.1, makeBoxBounds=False, makeCircBounds=True): """ This method will output a catalog of stars based on an OpSim pointing with a specific airmass. It searches OpSim for pointings with the specified airmass and RA, Dec within a box bounded by tol (in degrees). It creates observation meta data out of the first pointing found and uses that to construct the catalog. The catalog is output to stars_airmass_test.dat """ obsMD=bcm.OpSim3_61DBObject() #The code below will query the OpSim data base object created above. #The query will be based on a box in RA, Dec and a specific airmass value airmassConstraint = "airmass="+str(airmass) #an SQL constraint that the airmass must be equal to #the passed value skyBounds = SpatialBounds.getSpatialBounds('box', ra, dec, tol) #bounds on region of sky query = obsMD.executeConstrainedQuery(skyBounds, constraint=airmassConstraint) #convert q into observation meta data for use in a catalog obsMetaData = obsMD.getObservationMetaData(query['Opsim_obshistid'][0],radiusDeg,makeBoxBounds=makeBoxBounds, makeCircBounds=makeCircBounds) #create and output a reference catalog of stars based on our query to opSim dbobj = CatalogDBObject.from_objid('allstars') catalog = dbobj.getCatalog('ref_catalog_star', obs_metadata = obsMetaData) catalog.write_catalog('stars_airmass_test.dat')
def testNonsenseSelectOnlySomeColumns(self): """ Test a query performed only a subset of the available columns """ myNonsense = CatalogDBObject.from_objid('Nonsense') mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseMag'] query = myNonsense.query_columns(colnames=mycolumns, constraint = 'ra < 45.', chunk_size=100) goodPoints = [] for chunk in query: for row in chunk: self.assertTrue(row[1]<45.0) dex = numpy.where(self.baselineData['id'] == row[0])[0][0] goodPoints.append(row[0]) self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3) self.assertAlmostEqual(self.baselineData['mag'][dex],row[2],3) for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]: self.assertTrue(entry[1]>45.0)
def testHybridVariability(self): """ Test that we can generate a catalog which inherits from multiple variability mixins (in this case, TestVariability and VariabilityStars). This is to make sure that the register_method and register_class decorators do not mangle inheritance of methods from mixins. """ makeHybridTable() myDB = CatalogDBObject.from_objid('hybridTest') myCatalog = myDB.getCatalog('testVariabilityCatalog', obs_metadata=self.obs_metadata) myCatalog.write_catalog('hybridTestCatalog.dat', chunk_size=1000) if os.path.exists('hybridTestCatalog.dat'): os.unlink('hybridTestCatalog.dat') # make sure order of mixin inheritance does not matter myCatalog = myDB.getCatalog('otherVariabilityCatalog', obs_metadata=self.obs_metadata) myCatalog.write_catalog('hybridTestCatalog.dat', chunk_size=1000) if os.path.exists('hybridTestCatalog.dat'): os.unlink('hybridTestCatalog.dat') # make sure that, if a catalog does not contain a variability method, # an error is thrown; verify that it contains the correct error message myCatalog = myDB.getCatalog('stellarVariabilityCatalog', obs_metadata=self.obs_metadata) with self.assertRaises(RuntimeError) as context: myCatalog.write_catalog('hybridTestCatalog.dat') expectedMessage = "Your InstanceCatalog does not contain a variability method" expectedMessage += " corresponding to 'testVar'" self.assertTrue(context.exception.message==expectedMessage) if os.path.exists('hybridTestCatalog.dat'): os.unlink('hybridTestCatalog.dat')
def testEb(self): makeEbTable() myDB = CatalogDBObject.from_objid('ebTest') myCatalog = myDB.getCatalog('stellarVariabilityCatalog', obs_metadata=self.obs_metadata) myCatalog.write_catalog('ebTestCatalog.dat', chunk_size=1000) if os.path.exists('ebTestCatalog.dat'): os.unlink('ebTestCatalog.dat')
def testCannotBeNull(self): """ Test to make sure that the code for filtering out rows with null values in key rowss works. """ #each of these classes flags a different column with a different datatype as cannot_be_null availableCatalogs = [floatCannotBeNullCatalog, strCannotBeNullCatalog, unicodeCannotBeNullCatalog] dbobj = CatalogDBObject.from_objid('cannotBeNull') for catClass in availableCatalogs: cat = catClass(dbobj) fileName = 'cannotBeNullTestFile.txt' cat.write_catalog(fileName) dtype = numpy.dtype([('id',int),('n1',numpy.float64),('n2',numpy.float64),('n3',numpy.float64), ('n4',(str,40)), ('n5',(unicode,40))]) testData = numpy.genfromtxt(fileName,dtype=dtype,delimiter=',') j = 0 #a counter to keep track of the rows read in from the catalog for i in range(len(self.baselineOutput)): #self.baselineOutput contains all of the rows from the dbobj #first, we must assess whether or not the row we are currently #testing would, in fact, pass the cannot_be_null test validLine = True if isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],str) or \ isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],unicode): if self.baselineOutput[cat.cannot_be_null[0]][i].strip().lower() == 'none': validLine = False else: if numpy.isnan(self.baselineOutput[cat.cannot_be_null[0]][i]): validLine = False if validLine: #if the row in self.baslineOutput should be in the catalog, we now check #that baseline and testData agree on column values (there are some gymnastics #here because you cannot do an == on NaN's for (k,xx) in enumerate(self.baselineOutput[i]): if k<4: if not numpy.isnan(xx): msg = 'k: %d -- %s %s -- %s' % (k,str(xx),str(testData[j][k]),cat.cannot_be_null) self.assertAlmostEqual(xx, testData[j][k],3, msg=msg) else: self.assertTrue(numpy.isnan(testData[j][k])) else: msg = '%s (%s) is not %s (%s)' % (xx,type(xx),testData[j][k],type(testData[j][k])) self.assertEqual(xx.strip(),testData[j][k].strip(), msg=msg) j+=1 self.assertEqual(i,99) #make sure that we tested all of the baseline rows self.assertEqual(j,len(testData)) #make sure that we tested all of the testData rows msg = '%d >= %d' % (j,i) self.assertTrue(j<i, msg=msg) #make sure that some rows did not make it into the catalog if os.path.exists(fileName): os.unlink(fileName)
def testAgn(self): makeAgnTable() myDB = CatalogDBObject.from_objid('agnTest') myCatalog = myDB.getCatalog('galaxyVariabilityCatalog', obs_metadata=self.obs_metadata) myCatalog.write_catalog('agnTestCatalog.dat', chunk_size=1000) if os.path.exists('agnTestCatalog.dat'): os.unlink('agnTestCatalog.dat')
def __init__(self, objid, phosimCatalogObject): while True: # This loop is a workaround for UW catsim db connection intermittency. try: self.db_obj = CatalogDBObject.from_objid(objid) break except RuntimeError: continue self.cat_obj = phosimCatalogObject
def __init__(self, objid, phosimCatalogObject): while True: # This loop is a workaround for UW catsim db connection intermittency. try: self.db_obj = CatalogDBObject.from_objid(objid) break except RuntimeError: continue self.cat_obj = phosimCatalogObject
def testArbitraryQuery(self): """ Test method to directly execute an arbitrary SQL query (inherited from DBObject class) """ myNonsense = CatalogDBObject.from_objid('Nonsense') query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id' results = myNonsense.execute_arbitrary(query) self.assertEqual(len(results),1250) for row in results: self.assertEqual(row[0],row[2]) self.assertAlmostEqual(row[1],0.5*row[3],6)
def testNonsenseArbitraryConstraints(self): """ Test a query with a user-specified constraint on the magnitude column """ myNonsense = CatalogDBObject.from_objid('Nonsense') raMin = 50.0 raMax = 150.0 decMax = 30.0 decMin = -20.0 raCenter=0.5*(raMin+raMax) decCenter=0.5*(decMin+decMax) mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseDecJ2000','NonsenseMag'] boxObsMd = ObservationMetaData(boundType='box',unrefractedRA=raCenter,unrefractedDec=decCenter, boundLength=numpy.array([0.5*(raMax-raMin),0.5*(decMax-decMin)]), mjd=52000.,bandpassName='r') boxQuery = myNonsense.query_columns(colnames = mycolumns, obs_metadata=boxObsMd, chunk_size=100, constraint = 'mag > 11.0') raMin = numpy.radians(raMin) raMax = numpy.radians(raMax) decMin = numpy.radians(decMin) decMax = numpy.radians(decMax) goodPoints = [] for chunk in boxQuery: for row in chunk: self.assertTrue(row[1]<raMax) self.assertTrue(row[1]>raMin) self.assertTrue(row[2]<decMax) self.assertTrue(row[2]>decMin) self.assertTrue(row[3]>11.0) dex = numpy.where(self.baselineData['id'] == row[0])[0][0] #keep a list of the points returned by the query goodPoints.append(row[0]) self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3) self.assertAlmostEqual(numpy.radians(self.baselineData['dec'][dex]),row[2],3) self.assertAlmostEqual(self.baselineData['mag'][dex],row[3],3) for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]: #make sure that the points not returned by the query did, in fact, violate one of the #constraints of the query (either the box bound or the magnitude cut off) switch = (entry[1] > raMax or entry[1] < raMin or entry[2] >decMax or entry[2] < decMin or entry[3]<11.0) self.assertTrue(switch)
def testNonsenseBoxConstraints(self): """ Test that a query performed on a box bound gets all of the points (and only all of the points) inside that box bound. """ myNonsense = CatalogDBObject.from_objid('Nonsense') raMin = 50.0 raMax = 150.0 decMax = 30.0 decMin = -20.0 raCenter = 0.5*(raMin+raMax) decCenter = 0.5*(decMin+decMax) mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseDecJ2000','NonsenseMag'] boxObsMd = ObservationMetaData(boundType='box',unrefractedDec=decCenter, unrefractedRA=raCenter, boundLength=numpy.array([0.5*(raMax-raMin),0.5*(decMax-decMin)]),mjd=52000.,bandpassName='r') boxQuery = myNonsense.query_columns(obs_metadata=boxObsMd, chunk_size=100, colnames=mycolumns) raMin = numpy.radians(raMin) raMax = numpy.radians(raMax) decMin = numpy.radians(decMin) decMax = numpy.radians(decMax) goodPoints = [] for chunk in boxQuery: for row in chunk: self.assertTrue(row[1]<raMax) self.assertTrue(row[1]>raMin) self.assertTrue(row[2]<decMax) self.assertTrue(row[2]>decMin) dex = numpy.where(self.baselineData['id'] == row[0])[0][0] #keep a list of which points were returned by teh query goodPoints.append(row[0]) self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3) self.assertAlmostEqual(numpy.radians(self.baselineData['dec'][dex]),row[2],3) self.assertAlmostEqual(self.baselineData['mag'][dex],row[3],3) for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]: #make sure that the points not returned by the query are, in fact, outside of the #box bound switch = (entry[1] > raMax or entry[1] < raMin or entry[2] >decMax or entry[2] < decMin) self.assertTrue(switch)
def testRealQueryConstraints(self): mystars = CatalogDBObject.from_objid('testCatalogDBObjectTeststars') mycolumns = ['id','raJ2000','decJ2000','umag','gmag','rmag','imag','zmag','ymag'] #recall that ra and dec are stored in degrees in the data base myquery = mystars.query_columns(colnames = mycolumns, constraint = 'ra < 90. and ra > 45.') tol=1.0e-3 for chunk in myquery: for star in chunk: self.assertTrue(numpy.degrees(star[1])<90.0+tol) self.assertTrue(numpy.degrees(star[1])>45.0-tol)
def testChunking(self): """ Test that a query with a specified chunk_size does, in fact, return chunks of that size """ mystars = CatalogDBObject.from_objid('testCatalogDBObjectTeststars') mycolumns = ['id','raJ2000','decJ2000','umag','gmag'] myquery = mystars.query_columns(colnames = mycolumns, chunk_size = 1000) for chunk in myquery: self.assertEqual(chunk.size,1000) for row in chunk: self.assertTrue(len(row),5)
def testArbitraryChunkIterator(self): """ Test method to create a ChunkIterator from an arbitrary SQL query (inherited from DBObject class) """ myNonsense = CatalogDBObject.from_objid('Nonsense') query = 'SELECT test.id, test.mag, test2.id, test2.mag FROM test, test2 WHERE test.id=test2.id' dtype = numpy.dtype([('id1',int),('mag1',float),('id2',int),('mag2',float)]) results = myNonsense.get_chunk_iterator(query,chunk_size=100,dtype=dtype) i = 0 for chunk in results: for row in chunk: self.assertEqual(row[0],row[2]) self.assertAlmostEqual(row[1],0.5*row[3],6) i += 1 self.assertEqual(i,1250)
def testAmcvn(self): #Note: this test assumes that the parameters for the Amcvn variability #model occur in a standard varParamStr column in the database. #Actually, the current database of Amcvn events simply store the variability #parameters as independent columns in the database. #The varParamStr formalism is how the applyAmcvn method is written, however, #so that is what we will test. makeAmcvnTable() myDB = CatalogDBObject.from_objid('amcvnTest') myCatalog = myDB.getCatalog('stellarVariabilityCatalog', obs_metadata=self.obs_metadata) myCatalog.write_catalog('amcvnTestCatalog.dat', chunk_size=1000) if os.path.exists('amcvnTestCatalog.dat'): os.unlink('amcvnTestCatalog.dat')
def testNonsenseCircularConstraints(self): """ Test that a query performed on a circle bound gets all of the objects (and only all of the objects) within that circle """ myNonsense = CatalogDBObject.from_objid('Nonsense') radius = 20.0 raCenter = 210.0 decCenter = -60.0 mycolumns = ['NonsenseId','NonsenseRaJ2000','NonsenseDecJ2000','NonsenseMag'] circObsMd = ObservationMetaData(boundType='circle', unrefractedRA=raCenter,unrefractedDec=decCenter, boundLength=radius, mjd=52000., bandpassName='r') circQuery = myNonsense.query_columns(colnames = mycolumns, obs_metadata=circObsMd, chunk_size=100) raCenter = numpy.radians(raCenter) decCenter = numpy.radians(decCenter) radius = numpy.radians(radius) goodPoints = [] for chunk in circQuery: for row in chunk: distance = haversine(raCenter,decCenter,row[1],row[2]) self.assertTrue(distance<radius) dex = numpy.where(self.baselineData['id'] == row[0])[0][0] #store a list of which objects fell within our circle bound goodPoints.append(row[0]) self.assertAlmostEqual(numpy.radians(self.baselineData['ra'][dex]),row[1],3) self.assertAlmostEqual(numpy.radians(self.baselineData['dec'][dex]),row[2],3) self.assertAlmostEqual(self.baselineData['mag'][dex],row[3],3) for entry in [xx for xx in self.baselineData if xx[0] not in goodPoints]: #make sure that all of the points not returned by the query were, in fact, outside of #the circle bound distance = haversine(raCenter,decCenter,numpy.radians(entry[1]),numpy.radians(entry[2])) self.assertTrue(distance>radius)
def testObsCat(self): objname = 'wdstars' dbobj = CatalogDBObject.from_objid(objname) obs_metadata = dbobj.testObservationMetaData # To cover the central ~raft obs_metadata.boundLength = 0.4 opsMetadata = {'Opsim_rotskypos':(0., float), 'Unrefracted_RA':(numpy.radians(obs_metadata.unrefractedRA), float), 'Unrefracted_Dec':(numpy.radians(obs_metadata.unrefractedDec), float)} obs_metadata.phoSimMetaData = opsMetadata cat = dbobj.getCatalog('obs_star_cat', obs_metadata) if os.path.exists('testCat.out'): os.unlink('testCat.out') try: cat.write_catalog('testCat.out') finally: if os.path.exists('testCat.out'): os.unlink('testCat.out')
def exampleReferenceCatalog(): """ This method outputs a reference catalog of galaxies (i.e. a catalog of galaxies in which the columns are simply the data stored in the database). The catalog class is defined in python/lsst/sims/catUtils/exampleCatalogDefinitions/refCatalogExamples.py The catalog is output to the file test_reference.dat """ obs_metadata = ObservationMetaData(boundType='circle',unrefractedRA=0.0,unrefractedDec=0.0, boundLength=0.01) dbobj = CatalogDBObject.from_objid('galaxyBase') t = dbobj.getCatalog('ref_catalog_galaxy', obs_metadata=obs_metadata) filename = 'test_reference.dat' t.write_catalog(filename, chunk_size=10)
def testObsCat(self): objname = 'wdstars' catName = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace', 'testObsCat.txt') try: dbobj = CatalogDBObject.from_objid(objname) obs_metadata = dbobj.testObservationMetaData # To cover the central ~raft obs_metadata.boundLength = 0.4 opsMetadata = {'Opsim_rotskypos':(0., float), 'pointingRA':(numpy.radians(obs_metadata.pointingRA), float), 'pointingDec':(numpy.radians(obs_metadata.pointingDec), float)} obs_metadata.phoSimMetaData = opsMetadata cat = dbobj.getCatalog('obs_star_cat', obs_metadata) if os.path.exists(catName): os.unlink(catName) try: cat.write_catalog(catName) dtypeList = [(name, numpy.float) for name in cat._column_outputs] testData = numpy.genfromtxt(catName, delimiter = ', ', dtype=numpy.dtype(dtypeList)) self.assertGreater(len(testData), 0) finally: if os.path.exists(catName): os.unlink(catName) print '\ntestObsCat successfully connected to fatboy' except: trace = traceback.extract_tb(sys.exc_info()[2], limit=20) msg = sys.exc_info()[1].args[0] if 'Failed to connect' in msg or failedOnFatboy(trace): # if the exception was because of a failed connection # to fatboy, ignore it. print '\ntestObsCat failed to connect to fatboy' print 'Sometimes that happens. Do not worry.' pass else: raise
def setUpClass(cls): # delete previous test db if present if os.path.exists('testData/sncat.db'): print 'deleting previous database' os.unlink('testData/sncat.db') mjds = [570123.15 + 3.*i for i in range(2)] galDB = CatalogDBObject.from_objid('galaxyTiled') for i, myMJD in enumerate(mjds): myObsMD = ObservationMetaData(boundType='circle', boundLength=0.015, unrefractedRA=5.0, unrefractedDec=15.0, bandpassName= ['u', 'g', 'r', 'i', 'z', 'y'], mjd=myMJD) catalog = SNIaCatalog(db_obj=galDB, obs_metadata=myObsMD) fname = "testData/SNIaCat_" + str(i) + ".txt" print fname, myObsMD.mjd catalog.write_catalog(fname)
def examplePhoSimNoOpSim(): """ This method outputs phoSim input files based on arbitrary input coordinates (rather than an OpSim pointing). catalog_test_stars_rd.dat is a file created from a specified RA, Dec pointing catalog_test_stars_aa.dat is a file created from a specified Alt, Az pointing """ raDeg= 15. decDeg = -30. mjd = 51999.75 #source code for the method below is found in python.lsst.sims.catUtils.observationMetaDataUtils.py # #basically, it returns a dict of data needed by phoSim to specify a given pointing #(ra and dec of the moon, rotation of the sky relative to the telescope, etc.) md = makeObsParamsRaDecTel(math.radians(raDeg), math.radians(decDeg), mjd, 'r') obs_metadata_rd = ObservationMetaData(boundType='circle', boundLength=0.1, mjd=mjd, phoSimMetaData=md) azRad = math.radians(220.) altRad = math.radians(79.) md = makeObsParamsAzAltTel(azRad, altRad, mjd, 'r') raDeg = math.degrees(md['Unrefracted_RA'][0]) decDeg = math.degrees(md['Unrefracted_Dec'][0]) obs_metadata_aa = ObservationMetaData(boundType='circle', boundLength=0.1, mjd=mjd, phoSimMetaData=md) dbobj = CatalogDBObject.from_objid('msstars') t = dbobj.getCatalog('phoSim_catalog_POINT', obs_metadata= obs_metadata_rd) t.write_catalog('catalog_test_stars_rd.dat') t = dbobj.getCatalog('phoSim_catalog_POINT', obs_metadata= obs_metadata_aa) t.write_catalog('catalog_test_stars_aa.dat')
#!/usr/bin/env python # To benchmark an instance Catalog, first look at # the instance of CatalogDBObject import numpy as np from benchmarkInstanceCatalogs import QueryBenchMarks from lsst.sims.catalogs.generation.db import CatalogDBObject import lsst.sims.catUtils.baseCatalogModels as bcm from lsst.sims.catalogs.measures.instance import InstanceCatalog galDB = CatalogDBObject.from_objid('galaxyTiled') # Create a child of the InstanceCatalog Class class galCopy(InstanceCatalog): column_outputs = ['id', 'raJ2000', 'decJ2000', 'redshift'] override_formats = {'raJ2000': '%8e', 'decJ2000': '%8e'} # Sizes to be used for querying boundLens = np.arange(0.05, 1.8, 0.05) # Instantiate a benchmark object opsimDBHDF ='/astro/users/rbiswas/data/LSST/OpSimData/storage.h5' gcb = QueryBenchMarks.fromOpSimDF(instanceCatChild=galCopy, dbObject=galDB, opSimHDF=opsimDBHDF, boundLens=boundLens, constraints='r_ab < 24.0', numSamps=3, name='magneto_test_rless24') # Look at the size print gcb.coords.size print gcb.boundLens.size
return self.parameters[idx] if __name__ == "__main__": print sncosmo.__file__ print SNIa.__class__ galDB = CatalogDBObject.from_objid ('galaxyBase') catalogIterator = galDB.query_columns(colnames=['id','redshift', 'ra', 'dec', 'mass_stellar'], constraint='redshift between 0.01 and 0.1') dtype = None for chunk in catalogIterator: if dtype is None: dtype = chunk.dtype for record in chunk: id = np.int(record['id']) mass = np.float(record['mass_stellar'])*10.0**10.0 ra = np.float(record['ra']) dec = np.float(record['dec']) redshift = np.float(record['redshift']) sn = SNIa( galid=id, snid =id, ra=ra, dec=dec , z=redshift)
#If you want to use the LSST camera, uncomment the line below. #You can similarly assign any camera object you want here #camera = LsstSimMapper().camera #select an OpSim pointing obsMD = OpSim3_61DBObject() obs_metadata = obsMD.getObservationMetaData(88625744, 0.05, makeCircBounds = True) catName = './outputs_cats/galSim_galaxies_example.txt' cstrn = "redshift>"+str(zdn)+" and redshift<="+str(zup) bulges = CatalogDBObject.from_objid('galaxyBulge') bulge_galSim = testGalSimGalaxies(bulges, obs_metadata=obs_metadata, column_outputs=['redshift'],constraint=cstrn) bulge_galSim.noise_and_background = None bulge_galSim.PSF = None bulge_galSim.write_catalog(catName, write_header=True, write_mode='a') print 'done with bulges' disks = CatalogDBObject.from_objid('galaxyDisk') disk_galSim = testGalSimGalaxies(disks, obs_metadata=obs_metadata, column_outputs=['redshift'],constraint=cstrn) disk_galSim.copyGalSimInterpreter(bulge_galSim) disk_galSim.noise_and_background = None disk_galSim.PSF = None disk_galSim.write_catalog(catName, write_header=True, write_mode='a') print 'done with disks'
from lsst.sims.catalogs.generation.db import ObservationMetaData, CatalogDBObject from lsst.sims.catUtils.baseCatalogModels import OpSim3_61DBObject from lsst.sims.catUtils.exampleCatalogDefinitions.phoSimCatalogExamples import \ PhoSimCatalogPoint, PhoSimCatalogSersic2D, PhoSimCatalogZPoint from sprinkler import sprinklerAGN from lsst.sims.catUtils.baseCatalogModels import * starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'] obsMD = OpSim3_61DBObject() obs_metadata = obsMD.getObservationMetaData(88625744, 0.05, makeCircBounds = True) doHeader= True for starName in starObjNames: stars = CatalogDBObject.from_objid(starName) star_phoSim=PhoSimCatalogPoint(stars,obs_metadata=obs_metadata) #the class for phoSim input files #containing point sources if (doHeader): with open("phoSim_example.txt","w") as fh: star_phoSim.write_header(fh) doHeader = False #below, write_header=False prevents the code from overwriting the header just written #write_mode = 'a' allows the code to append the new objects to the output file, rather #than overwriting the file for each different class of object. star_phoSim.write_catalog("phoSim_example.txt",write_mode='a',write_header=False,chunk_size=20000) gals = CatalogDBObject.from_objid('galaxyBulge') #now append a bunch of objects with 2D sersic profiles to our output file
Note that because all angles are handled inside of the stack as radians, the returned angular distance will also be in radians """ r0 = self.column_by_name('raJ2000') d0 = self.column_by_name('decJ2000') r1 = self.column_by_name('raObserved') d1 = self.column_by_name('decObserved') return haversine(r0, d0, r1, d1) #write the catalog directly myDB = CatalogDBObject.from_objid('allstars') obs_metadata = ObservationMetaData(unrefractedRA=220.0, unrefractedDec=19.0, boundType='circle', boundLength=0.1, mjd=52000.0) cat = TutorialCatalog(myDB, obs_metadata=obs_metadata) cat.write_catalog('tutorial_astrometry_photometry.txt') #write the catalog using CatalogDBObject.getCatalog() obs_metadata = ObservationMetaData(unrefractedRA=120.0, unrefractedDec=-5.0, boundType='circle', boundLength=0.1, mjd=52000.0) cat = myDB.getCatalog('tutorial_catalog', obs_metadata=obs_metadata)
from lsst.sims.catalogs.generation.db import CatalogDBObject from lsst.sims.utils import ObservationMetaData # The following is to get the object ids in the registry import lsst.sims.catUtils.baseCatalogModels as bcm from lsst.sims.catUtils.exampleCatalogDefinitions import RefCatalogGalaxyBase if __name__ == "__main__": obs_metadata = ObservationMetaData(boundType="circle", unrefractedRA=0.0, unrefractedDec=0.0, boundLength=0.01) dbobj = CatalogDBObject.from_objid("galaxyBase") t = dbobj.getCatalog("galaxy_photometry_cat", obs_metadata=obs_metadata) filename = "galaxy_photometry_test.dat" t.write_catalog(filename, chunk_size=10000)
from lsst.sims.catUtils.exampleCatalogDefinitions.phoSimCatalogExamples import \ PhoSimCatalogPoint, PhoSimCatalogSersic2D, PhoSimCatalogZPoint from sprinkler import sprinklerAGN from lsst.sims.catUtils.baseCatalogModels import * starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'] obsMD = OpSim3_61DBObject() obs_metadata = obsMD.getObservationMetaData(88625744, 0.05, makeCircBounds=True) doHeader = True for starName in starObjNames: stars = CatalogDBObject.from_objid(starName) star_phoSim = PhoSimCatalogPoint( stars, obs_metadata=obs_metadata) #the class for phoSim input files #containing point sources if (doHeader): with open("phoSim_example.txt", "w") as fh: star_phoSim.write_header(fh) doHeader = False #below, write_header=False prevents the code from overwriting the header just written #write_mode = 'a' allows the code to append the new objects to the output file, rather #than overwriting the file for each different class of object. star_phoSim.write_catalog("phoSim_example.txt", write_mode='a', write_header=False, chunk_size=20000)
def testCannotBeNull(self): """ Test to make sure that the code for filtering out rows with null values in key rowss works. """ #each of these classes flags a different column with a different datatype as cannot_be_null availableCatalogs = [ floatCannotBeNullCatalog, strCannotBeNullCatalog, unicodeCannotBeNullCatalog ] dbobj = CatalogDBObject.from_objid('cannotBeNull') for catClass in availableCatalogs: cat = catClass(dbobj) fileName = 'cannotBeNullTestFile.txt' cat.write_catalog(fileName) dtype = numpy.dtype([('id', int), ('n1', numpy.float64), ('n2', numpy.float64), ('n3', numpy.float64), ('n4', (str, 40)), ('n5', (unicode, 40))]) testData = numpy.genfromtxt(fileName, dtype=dtype, delimiter=',') j = 0 #a counter to keep track of the rows read in from the catalog for i in range(len(self.baselineOutput)): #self.baselineOutput contains all of the rows from the dbobj #first, we must assess whether or not the row we are currently #testing would, in fact, pass the cannot_be_null test validLine = True if isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],str) or \ isinstance(self.baselineOutput[cat.cannot_be_null[0]][i],unicode): if self.baselineOutput[cat.cannot_be_null[0]][i].strip( ).lower() == 'none': validLine = False else: if numpy.isnan( self.baselineOutput[cat.cannot_be_null[0]][i]): validLine = False if validLine: #if the row in self.baslineOutput should be in the catalog, we now check #that baseline and testData agree on column values (there are some gymnastics #here because you cannot do an == on NaN's for (k, xx) in enumerate(self.baselineOutput[i]): if k < 4: if not numpy.isnan(xx): msg = 'k: %d -- %s %s -- %s' % ( k, str(xx), str( testData[j][k]), cat.cannot_be_null) self.assertAlmostEqual(xx, testData[j][k], 3, msg=msg) else: self.assertTrue(numpy.isnan(testData[j][k])) else: msg = '%s (%s) is not %s (%s)' % (xx, type( xx), testData[j][k], type(testData[j][k])) self.assertEqual(xx.strip(), testData[j][k].strip(), msg=msg) j += 1 self.assertEqual( i, 99) #make sure that we tested all of the baseline rows self.assertEqual(j, len( testData)) #make sure that we tested all of the testData rows msg = '%d >= %d' % (j, i) self.assertTrue( j < i, msg=msg ) #make sure that some rows did not make it into the catalog if os.path.exists(fileName): os.unlink(fileName)