def setUp(self): dbPath = os.path.join(getPackageDir('sims_data'), 'OpSimData/opsimblitz1_1133_sqlite.db') self.gen = ObservationMetaDataGenerator(database=dbPath, driver='sqlite')
def make_refcat(opsim_db, obsHistID, boundLength, outfile, catsim_db_info=None, chunk_size=20000): """ Create a reference catalog of stars to use for astrometry from the CatSim db tables. Parameters ---------- opsim_db : str OpSim database sqlite file obsHistID : int Visit number to provide the center of the extraction region. boundLength : float Radius of the extraction region in units of degrees. outfile : str Filename for the reference catalog output file. catsim_db_info : dict, optional Connection information (host, port, database, driver) for the CatSim database. Default: connection info for the UW fatboy server. chunk_size : int, optional The memory chunk size to pass to InstanceCatalog.write_catalog """ if catsim_db_info is None: catsim_db_info = catsim_uw generator = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') obs_metadata = generator.getObservationMetaData(obsHistID=obsHistID, boundLength=boundLength)[0] stars = CatalogDBObject.from_objid('allstars', **catsim_db_info) ref_stars = SimulationReference(stars, obs_metadata=obs_metadata) ref_stars.write_catalog(outfile, write_mode='w', write_header=True, chunk_size=chunk_size)
def setUp(self): self.obsHistID = 1418971 obs_gen = ObservationMetaDataGenerator(database=os.environ['OPSIMDB'], driver='sqlite') self.obs_md \ = obs_gen.getObservationMetaData(obsHistID=self.obsHistID)[0] self.outfile = 'phosim_instcat_%i.txt' % self.obsHistID
def __init__(self, opsim_db, db_config=None, logger=None): """ Constructor. Parameters ---------- opsim_db : str sqlite3 db file containing observing plan. db_config : dict, optional Dictionary of database connection parameters. Parameters for connecting to fatboy.phys.washington.edu from a whitelisted machine will be used. logger : logging.logger, optional Logger object. """ self.gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') if db_config is not None: self.db_config = db_config else: self.db_config = dict(database='LSSTCATSIM', port=1433, host='fatboy.phys.washington.edu', driver='mssql+pymssql') if logger is None: logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout) logger = logging.getLogger() self.logger = logger
def testCreationOfPhoSimCatalog_3(self): """ Make sure that we can create PhoSim input catalogs using the returned ObservationMetaData. Test that an error is actually raised if we try to build a PhoSim catalog with a v3 header map using a v4 ObservationMetaData """ dbName = tempfile.mktemp(dir=ROOT, prefix='obsMetaDataGeneratorTest-', suffix='.db') makePhoSimTestDB(filename=dbName) bulgeDB = testGalaxyBulgeDBObj(driver='sqlite', database=dbName) opsim_db = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'astro-lsst-01_2014.db') assert os.path.isfile(opsim_db) gen = ObservationMetaDataGenerator(opsim_db, driver='sqlite') results = gen.getObservationMetaData(fieldRA=(70.0, 85.0), telescopeFilter='i') self.assertGreater(len(results), 0) testCat = PhoSimCatalogSersic2D(bulgeDB, obs_metadata=results[0]) testCat.phoSimHeaderMap = DefaultPhoSimHeaderMap with lsst.utils.tests.getTempFilePath('.txt') as catName: with self.assertRaises(RuntimeError): testCat.write_catalog(catName) if os.path.exists(dbName): os.unlink(dbName)
def test_query(self): """ Use ObservationMetaData to query an OpSim-like database that contains dithering columns. Make sure that the dithering columns get carried over into the OpsimMetaData of the resulting ObservationMetaData. """ gen = ObservationMetaDataGenerator(database=self.fake_db_name, driver='sqlite') obs_list = gen.getObservationMetaData(fieldRA=(0.0, 180.0)) self.assertGreater(len(obs_list), 0) found_list = [] for obs in obs_list: obsid = obs.OpsimMetaData['obsHistID'] control_dict = self.db_control[obsid] self.assertAlmostEqual(obs._pointingRA, control_dict['ra'], 11) self.assertAlmostEqual(obs._pointingDec, control_dict['dec'], 11) self.assertAlmostEqual(obs._rotSkyPos, control_dict['rot'], 11) self.assertAlmostEqual(obs.OpsimMetaData['m5'], control_dict['m5'], 11) self.assertAlmostEqual(obs.OpsimMetaData['raTestDithering'], control_dict['raDith'], 11) self.assertAlmostEqual(obs.OpsimMetaData['decTestDithering'], control_dict['decDith'], 11) self.assertAlmostEqual(obs.mjd.TAI, control_dict['mjd'], 11) self.assertEqual(obs.bandpass, 'g') self.assertGreaterEqual(obs.pointingRA, 0.0) self.assertLessEqual(obs.pointingRA, 180.0) found_list.append(obs.OpsimMetaData['obsHistID']) # check that the entries not returned do, in fact, violate the query for ix in range(len(self.db_control)): if ix not in found_list: self.assertGreater(self.db_control[ix]['ra'], np.radians(180.0))
def testObsMetaDataBounds(self): """ Make sure that the bound specifications (i.e. a circle or a box on the sky) are correctly passed through to the resulting ObservationMetaData """ gen = ObservationMetaDataGenerator() #Test a cirlce with a specified radius results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i', boundLength=0.9) ct = 0 for obs_metadata in results: self.assertTrue(isinstance(obs_metadata.bounds,CircleBounds)) # include some wiggle room, in case ObservationMetaData needs to # adjust the boundLength to accommodate the transformation between # ICRS and observed coordinates self.assertGreaterEqual(obs_metadata.bounds.radiusdeg, 0.9) self.assertLess(obs_metadata.bounds.radiusdeg, 0.95) self.assertAlmostEqual(obs_metadata.bounds.RA, obs_metadata.phoSimMetaData['pointingRA'][0], 5) self.assertAlmostEqual(obs_metadata.bounds.DEC, obs_metadata.phoSimMetaData['pointingDec'][0], 5) ct += 1 #Make sure that some ObservationMetaData were tested self.assertGreater(ct, 0) boundLengthList = [1.2, (1.2, 0.6)] for boundLength in boundLengthList: results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i', boundType='box', boundLength=boundLength) if hasattr(boundLength, '__len__'): dra = boundLength[0] ddec = boundLength[1] else: dra = boundLength ddec = boundLength ct = 0 for obs_metadata in results: RAdeg = numpy.degrees(obs_metadata.phoSimMetaData['pointingRA'][0]) DECdeg = numpy.degrees(obs_metadata.phoSimMetaData['pointingDec'][0]) self.assertTrue(isinstance(obs_metadata.bounds,BoxBounds)) self.assertAlmostEqual(obs_metadata.bounds.RAminDeg, RAdeg-dra, 10) self.assertAlmostEqual(obs_metadata.bounds.RAmaxDeg, RAdeg+dra, 10) self.assertAlmostEqual(obs_metadata.bounds.DECminDeg, DECdeg-ddec, 10) self.assertAlmostEqual(obs_metadata.bounds.DECmaxDeg, DECdeg+ddec, 10) self.assertAlmostEqual(obs_metadata.bounds.RA, obs_metadata.phoSimMetaData['pointingRA'][0], 5) self.assertAlmostEqual(obs_metadata.bounds.DEC, obs_metadata.phoSimMetaData['pointingDec'][0], 5) ct += 1 #Make sure that some ObservationMetaData were tested self.assertGreater(ct, 0)
def _read_pointing_info(self, opsim_db): try: self.ratel = self.eimage[0].header['RATEL'] self.dectel = self.eimage[0].header['DECTEL'] self.rotangle = self.eimage[0].header['ROTANGLE'] return except KeyError: if opsim_db is None: raise RuntimeError("eimage file does not have pointing info. " "Need an opsim db file.") # Read from the opsim db. # We need an ObservationMetaData object to use the getRotSkyPos # function. obs_gen = ObservationMetaDataGenerator(database=opsim_db, driver="sqlite") obs_md = obs_gen.getObservationMetaData(obsHistID=self.visit, boundType='circle', boundLength=0)[0] # Extract pointing info from opsim db for desired visit. conn = sqlite3.connect(opsim_db) query = """select descDitheredRA, descDitheredDec, descDitheredRotTelPos from summary where obshistid={}""".format(self.visit) curs = conn.execute(query) ra, dec, rottelpos = [np.degrees(x) for x in curs][0] conn.close() self.ratel, self.dectel = ra, dec obs_md.pointingRA = ra obs_md.pointingDec = dec self.rotangle = getRotSkyPos(ra, dec, obs_md, rottelpos)
def test_query(self): """ Use ObservationMetaData to query an OpSim-like database that contains dithering columns. Make sure that the dithering columns get carried over into the OpsimMetaData of the resulting ObservationMetaData. """ gen = ObservationMetaDataGenerator(database=self.fake_db_name, driver='sqlite') obs_list = gen.getObservationMetaData(fieldRA=(0.0, 180.0)) self.assertGreater(len(obs_list), 0) found_list = [] for obs in obs_list: obsid = obs.OpsimMetaData['obsHistID'] control_dict = self.db_control[obsid] self.assertAlmostEqual(obs._pointingRA, control_dict['ra'], 11) self.assertAlmostEqual(obs._pointingDec, control_dict['dec'], 11) self.assertAlmostEqual(obs._rotSkyPos, control_dict['rot'], 11) self.assertAlmostEqual(obs.OpsimMetaData['m5'], control_dict['m5'], 11) self.assertAlmostEqual(obs.OpsimMetaData['raTestDithering'], control_dict['raDith'], 11) self.assertAlmostEqual(obs.OpsimMetaData['decTestDithering'], control_dict['decDith'], 11) self.assertAlmostEqual(obs.mjd.TAI, control_dict['mjd'], 11) self.assertEqual(obs.bandpass, 'g') self.assertGreaterEqual(obs.pointingRA, 0.0) self.assertLessEqual(obs.pointingRA, 180.0) found_list.append(obs.OpsimMetaData['obsHistID']) # check that the entries not returned do, in fact, violate the query for ix in range(len(self.db_control)): if ix not in found_list: self.assertGreater(self.db_control[ix]['ra'], np.radians(180.0))
def __init__( self, opsim_db='/global/projecta/projectdirs/lsst/groups/SSim/DC2/minion_1016_desc_dithered_v4.db' ): self.conn = sqlite3.connect(opsim_db) self.obs_gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') self._cache = dict()
def setUpClass(cls): opsimdb = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'opsimblitz1_1133_sqlite.db') obs_gen = ObservationMetaDataGenerator(opsimdb) cls.obs_dict = {} for band in 'ugrizy': obs_list = obs_gen.getObservationMetaData(telescopeFilter=band, limit=10) assert len(obs_list) > 0 cls.obs_dict[band] = obs_list[0]
def testQueryLimit(self): """ Test that, when we specify a limit on the number of ObservationMetaData we want returned, that limit is respected """ gen = ObservationMetaDataGenerator() results = gen.getObservationMetaData(fieldRA=(numpy.degrees(1.370916), numpy.degrees(1.5348635)), limit=20) self.assertEqual(len(results),20)
def test_multiband_light_curves(self): """ Check that multi-band light curves are returned correctly. """ raRange = (78.0, 82.0) decRange = (-69.0, -65.0) bandpass = ('r', 'g') gen = StellarLightCurveGenerator(self.stellar_db, self.opsimDb) pointings = gen.get_pointings(raRange, decRange, bandpass=bandpass) lc_dict, truth_info = gen.light_curves_from_pointings(pointings) self.assertGreater(len(lc_dict), 2) obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite') control_pointings_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, telescopeFilter='r', boundLength=1.75) control_pointings_g = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, telescopeFilter='g', boundLength=1.75) self.assertGreater(len(control_pointings_g), 0) self.assertGreater(len(control_pointings_r), 0) ct = 0 for obs in control_pointings_r: cat = stellarControlCatalog(self.stellar_db, obs_metadata=obs) for star_obj in cat.iter_catalog(): ct += 1 lc = lc_dict[star_obj[0]]['r'] dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) for obs in control_pointings_g: cat = stellarControlCatalog(self.stellar_db, obs_metadata=obs) for star_obj in cat.iter_catalog(): ct += 1 lc = lc_dict[star_obj[0]]['g'] dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) self.assertLess(np.abs(lc['mag'][dex]-star_obj[3]), 1.0e-7) self.assertLess(np.abs(lc['error'][dex]-star_obj[4]), 1.0e-7) # Verify that the same number of objects and observations were found in the # catalogs and the LightCurveGenerator output total_ct = 0 for obj_name in lc_dict: for bandpass in lc_dict[obj_name]: total_ct += len(lc_dict[obj_name][bandpass]['mjd']) self.assertEqual(ct, total_ct)
def __init__(self, catalogdb, opsimdb, opsimdriver="sqlite"): self._generator = ObservationMetaDataGenerator(database=opsimdb, driver=opsimdriver) self._catalogdb = catalogdb # optional constraint on query to catalog database # (usually 'varParamStr IS NOT NULL') if not hasattr(self, '_constraint'): self._constraint = None
def test_sne_multiband_light_curves(self): """ Generate some super nova light curves. Verify that they come up with the same magnitudes and uncertainties as supernova catalogs. Use multiband light curves. """ gen = SNIaLightCurveGenerator(self.db, self.opsimDb) raRange = (78.0, 85.0) decRange = (-69.0, -65.0) pointings = gen.get_pointings(raRange, decRange, bandpass=('r', 'z')) gen.sn_universe._midSurveyTime = 49000.0 gen.sn_universe._snFrequency = 0.001 self.assertGreater(len(pointings), 1) lc_dict, truth = gen.light_curves_from_pointings(pointings) self.assertGreater(len(lc_dict), 0) obs_gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite') control_obs_r = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, telescopeFilter='r', boundLength=1.75) control_obs_z = obs_gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, telescopeFilter='z', boundLength=1.75) self.assertGreater(len(control_obs_r), 0) self.assertGreater(len(control_obs_z), 0) ct_r = 0 for obs in control_obs_r: cat = SNIaLightCurveControlCatalog(self.db, obs_metadata=obs) for sn in cat.iter_catalog(): if sn[1] > 0.0: ct_r += 1 lc = lc_dict[sn[0]]['r'] dex = np.argmin(np.abs(lc['mjd'] - obs.mjd.TAI)) self.assertLess(np.abs(lc['mjd'][dex] - obs.mjd.TAI), 1.0e-7) self.assertLess(np.abs(lc['flux'][dex] - sn[1]), 1.0e-7) self.assertLess(np.abs(lc['error'][dex] - sn[2]), 1.0e-7) self.assertGreater(ct_r, 0) ct_z = 0 for obs in control_obs_z: cat = SNIaLightCurveControlCatalog(self.db, obs_metadata=obs) for sn in cat.iter_catalog(): if sn[1] > 0.0: ct_z += 1 lc = lc_dict[sn[0]]['z'] dex = np.argmin(np.abs(lc['mjd'] - obs.mjd.TAI)) self.assertLess(np.abs(lc['mjd'][dex] - obs.mjd.TAI), 1.0e-7) self.assertLess(np.abs(lc['flux'][dex] - sn[1]), 1.0e-7) self.assertLess(np.abs(lc['error'][dex] - sn[2]), 1.0e-7) self.assertGreater(ct_z, 0)
def _set_obs_md_results(self, opsim_db, fieldRA, fieldDec, boundLength, pickle_file): if pickle_file is not None and os.path.isfile(pickle_file): self.obs_md_results = pickle.load(open(pickle_file)) else: # Generate the observation metadata from the db file. gen = ObservationMetaDataGenerator(database=opsim_db, driver="sqlite") self.obs_md_results = gen.getObservationMetaData( fieldRA=fieldRA, fieldDec=fieldDec, boundLength=boundLength ) if pickle_file is not None: pickle.dump(self.obs_md_results, open(pickle_file, "w"))
def _set_obs_md_results(self, opsim_db, fieldRA, fieldDec, boundLength, pickle_file): if pickle_file is not None and os.path.isfile(pickle_file): self.obs_md_results = pickle.load(open(pickle_file)) else: # Generate the observation metadata from the db file. gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') self.obs_md_results = gen.getObservationMetaData( fieldRA=fieldRA, fieldDec=fieldDec, boundLength=boundLength) if pickle_file is not None: pickle.dump(self.obs_md_results, open(pickle_file, 'w'))
def testQueryOnFilter(self): """ Test that queries on the filter work. """ gen = ObservationMetaDataGenerator() results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i') ct = 0 for obs_metadata in results: self.assertAlmostEqual(obs_metadata.phoSimMetaData['pointingRA'][0],1.370916) self.assertEqual(obs_metadata.phoSimMetaData['Opsim_filter'][0],'i') ct += 1 #Make sure that more than zero ObservationMetaData were returned self.assertGreater(ct, 0)
def testPassInOtherQuery(self): """ Test that you can pass OpSim pointings generated from another source into an ObservationMetaDataGenerator and still get ObservationMetaData out """ pointing_list = self.gen.getOpSimRecords(fieldRA=np.degrees(1.370916)) self.assertGreater(len(pointing_list), 1) local_gen = ObservationMetaDataGenerator() obs_list = local_gen.ObservationMetaDataFromPointingArray(pointing_list) self.assertEqual(len(obs_list), len(pointing_list)) for pp in pointing_list: obs = local_gen.ObservationMetaDataFromPointing(pp) self.assertIsInstance(obs, ObservationMetaData)
def testIncompletDB(self): """ Test that if the mock OpSim database does not have all required columns, an exception is raised. """ scratch_dir = os.path.join(getPackageDir('sims_catUtils'), 'tests', 'scratchSpace') opsim_db_name = os.path.join(scratch_dir, 'incomplete_mock_opsim_sqlite.db') if os.path.exists(opsim_db_name): os.unlink(opsim_db_name) conn = sqlite3.connect(opsim_db_name) c = conn.cursor() c.execute('''CREATE TABLE Summary (obsHistID int, expMJD real, ''' '''fieldRA real, filter text)''') conn.commit() rng = np.random.RandomState(77) n_pointings = 100 ra_data = rng.random_sample(n_pointings) * 2.0 * np.pi mjd_data = rng.random_sample(n_pointings) * 1000.0 + 59580.0 filter_dexes = rng.randint(0, 6, n_pointings) bands = ('u', 'g', 'r', 'i', 'z', 'y') filter_data = [] for ii in filter_dexes: filter_data.append(bands[ii]) for ii in range(n_pointings): cmd = '''INSERT INTO Summary VALUES(%i, %f, %f, '%s')''' % \ (ii, mjd_data[ii], ra_data[ii], filter_data[ii]) c.execute(cmd) conn.commit() conn.close() incomplete_obs_gen = ObservationMetaDataGenerator( database=opsim_db_name) with self.assertRaises(RuntimeError) as context: incomplete_obs_gen.getObservationMetaData(telescopeFilter='r') self.assertIn( "ObservationMetaDataGenerator requires that the database", context.exception.args[0]) if os.path.exists(opsim_db_name): os.unlink(opsim_db_name)
def testCreationOfPhoSimCatalog(self): """ Make sure that we can create PhoSim input catalogs using the returned ObservationMetaData. This test will just make sure that all of the expected header entries are there. """ dbName = 'obsMetaDataGeneratorTest.db' catName = 'testPhoSimFromObsMetaDataGenerator.txt' if os.path.exists(dbName): os.unlink(dbName) junk_obs_metadata = makePhoSimTestDB(filename=dbName) bulgeDB = testGalaxyBulge(driver='sqlite', database=dbName) gen = ObservationMetaDataGenerator() results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916),telescopeFilter='i') testCat = PhoSimCatalogSersic2D(bulgeDB, obs_metadata=results[0]) testCat.write_catalog(catName) filterTranslation=['u','g','r','i','z','y'] with open(catName) as inputFile: lines = inputFile.readlines() ix = 0 for control in gen.columnMapping: if control[0] != 'm5' and control[0]!='skyBrightness' and control[0]!='seeing': words = lines[ix].split() self.assertEqual(control[2].replace('pointing', 'Unrefracted_'), words[0]) if control[0] != 'telescopeFilter': if control[4] is not None: value = control[4](float(words[1])) else: value = float(words[1]) self.assertAlmostEqual(value, results[0].phoSimMetaData[control[2]][0], 5) else: self.assertEqual(filterTranslation[int(words[1])],results[0].phoSimMetaData[control[2]][0]) ix += 1 if os.path.exists(catName): os.unlink(catName) if os.path.exists(dbName): os.unlink(dbName)
def __init__(self, catalogdb, opsimdb, opsimdriver="sqlite"): self._generator = ObservationMetaDataGenerator(database=opsimdb, driver=opsimdriver) self._catalogdb = catalogdb # optional constraint on query to catalog database # (usually 'varParamStr IS NOT NULL') if not hasattr(self, '_constraint'): self._constraint = None
def testIncompletDB(self): """ Test that if the mock OpSim database does not have all required columns, an exception is raised. """ opsim_db_name = tempfile.mktemp(dir=ROOT, prefix='incomplete_mock_opsim_sqlite-', suffix='.db') conn = sqlite3.connect(opsim_db_name) c = conn.cursor() c.execute('''CREATE TABLE Summary (obsHistID int, expMJD real, ''' '''fieldRA real, filter text)''') conn.commit() rng = np.random.RandomState(77) n_pointings = 100 ra_data = rng.random_sample(n_pointings)*2.0*np.pi mjd_data = rng.random_sample(n_pointings)*1000.0 + 59580.0 filter_dexes = rng.randint(0, 6, n_pointings) bands = ('u', 'g', 'r', 'i', 'z', 'y') filter_data = [] for ii in filter_dexes: filter_data.append(bands[ii]) for ii in range(n_pointings): cmd = '''INSERT INTO Summary VALUES(%i, %f, %f, '%s')''' % \ (ii, mjd_data[ii], ra_data[ii], filter_data[ii]) c.execute(cmd) conn.commit() conn.close() incomplete_obs_gen = ObservationMetaDataGenerator(database=opsim_db_name) with self.assertRaises(RuntimeError) as context: incomplete_obs_gen.getObservationMetaData(telescopeFilter='r') self.assertIn("ObservationMetaDataGenerator requires that the database", context.exception.args[0]) if os.path.exists(opsim_db_name): os.unlink(opsim_db_name)
def test_spatial_query(self): """ Test that spatial queries work """ db_dir = os.path.join(getPackageDir('sims_data'), 'OpSimData') assert os.path.isdir(db_dir) db_file = os.path.join(db_dir, 'astro-lsst-01_2014.db') obs_gen = ObservationMetaDataGenerator(db_file) obs_list = obs_gen.getObservationMetaData(fieldRA=(20.0, 40.0), fieldDec=(-30.0, -10.0)) self.assertGreater(len(obs_list), 10) with sqlite3.connect(db_file) as conn: cursor = conn.cursor() query = '''SELECT observationId, fieldRA, fieldDec, observationStartMJD, filter FROM SummaryAllProps WHERE fieldRA BETWEEN 20.0 AND 40.0 AND fieldDec BETWEEN -30.0 AND -10.0 ORDER BY observationId''' control = cursor.execute(query).fetchall() self.assertEqual(len(control), len(obs_list)) for ii in range(len(obs_list)): self.assertEqual(obs_list[ii].OpsimMetaData['observationId'], int(control[ii][0])) self.assertAlmostEqual(obs_list[ii].pointingRA, float(control[ii][1]), 10) self.assertAlmostEqual(obs_list[ii].pointingDec, float(control[ii][2]), 10) self.assertAlmostEqual(obs_list[ii].mjd.TAI, float(control[ii][3]), 7) self.assertEqual(obs_list[ii].bandpass, str(control[ii][4])) self.assertGreaterEqual(obs_list[ii].pointingRA, 20.0) self.assertLessEqual(obs_list[ii].pointingRA, 40.0) self.assertGreaterEqual(obs_list[ii].pointingDec, -30.0) self.assertLessEqual(obs_list[ii].pointingDec, -10.0)
def testOnNonExistentDatabase(self): """ Test that an exception is raised if you try to connect to an query a database that does not exist. """ test_name = 'non_existent.db' with self.assertRaises(RuntimeError) as context: ObservationMetaDataGenerator(database=test_name, driver='sqlite') self.assertEqual(context.exception.args[0], '%s does not exist' % test_name) self.assertFalse(os.path.exists(test_name))
def test_spatial_query(self): """ Test that spatial queries work """ db_dir = os.path.join(getPackageDir('sims_data'), 'OpSimData') assert os.path.isdir(db_dir) db_file = os.path.join(db_dir, 'astro-lsst-01_2014.db') obs_gen = ObservationMetaDataGenerator(db_file) obs_list = obs_gen.getObservationMetaData(fieldRA=(20.0, 40.0), fieldDec=(-30.0, -10.0)) self.assertGreater(len(obs_list), 10) with sqlite3.connect(db_file) as conn: cursor = conn.cursor() query = '''SELECT observationId, fieldRA, fieldDec, observationStartMJD, filter FROM SummaryAllProps WHERE fieldRA BETWEEN 20.0 AND 40.0 AND fieldDec BETWEEN -30.0 AND -10.0 ORDER BY observationId''' control = cursor.execute(query).fetchall() self.assertEqual(len(control), len(obs_list)) for ii in range(len(obs_list)): self.assertEqual(obs_list[ii].OpsimMetaData['observationId'], int(control[ii][0])) self.assertAlmostEqual(obs_list[ii].pointingRA, float(control[ii][1]), 10) self.assertAlmostEqual(obs_list[ii].pointingDec, float(control[ii][2]), 10) self.assertAlmostEqual(obs_list[ii].mjd.TAI, float(control[ii][3]), 7) self.assertEqual(obs_list[ii].bandpass, str(control[ii][4])) self.assertGreaterEqual(obs_list[ii].pointingRA, 20.0) self.assertLessEqual(obs_list[ii].pointingRA, 40.0) self.assertGreaterEqual(obs_list[ii].pointingDec, -30.0) self.assertLessEqual(obs_list[ii].pointingDec, -10.0)
class OpsimdbInterface(object): def __init__( self, opsim_db='/global/projecta/projectdirs/lsst/groups/SSim/DC2/minion_1016_desc_dithered_v4.db' ): self.conn = sqlite3.connect(opsim_db) self.obs_gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') self._cache = dict() def get_obs_md(self, obsHistID): if obsHistID not in self._cache: self._cache[obsHistID] = self._get_obs_md(obsHistID) return self._cache[obsHistID] def _get_obs_md(self, obsHistID): curs = self.conn.execute( 'select descDitheredRA, descDitheredDec, descDitheredRotTelPos from Summary where obsHistID={}' .format(obsHistID)) data = [x for x in curs][0] ra, dec = [x * 180. / np.pi for x in data[:2]] rottelpos = data[2] obs_md = self.obs_gen.getObservationMetaData(obsHistID=obsHistID, boundType='circle', boundLength=0.1)[0] obs_md.pointingRA = ra obs_md.pointingDec = dec obs_md.OpsimMetaData['rotTelPos'] = rottelpos obs_md.rotSkyPos = getRotSkyPos(obs_md._pointingRA, obs_md._pointingDec, obs_md, rottelpos) * 180. / np.pi return obs_md def plot_fov(self, obsHistID, radius=2.047): obs_md = self.get_obs_md(obsHistID) ra, dec = obs_md.pointingRA, obs_md.pointingDec phi = np.linspace(0, 2 * np.pi, 100) radius /= np.cos(dec * np.pi / 180.) plt.errorbar(radius * np.sin(phi) + ra, radius * np.cos(phi) + dec, fmt='--', label='{} fov'.format(obsHistID))
class DescObsMdGenerator: def __init__(self, opsim_db_file): self.obs_gen = ObservationMetaDataGenerator(database=opsim_db_file, driver='sqlite') self.opsim_db_file = opsim_db_file def create(self, visit): obs_md = self.obs_gen.getObservationMetaData(obsHistID=visit, boundType='circle', boundLength=0)[0] query = f'''select descDitheredRA, descDitheredDec, descDitheredRotTelPos from summary where obsHistID={visit}''' with sqlite3.connect(self.opsim_db_file) as conn: curs = conn.execute(query) ra, dec, rottelpos = [np.degrees(_) for _ in curs][0] obs_md.pointingRA = ra obs_md.pointingDec = dec obs_md.rotSkyPos = getRotSkyPos(ra, dec, obs_md, rottelpos) return obs_md
def __init__(self, opsimdb, descqa_catalog, dither=True, min_mag=10, minsource=100, proper_motion=False, imsim_catalog=False): """ Parameters ---------- obsimdb: str OpSim db filename. descqa_catalog: str Name of the DESCQA galaxy catalog. dither: bool [True] Flag to enable the dithering included in the opsim db file. min_mag: float [10] Minimum value of the star magnitude at 500nm to include. minsource: int [100] Minimum number of objects for phosim.py to simulate a chip. proper_motion: bool [True] Flag to enable application of proper motion to stars. imsim_catalog: bool [False] Flag to write an imsim-style object catalog. """ if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) self.descqa_catalog = descqa_catalog self.dither = dither self.min_mag = min_mag self.minsource = minsource self.proper_motion = proper_motion self.imsim_catalog = imsim_catalog self.obs_gen = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') self.star_db = StarObj(database='LSSTCATSIM', host='fatboy.phys.washington.edu', port=1433, driver='mssql+pymssql') self.instcats = get_instance_catalogs(imsim_catalog)
args = parser.parse_args() camera = lsst_camera() det_name_list = [] for det in camera: if det.getType() != SCIENCE: continue det_name_list.append(det.getName()) det_name_list.sort() opsimdb = os.path.join('/Users', 'danielsf', 'physics', 'lsst_150412', 'Development', 'garage', 'OpSimData', 'minion_1016_sqlite.db') assert os.path.exists(opsimdb) obs_gen = ObservationMetaDataGenerator(database=opsimdb) obs_list = obs_gen.getObservationMetaData(obsHistID=args.obs) obs = obs_list[0] filter_name = obs.bandpass site_no_atm = Site(name="LSST", pressure=0.0, humidity=0.0) obs.site = site_no_atm assert np.abs(obs.site.pressure) < 1.0e-6 assert np.abs(obs.site.humidity) < 1.0e-6 xpix_0 = np.arange(200.0, 3800.0, 1000.0) ypix_0 = np.arange(200.0, 3800.0, 1000.0) pix_grid = np.meshgrid(xpix_0, ypix_0) cam_xpix_in = pix_grid[0].flatten() cam_ypix_in = pix_grid[1].flatten() camera_wrapper = LSSTCameraWrapper()
def testObsMetaDataBounds(self): """ Make sure that the bound specifications (i.e. a circle or a box on the sky) are correctly passed through to the resulting ObservationMetaData """ gen = ObservationMetaDataGenerator() #Test a cirlce with a specified radius results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i', boundLength=0.9) ct = 0 for obs_metadata in results: self.assertTrue(isinstance(obs_metadata.bounds,CircleBounds)) self.assertAlmostEqual(obs_metadata.bounds.radiusdeg,0.9,10) self.assertAlmostEqual(obs_metadata.bounds.RA,obs_metadata.phoSimMetaData['Unrefracted_RA'][0],10) self.assertAlmostEqual(obs_metadata.bounds.DEC,obs_metadata.phoSimMetaData['Unrefracted_Dec'][0],10) ct += 1 #Make sure that some ObservationMetaData were tested self.assertTrue(ct>0) #test a square results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i', boundType='box', boundLength=1.2) ct = 0 for obs_metadata in results: RAdeg = numpy.degrees(obs_metadata.phoSimMetaData['Unrefracted_RA'][0]) DECdeg = numpy.degrees(obs_metadata.phoSimMetaData['Unrefracted_Dec'][0]) self.assertTrue(isinstance(obs_metadata.bounds,BoxBounds)) self.assertAlmostEqual(obs_metadata.bounds.RAminDeg,RAdeg-1.2,10) self.assertAlmostEqual(obs_metadata.bounds.RAmaxDeg,RAdeg+1.2,10) self.assertAlmostEqual(obs_metadata.bounds.DECminDeg,DECdeg-1.2,10) self.assertAlmostEqual(obs_metadata.bounds.DECmaxDeg,DECdeg+1.2,10) self.assertAlmostEqual(obs_metadata.bounds.RA,obs_metadata.phoSimMetaData['Unrefracted_RA'][0],10) self.assertAlmostEqual(obs_metadata.bounds.DEC,obs_metadata.phoSimMetaData['Unrefracted_Dec'][0],10) ct += 1 #Make sure that some ObservationMetaData were tested self.assertTrue(ct>0) #test a rectangle results = gen.getObservationMetaData(fieldRA=numpy.degrees(1.370916), telescopeFilter='i', boundType='box', boundLength=(1.2,0.6)) ct = 0 for obs_metadata in results: RAdeg = numpy.degrees(obs_metadata.phoSimMetaData['Unrefracted_RA'][0]) DECdeg = numpy.degrees(obs_metadata.phoSimMetaData['Unrefracted_Dec'][0]) self.assertTrue(isinstance(obs_metadata.bounds,BoxBounds)) self.assertAlmostEqual(obs_metadata.bounds.RAminDeg,RAdeg-1.2,10) self.assertAlmostEqual(obs_metadata.bounds.RAmaxDeg,RAdeg+1.2,10) self.assertAlmostEqual(obs_metadata.bounds.DECminDeg,DECdeg-0.6,10) self.assertAlmostEqual(obs_metadata.bounds.DECmaxDeg,DECdeg+0.6,10) self.assertAlmostEqual(obs_metadata.bounds.RA,obs_metadata.phoSimMetaData['Unrefracted_RA'][0],10) self.assertAlmostEqual(obs_metadata.bounds.DEC,obs_metadata.phoSimMetaData['Unrefracted_Dec'][0],10) ct += 1 #Make sure that some ObservationMetaData were tested self.assertTrue(ct>0)
def __init__(self, opsimdb, descqa_catalog, dither=True, min_mag=10, minsource=100, proper_motion=False, imsim_catalog=False, protoDC2_ra=0, protoDC2_dec=0, agn_db_name=None, sprinkler=False): """ Parameters ---------- obsimdb: str OpSim db filename. descqa_catalog: str Name of the DESCQA galaxy catalog. dither: bool [True] Flag to enable the dithering included in the opsim db file. min_mag: float [10] Minimum value of the star magnitude at 500nm to include. minsource: int [100] Minimum number of objects for phosim.py to simulate a chip. proper_motion: bool [True] Flag to enable application of proper motion to stars. imsim_catalog: bool [False] Flag to write an imsim-style object catalog. protoDC2_ra: float [0] Desired RA (J2000 degrees) of protoDC2 center. protoDC2_dec: float [0] Desired Dec (J2000 degrees) of protoDC2 center. agn_db_name: str [None] Filename of the agn parameter sqlite db file. sprinkler: bool [False] Flag to enable the Sprinkler. """ if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) # load the data for the parametrized light # curve stellar variability model into a # global cache plc = ParametrizedLightCurveMixin() plc.load_parametrized_light_curves() self.descqa_catalog = descqa_catalog self.dither = dither self.min_mag = min_mag self.minsource = minsource self.proper_motion = proper_motion self.imsim_catalog = imsim_catalog self.protoDC2_ra = protoDC2_ra self.protoDC2_dec = protoDC2_dec self.phot_params = PhotometricParameters(nexp=1, exptime=30) self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() self.obs_gen = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') self.star_db = StarObj(database='LSSTCATSIM', host='fatboy.phys.washington.edu', port=1433, driver='mssql+pymssql') if agn_db_name is None: raise IOError("Need to specify an Proto DC2 AGN database.") else: if os.path.exists(agn_db_name): self.agn_db_name = agn_db_name else: raise IOError("Path to Proto DC2 AGN database does not exist.") self.sprinkler = sprinkler self.instcats = get_instance_catalogs(imsim_catalog)
class ssmCatCamera(ssmCat): column_outputs = basic_columns + ['chipName'] camera = LsstSimMapper().camera cannot_be_null = ['visibility', 'chipName'] transformations = {'raJ2000': np.degrees, 'decJ2000': np.degrees, 'velRa': np.degrees, 'velDec': np.degrees} default_formats = {'f':'%.13f'} ###### t = time.time() # Get opsim data. opsdb = '/Users/lynnej/opsim/db/minion_1016_newsky.db' generator = ObservationMetaDataGenerator(database=opsdb, driver='sqlite') night = 203 query = 'select min(expMJD), max(expMJD) from summary where night=%d and filter="r"' %(night) res = generator.opsimdb.execute_arbitrary(query) expMJD_min = res[0][0] expMJD_max = res[0][1] # Test image (deep, r band, near ecliptic) obsMetaDataResults = generator.getObservationMetaData(expMJD=(expMJD_min, expMJD_max), boundLength=2.2) dt, t = dtime(t) print('To query opsim database: %f seconds' %(dt)) write_header = True write_mode = 'w'
def test_ssm_catalog_creation(self): t = time.time() # Fake opsim data. database = os.path.join(getPackageDir('SIMS_DATA'), 'OpSimData/opsimblitz1_1133_sqlite.db') generator = ObservationMetaDataGenerator(database=database, driver='sqlite') night = 20 query = 'select min(expMJD), max(expMJD) from summary where night=%d' % ( night) res = generator.opsimdb.execute_arbitrary(query) expMJD_min = res[0][0] expMJD_max = res[0][1] obsMetaDataResults = generator.getObservationMetaData( expMJD=(expMJD_min, expMJD_max), limit=3, boundLength=2.2) dt, t = dtime(t) print('To query opsim database: %f seconds' % (dt)) write_header = True write_mode = 'w' try: ssmObj = SolarSystemObj() for obsMeta in obsMetaDataResults: # But moving objects databases are not currently complete for all years. # Push forward to night=747. # (note that we need the phosim dictionary as well) newMJD = 59590.2 # this MJD is artificially chosen to be in the # time span of the new baseline simulated survey obs = ObservationMetaData( mjd=newMJD, pointingRA=obsMeta.pointingRA, pointingDec=obsMeta.pointingDec, bandpassName=obsMeta.bandpass, rotSkyPos=obsMeta.rotSkyPos, m5=obsMeta.m5[obsMeta.bandpass], seeing=obsMeta.seeing[obsMeta.bandpass], boundLength=obsMeta.boundLength, boundType=obsMeta.boundType) obs._OpsimMetaData = {'visitExpTime': 30} mySsmDb = ssmCatCamera(ssmObj, obs_metadata=obs) photParams = PhotometricParameters( exptime=obs.OpsimMetaData['visitExpTime'], nexp=1, bandpass=obs.bandpass) mySsmDb.photParams = photParams try: with lsst.utils.tests.getTempFilePath( '.txt') as output_cat: mySsmDb.write_catalog(output_cat, write_header=write_header, write_mode=write_mode) # verify that we did not write an empty catalog with open(output_cat, 'r') as input_file: lines = input_file.readlines() msg = 'MJD is %.3f' % obs.mjd.TAI self.assertGreater(len(lines), 1, msg=msg) except: # This is because the solar system object 'tables' # don't actually connect to tables on fatboy; they just # call methods stored on fatboy. Therefore, the connection # failure will not be noticed until this part of the test msg = sys.exc_info()[1].args[0] if 'DB-Lib error' in msg: reassure() continue else: raise write_mode = 'a' write_header = False dt, t = dtime(t) print( 'To query solar system objects: %f seconds (obs MJD time %f)' % (dt, obs.mjd.TAI)) except: trace = traceback.extract_tb(sys.exc_info()[2], limit=20) msg = sys.exc_info()[1].args[0] if 'Failed to connect' in msg or failedOnFatboy(trace): # if the exception was because of a failed connection # to fatboy, ignore it. reassure() pass else: raise
def setUpClass(cls): # Set directory where scratch work will be done cls.scratchDir = tempfile.mkdtemp(dir=ROOT, prefix='scratchSpace-') # ObsMetaData instance with spatial window within which we will # put galaxies in a fake galaxy catalog cls.obsMetaDataforCat = ObservationMetaData(boundType='circle', boundLength=np.degrees(0.25), pointingRA=np.degrees(0.13), pointingDec=np.degrees(-1.2), bandpassName=['r'], mjd=49350.) # Randomly generate self.size Galaxy positions within the spatial window # of obsMetaDataforCat cls.dbname = os.path.join(cls.scratchDir, 'galcat.db') cls.size = 1000 cls.GalaxyPositionSamps = sample_obsmetadata(obsmetadata=cls.obsMetaDataforCat, size=cls.size) # Create a galaxy Table overlapping with the obsMetaData Spatial Bounds # using positions from the samples above and a database name given by # self.dbname vals = cls._createFakeGalaxyDB() cls.valName = os.path.join(cls.scratchDir, 'valsFromTest.dat') with open(cls.valName, 'w') as f: for i, v in enumerate(vals[0]): f.write(str(np.radians(vals[0][i])) + ' ' + str(np.radians(vals[1][i])) + '\n') # fig, ax = plt.subplots() # ax.plot(vals[0][:1000], vals[1][: 1000], '.') # ax.plot([0.13], [-1.2], 'rs', markersize=8) # fig.savefig(os.path.join(cls.scratchDir, 'match_galDBPosns.pdf')) # Read it into a CatalogDBObject galDB class MyGalaxyCatalog(CatalogDBObject): ''' Create a like CatalogDBObject connecting to a local sqlite database ''' objid = 'mytestgals' tableid = 'gals' idColKey = 'id' objectTypeId = 0 appendint = 10000 database = cls.dbname # dbAddress = './testData/galcat.db' raColName = 'raJ2000' decColName = 'decJ2000' driver = 'sqlite' # columns required to convert the ra, dec values in degrees # to radians again columns = [('id', 'id', int), ('raJ2000', 'raJ2000 * PI()/ 180. '), ('decJ2000', 'decJ2000 * PI()/ 180.'), ('redshift', 'redshift')] cls.galDB = MyGalaxyCatalog(database=cls.dbname) # Generate a set of Observation MetaData Outputs that overlap # the galaxies in space opsimPath = os.path.join(getPackageDir('sims_data'), 'OpSimData') opsimDB = os.path.join(opsimPath, 'opsimblitz1_1133_sqlite.db') generator = ObservationMetaDataGenerator(database=opsimDB) cls.obsMetaDataResults = generator.getObservationMetaData(limit=100, fieldRA=(5.0, 8.0), fieldDec=(-85., -60.), expMJD=(49300., 49400.), boundLength=0.15, boundType='circle') sncatalog = SNIaCatalog(db_obj=cls.galDB, obs_metadata=cls.obsMetaDataResults[6], column_outputs=['t0', 'flux_u', 'flux_g', 'flux_r', 'flux_i', 'flux_z', 'flux_y', 'mag_u', 'mag_g', 'mag_r', 'mag_i', 'mag_z', 'mag_y', 'adu_u', 'adu_g', 'adu_r', 'adu_i', 'adu_z', 'adu_y', 'mwebv']) sncatalog.suppressDimSN = True sncatalog.midSurveyTime = sncatalog.mjdobs - 20. sncatalog.snFrequency = 1.0 cls.fullCatalog = os.path.join(cls.scratchDir, 'testSNCatalogTest.dat') sncatalog.write_catalog(cls.fullCatalog) # Create a SNCatalog based on GalDB, and having times of explosions # overlapping the times in obsMetaData cls.fnameList = cls._writeManySNCatalogs(cls.obsMetaDataResults)
class InstanceCatalogMaker(object): """ Class for creating instance catalogs. Attributes ---------- gen : lsst.sims.catUtils.utils.ObservationMetaDataGenerator db_config : dict Dictionary of database connection parameters. logger : logging.logger Logger object. """ star_objs = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'] gal_objs = ['galaxyBulge', 'galaxyDisk'] def __init__(self, opsim_db, db_config=None, logger=None): """ Constructor. Parameters ---------- opsim_db : str sqlite3 db file containing observing plan. db_config : dict, optional Dictionary of database connection parameters. Parameters for connecting to fatboy.phys.washington.edu from a whitelisted machine will be used. logger : logging.logger, optional Logger object. """ self.gen = ObservationMetaDataGenerator(database=opsim_db, driver='sqlite') if db_config is not None: self.db_config = db_config else: self.db_config = dict(database='LSSTCATSIM', port=1433, host='fatboy.phys.washington.edu', driver='mssql+pymssql') if logger is None: logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout) logger = logging.getLogger() self.logger = logger def make_instance_catalog(self, obsHistID, band, boundLength, outfile=None): """ Method to create instance catalogs. Parameters ---------- obsHistID : int obsHistID for the desired visit from the opsim db file. band : str Desired LSST filter to use, ugrizy. boundLength : float Radius in degrees of sky cone in which to produce objects. outfile : str, optional File name of the instance catalog to be produced. If None, a default name will be generated, e.g., phosim_input_0000230_r_0.3deg.txt. """ if outfile is None: outfile = 'phosim_input_%07i_%s_%.1fdeg.txt' % (obsHistID, band, boundLength) obs_md = self.gen.getObservationMetaData(obsHistID=obsHistID, boundLength=boundLength)[0] do_header = True for objid in self.star_objs: self.logger.info("processing %s", objid) db_obj = CatalogDBObject.from_objid(objid, **self.db_config) phosim_object = PhoSimCatalogPoint(db_obj, obs_metadata=obs_md) if do_header: with open(outfile, 'w') as file_obj: phosim_object.write_header(file_obj) do_header = False phosim_object.write_catalog(outfile, write_mode='a', write_header=False, chunk_size=20000) for objid in self.gal_objs: self.logger.info("processing %s", objid) db_obj = CatalogDBObject.from_objid(objid, **self.db_config) phosim_object = PhoSimCatalogSersic2D(db_obj, obs_metadata=obs_md) phosim_object.write_catalog(outfile, write_mode='a', write_header=False, chunk_size=20000)
def test_ssm_catalog_creation(self): t = time.time() # Fake opsim data. database = os.path.join(getPackageDir('SIMS_DATA'), 'OpSimData/opsimblitz1_1133_sqlite.db') generator = ObservationMetaDataGenerator(database=database, driver='sqlite') night = 20 query = 'select min(expMJD), max(expMJD) from summary where night=%d' % (night) res = generator.opsimdb.execute_arbitrary(query) expMJD_min = res[0][0] expMJD_max = res[0][1] obsMetaDataResults = generator.getObservationMetaData(expMJD=(expMJD_min, expMJD_max), limit=3, boundLength=2.2) dt, t = dtime(t) print('To query opsim database: %f seconds' % (dt)) write_header = True write_mode = 'w' try: ssmObj = SolarSystemObj() for obsMeta in obsMetaDataResults: # But moving objects databases are not currently complete for all years. # Push forward to night=747. # (note that we need the phosim dictionary as well) newMJD = 59590.2 # this MJD is artificially chosen to be in the # time span of the new baseline simulated survey obs = ObservationMetaData(mjd=newMJD, pointingRA=obsMeta.pointingRA, pointingDec=obsMeta.pointingDec, bandpassName=obsMeta.bandpass, rotSkyPos=obsMeta.rotSkyPos, m5=obsMeta.m5[obsMeta.bandpass], seeing=obsMeta.seeing[obsMeta.bandpass], boundLength=obsMeta.boundLength, boundType=obsMeta.boundType) obs._OpsimMetaData = {'visitExpTime': 30} mySsmDb = ssmCatCamera(ssmObj, obs_metadata = obs) photParams = PhotometricParameters(exptime = obs.OpsimMetaData['visitExpTime'], nexp=1, bandpass=obs.bandpass) mySsmDb.photParams = photParams try: with lsst.utils.tests.getTempFilePath('.txt') as output_cat: mySsmDb.write_catalog(output_cat, write_header=write_header, write_mode=write_mode) # verify that we did not write an empty catalog with open(output_cat, 'r') as input_file: lines = input_file.readlines() msg = 'MJD is %.3f' % obs.mjd.TAI self.assertGreater(len(lines), 1, msg=msg) except: # This is because the solar system object 'tables' # don't actually connect to tables on fatboy; they just # call methods stored on fatboy. Therefore, the connection # failure will not be noticed until this part of the test msg = sys.exc_info()[1].args[0] if 'DB-Lib error' in msg: reassure() continue else: raise write_mode = 'a' write_header = False dt, t = dtime(t) print('To query solar system objects: %f seconds (obs MJD time %f)' % (dt, obs.mjd.TAI)) except: trace = traceback.extract_tb(sys.exc_info()[2], limit=20) msg = sys.exc_info()[1].args[0] if 'Failed to connect' in msg or failedOnFatboy(trace): # if the exception was because of a failed connection # to fatboy, ignore it. reassure() pass else: raise
import pickle import os import time from lsst.sims.catUtils.utils import ObservationMetaDataGenerator opsim_dir = '/global/projecta/projectdirs/lsst/groups/SSim/DC2' opsim_file = os.path.join(opsim_dir, 'minion_1016_desc_dithered_v4.db') assert os.path.isfile(opsim_file) out_file = os.path.join(os.environ['SCRATCH'], 'minion_1016_desc_dithered_dict.p') obs_gen = ObservationMetaDataGenerator(opsim_file) t_start = time.time() obs_md = obs_gen.getObservationMetaData(boundLength=2.1, boundType='circle', obsHistID=(-10, 1000000000)) print('getting records took %e' % (time.time() - t_start)) out_dict = {} t_start = time.time() for obs in obs_md: out_dict[obs.OpsimMetaData['obsHistID']] = obs with open(out_file, 'wb') as out_file: pickle.dump(out_dict, out_file) print('output took %e' % (time.time() - t_start))
def testQueryOnRanges(self): """ Test that ObservationMetaData objects returned by queries of the form min < value < max are, in fact, within that range. Test when querying on both a single and two columns. """ gen = ObservationMetaDataGenerator() #An list containing the bounds of our queries. #The order of the tuples must correspond to the order of #self.columnMapping in ObservationMetaDataGenerator. #This was generated with a separate script which printed #the median and maximum values of all of the quantities #in our test opsim database bounds = [ ('obsHistID',(5973, 7000)), ('fieldRA',(numpy.degrees(1.370916), numpy.degrees(1.40))), ('rawSeeing',(0.728562, 0.9)), ('seeing', (0.7, 0.9)), ('dist2Moon',(numpy.degrees(1.570307), numpy.degrees(1.9))), ('expMJD',(49367.129396, 49370.0)), ('airmass',(1.420459, 1.6)), ('m5',(22.815249, 23.0)), ('skyBrightness',(19.017605, 19.5))] #test querying on a single column for line in bounds: tag = line[0] # find the index of the entry in columnMapping that # corresponds to this bound for ii in range(len(gen.columnMapping)): if gen.columnMapping[ii][0] == tag: break if tag != 'telescopeFilter' and tag != 'visitExpTime': args = {} args[tag] = line[1] results = gen.getObservationMetaData(**args) if tag == 'skyBrightness': ct = 0 for obs_metadata in results: self.assertLess(obs_metadata.skyBrightness, line[1][1]) self.assertGreater(obs_metadata.skyBrightness, line[1][0]) ct += 1 self.assertGreater(ct, 0) elif tag == 'm5': ct = 0 for obs_metadata in results: self.assertLess(obs_metadata.m5[obs_metadata.bandpass], line[1][1]) self.assertGreater(obs_metadata.m5[obs_metadata.bandpass], line[1][0]) ct += 1 self.assertGreater(ct, 0) name = gen.columnMapping[ii][2] if name is not None: if gen.columnMapping[ii][4] is not None: xmin = gen.columnMapping[ii][4](line[1][0]) xmax = gen.columnMapping[ii][4](line[1][1]) else: xmin = line[1][0] xmax = line[1][1] ct = 0 for obs_metadata in results: ct += 1 self.assertLess(obs_metadata.phoSimMetaData[name][0], xmax) self.assertGreater(obs_metadata.phoSimMetaData[name][0], xmin) #make sure that we did not accidentally choose values such that #no ObservationMetaData were ever returned self.assertGreater(ct, 0) #test querying on two columns at once ct = 0 for ix in range(len(bounds)): tag1 = bounds[ix][0] for ii in range(len(gen.columnMapping)): if gen.columnMapping[ii][0] == tag1: break if tag1 != 'telescopeFilter' and tag1 != 'visitExpTime': name1 = gen.columnMapping[ii][2] if gen.columnMapping[ii][4] is not None: xmin = gen.columnMapping[ii][4](bounds[ix][1][0]) xmax = gen.columnMapping[ii][4](bounds[ix][1][1]) else: xmin = bounds[ix][1][0] xmax = bounds[ix][1][1] for jx in range(ii+1, len(bounds)): tag2 = bounds[jx][0] for jj in range(len(gen.columnMapping)): if gen.columnMapping[jj][0] == tag2: break if tag2 != 'telescopeFilter' and tag2 != 'visitExpTime': name2 = gen.columnMapping[jj][2] if gen.columnMapping[jj][4] is not None: ymin = gen.columnMapping[jj][4](bounds[jx][1][0]) ymax = gen.columnMapping[jj][4](bounds[jx][1][1]) else: ymin = bounds[jx][1][0] ymax = bounds[jx][1][1] args = {} args[tag1] = bounds[ix][1] args[tag2] = bounds[jx][1] results = gen.getObservationMetaData(**args) if name1 is not None or name2 is not None: for obs_metadata in results: ct += 1 if name1 is not None: self.assertGreater(obs_metadata.phoSimMetaData[name1][0], xmin) self.assertLess(obs_metadata.phoSimMetaData[name1][0], xmax) if name2 is not None: self.assertGreater(obs_metadata.phoSimMetaData[name2][0], ymin) self.assertLess(obs_metadata.phoSimMetaData[name2][0], ymax) #Make sure that we didn't choose values such that no ObservationMetaData were #ever returned self.assertGreater(ct, 0)
'lsst_r', 'lsst_i', 'starnotgal', 'isvariable'] default_columns = [('isresolved', 0, int), ('isvariable', 0, int)] default_formats = {'S': '%s', 'f': '%.8f', 'i': '%i'} transformations = {'raJ2000': numpy.degrees, 'decJ2000': numpy.degrees} if __name__ == '__main__': parser = argparse.ArgumentParser(description='Generate the reference catalog') parser.add_argument('opsimDB', help='OpSim database sqlite file') parser.add_argument('-o', '--outfile', type=str, default='twinkles_ref.txt', help='Filename of output reference catalog') args = parser.parse_args() # you need to provide ObservationMetaDataGenerator with the connection # string to an OpSim output database. This is the connection string # to a test database that comes when you install CatSim. generator = ObservationMetaDataGenerator(database=args.opsimDB, driver='sqlite') obsMetaDataResults = generator.getObservationMetaData(fieldRA=(53, 54), fieldDec=(-29, -27), boundLength=0.3) # First get the reference catalog stars = CatalogDBObject.from_objid('allstars') while True: try: ref_stars = TwinklesReference(stars, obs_metadata=obsMetaDataResults[0]) break except RuntimeError: continue ref_stars.write_catalog(args.outfile, write_mode='w', write_header=True, chunk_size=20000)
def test_agn_light_curves(self): """ Test the AgnLightCurveGenerator by generating some AGN light curves and comparing them to the results obtained by generating a series of InstanceCatalogs containing the same objects at the same MJDs """ raRange = (78.0, 85.0) decRange = (-69.0, -65.0) bandpass = '******' lc_gen = AgnLightCurveGenerator(self.agn_db, self.opsimDb) pointings = lc_gen.get_pointings(raRange, decRange, bandpass=bandpass) for row in pointings: for obs in row: mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) obs.mjd = mjd test_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings) self.assertGreater(len(test_light_curves), 2) # make sure we got some light curves for unique_id in test_light_curves: # verify that the sources returned all do vary by making sure that the # np.diff run on the magnitudes reutrns something non-zero self.assertGreater(np.abs(np.diff(test_light_curves[unique_id][bandpass]['mag'])).max(), 0.0) self.assertGreater(len(test_light_curves[unique_id][bandpass]['mjd']), 0) # Now test that specifying a small chunk_size does not change the output # light curves chunk_light_curves, truth_info = lc_gen.light_curves_from_pointings(pointings, chunk_size=1) self.assertGreater(len(chunk_light_curves), 2) for unique_id in test_light_curves: self.assertEqual(len(test_light_curves[unique_id][bandpass]['mjd']), len(chunk_light_curves[unique_id][bandpass]['mjd'])) np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mjd'], chunk_light_curves[unique_id][bandpass]['mjd']) np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['mag'], chunk_light_curves[unique_id][bandpass]['mag']) np.testing.assert_array_equal(test_light_curves[unique_id][bandpass]['error'], chunk_light_curves[unique_id][bandpass]['error']) # Now find all of the ObservationMetaData that were included in our # light curves, generate InstanceCatalogs from them separately, # and verify that the contents of the InstanceCatalogs agree with # the contents of the light curves. gen = ObservationMetaDataGenerator(database=self.opsimDb, driver='sqlite') obs_list = gen.getObservationMetaData(fieldRA=raRange, fieldDec=decRange, telescopeFilter=bandpass, boundLength=1.75) for obs in obs_list: mjd = ModifiedJulianDate(TAI=obs.mjd.TAI-49000.0+59580.0) obs.mjd = mjd ct = 0 for obs in obs_list: cat = agnControlCatalog(self.agn_db, obs_metadata=obs) for agn_obj in cat.iter_catalog(): ct += 1 lc = test_light_curves[agn_obj[0]][bandpass] dex = np.argmin(np.abs(lc['mjd']-obs.mjd.TAI)) self.assertLess(np.abs(lc['mjd'][dex]-obs.mjd.TAI), 1.0e-7) self.assertLess(np.abs(lc['mag'][dex]-agn_obj[3]), 1.0e-7) self.assertLess(np.abs(lc['error'][dex]-agn_obj[4]), 1.0e-7) # Verify that the catalogs and LightCurveGenerator returned the # same number of observations total_ct = 0 for obj_name in test_light_curves: for band in test_light_curves[obj_name]: total_ct += len(test_light_curves[obj_name][band]['mjd']) self.assertEqual(ct, total_ct)
# # So, let us instantiate an ObservationMetaData object, and write the catalog associated with it ### Building Light Curves from Instance Catalogs # We would like to create a number of supernova instance catalogs and then build the light curves from the catalogs. To do this correctly, we would like to use the `observation_metadata` associated with a number of conscutive OpSIM pointings. # In[34]: opsimPath = os.path.join(eups.productDir('sims_data'),'OpSimData') opsimDB = os.path.join(opsimPath,'opsimblitz1_1133_sqlite.db') # from Tuscon AHM notebook from Scott # This OPSIM DB is provided in sims_data. This creates a list of opsim pointings # that I have checked. This is a tuned notebook generator = ObservationMetaDataGenerator() #database = opsimPath, driver='sqlite') obsMetaDataResults = generator.getObservationMetaData(limit=100, fieldRA=(5.0, 8.0), fieldDec=(-85.,-60.), expMJD=(49300., 49400.), boundLength=0.015, boundType='circle') # In[35]: # How many pointings do we have? print len(obsMetaDataResults) # What are the RA, DEC values of the pointings? And how do they compare with the positions of galaxies we created in the database?
class ssmCatCamera(ssmCat): column_outputs = basic_columns + ['chipName'] camera = LsstSimMapper().camera cannot_be_null = ['visibility', 'chipName'] transformations = {'raJ2000': np.degrees, 'decJ2000': np.degrees, 'velRa': np.degrees, 'velDec': np.degrees} default_formats = {'f':'%.13f'} ###### t = time.time() # Get opsim data. opsdb = '/Users/lynnej/opsim/db/enigma_1189_sqlite.db' generator = ObservationMetaDataGenerator(database=opsdb, driver='sqlite') night = 747 query = 'select min(expMJD), max(expMJD) from summary where night=%d' %(night) res = generator.opsimdb.execute_arbitrary(query) expMJD_min = res[0][0] expMJD_max = res[0][1] #obsMetaDataResults = generator.getObservationMetaData(expMJD=(expMJD_min, expMJD_max), boundLength=2.2) # Test image (deep, r band, near ecliptic) obsMetaDataResults = generator.getObservationMetaData(expMJD=50491.36028, boundLength=2.2) dt, t = dtime(t) print 'To query opsim database: %f seconds' %(dt)
def setUpClass(cls): print('setting up %s' % sims_clean_up.targets) cls.camera = obs_lsst_phosim.PhosimMapper().camera # These represent the dimmest magnitudes at which objects # are considered visible in each of the LSST filters # (taken from Table 2 of the overview paper) cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) cls.opsim_db = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'opsimblitz1_1133_sqlite.db') rng = np.random.RandomState(8123) obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db) cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2)) cls.obs_list = rng.choice(cls.obs_list, 10, replace=False) fieldid_list = [] for obs in cls.obs_list: fieldid_list.append(obs.OpsimMetaData['fieldID']) # make sure we have selected observations such that the # same field is revisited more than once assert len(np.unique(fieldid_list)) < len(fieldid_list) cls.input_dir = tempfile.mkdtemp(prefix='alertDataGen', dir=ROOT) cls.star_db_name = tempfile.mktemp(prefix='alertDataGen_star_db', dir=cls.input_dir, suffix='.db') conn = sqlite3.connect(cls.star_db_name) cursor = conn.cursor() cursor.execute('''CREATE TABLE stars (simobjid int, htmid int, ra real, dec real, umag real, gmag real, rmag real, imag real, zmag real, ymag real, px real, pmra real, pmdec real, vrad real, varParamStr text)''') conn.commit() n_stars = 10 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) id_offset = -n_stars for obs in cls.obs_list: id_offset += n_stars ra_0 = obs.pointingRA dec_0 = obs.pointingDec rr = rng.random_sample(n_stars) theta = rng.random_sample(n_stars)*2.0*np.pi ra = ra_0 + rr*np.cos(theta) dec = dec_0 + rr*np.sin(theta) var_period = rng.random_sample(n_stars)*0.25 var_amp = rng.random_sample(n_stars)*1.0 + 0.01 subset = rng.randint(0, high=len(var_amp)-1, size=3) var_amp[subset[:2]] = 0.0 var_amp[subset[-1]] = -1.0 umag = rng.random_sample(n_stars)*5.0 + 15.0 gmag = rng.random_sample(n_stars)*5.0 + 15.0 rmag = rng.random_sample(n_stars)*5.0 + 15.0 imag = rng.random_sample(n_stars)*5.0 + 15.0 zmag = rng.random_sample(n_stars)*5.0 + 15.0 ymag = rng.random_sample(n_stars)*5.0 + 15.0 px = rng.random_sample(n_stars)*0.1 # say it is arcsec pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr vrad = rng.random_sample(n_stars)*600.0 - 300.0 subset = rng.randint(0, high=n_stars-1, size=3) umag[subset] = 40.0 gmag[subset] = 40.0 rmag[subset] = 40.0 imag[subset] = 40.0 zmag[subset] = 40.0 ymag[subset] = 40.0 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6) cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6) u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4) g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4) r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4) i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4) z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4) y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4) cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4) cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4) cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4) cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4) cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4) cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4) cls.max_str_len = -1 for i_star in range(n_stars): if var_amp[i_star] >= -0.1: varParamStr = ('{"m":"alert_test", "p":{"amp":%.4f, "per": %.4f}}' % (var_amp[i_star], var_period[i_star])) else: varParamStr = 'None' if len(varParamStr) > cls.max_str_len: cls.max_str_len = len(varParamStr) htmid = findHtmid(ra[i_star], dec[i_star], 21) query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, '%s')''' % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star], umag[i_star], gmag[i_star], rmag[i_star], imag[i_star], zmag[i_star], ymag[i_star], px[i_star], pmra[i_star], pmdec[i_star], vrad[i_star], varParamStr)) cursor.execute(query) conn.commit() conn.close() cls.output_dir = tempfile.mkdtemp(dir=ROOT, prefix='alert_gen_output') cls.mag0_truth_dict = {} cls.mag0_truth_dict[0] = u_truth cls.mag0_truth_dict[1] = g_truth cls.mag0_truth_dict[2] = r_truth cls.mag0_truth_dict[3] = i_truth cls.mag0_truth_dict[4] = z_truth cls.mag0_truth_dict[5] = y_truth
class testGalSimAgn(GalSimAgn): bandpassNames = ['u', 'g'] #defined in galSimInterface/galSimUtilities.py PSF = SNRdocumentPSF() #If you want to use the LSST camera, uncomment the line below. #You can similarly assign any camera object you want here #camera = LsstSimMapper().camera #select an OpSim pointing opsimdb = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'opsimblitz1_1133_sqlite.db') obs_gen = ObservationMetaDataGenerator(database=opsimdb) obs_list = obs_gen.getObservationMetaData(obsHistID=10, boundLength=0.05) obs_metadata = obs_list[0] #grab a database of galaxies (in this case, galaxy bulges) stars = CatalogDBObject.from_objid('allstars') #now append a bunch of objects with 2D sersic profiles to our output file stars_galSim = testGalSimStars(stars, obs_metadata=obs_metadata) catName = 'galSim_compound_example.txt' stars_galSim.write_catalog(catName, chunk_size=100) print('done with stars') bulges = CatalogDBObject.from_objid('galaxyBulge')
from lsst.sims.catUtils.utils import ObservationMetaDataGenerator from lsst.sims.coordUtils import pupilCoordsFromFocalPlaneCoordsLSST from lsst.sims.utils import raDecFromPupilCoords from lsst.sims.utils import angularSeparation from lsst.sims.utils import Site from lsst.sims.photUtils import BandpassDict, Sed from lsst.sims.utils import altAzPaFromRaDec from lsst.sims.utils import ObservationMetaData bp_dict, hw_dict = BandpassDict.loadBandpassesFromFiles() opsimdb = os.path.join('/Users/danielsf/physics/lsst_150412', 'Development', 'garage', 'OpSimData', 'minion_1016_sqlite.db') obs_gen = ObservationMetaDataGenerator(opsimdb) obs_list = obs_gen.getObservationMetaData(moonAlt=(-90.0, -50.0), altitude=(55.0, 57.0), fieldDec=(-10.0, 10.0)) assert len(obs_list) > 0 obs_root = obs_list[0] obs_root.site = Site(name='LSST', pressure=0.0, humidity=0.0) phosim_header = DefaultPhoSimHeaderMap phosim_header['nsnap'] = 1 phosim_header['vistime'] = 30.0 galaxy_dir = os.path.join(getPackageDir('sims_sed_library'), 'galaxySED') galaxy_sed_list = os.listdir(galaxy_dir)
args = parser.parse_args() if args.out_dir is None: raise RuntimeError('must specify out_dir') if args.log_file is None: raise RuntimeError('must specify log file') if os.path.exists(args.log_file): raise RuntimeError('%s already exists' % args.log_file) if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) # get the list of ObservationMetaData to simulate obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite') obs_list = obs_gen.getObservationMetaData(night=(args.night0,args.night1)) del obs_gen sims_clean_up() gc.collect() # get the list of trixel htmids to simulate alert_gen = AlertDataGenerator() alert_gen.subdivide_obs(obs_list, htmid_level=6) n_tot_obs=0 for htmid in alert_gen.htmid_list: n_tot_obs += alert_gen.n_obs(htmid) with open(args.log_file, 'a') as out_file:
def __init__(self, opsimdb, descqa_catalog, dither=True, min_mag=10, minsource=100, proper_motion=False, protoDC2_ra=0, protoDC2_dec=0, star_db_name=None, sed_lookup_dir=None, agn_db_name=None, agn_threads=1, sn_db_name=None, sprinkler=False, host_image_dir=None, host_data_dir=None, config_dict=None, gzip_threads=3, objects_to_skip=()): """ Parameters ---------- opsimdb: str OpSim db filename. descqa_catalog: str Name of the DESCQA galaxy catalog. dither: bool [True] Flag to enable the dithering included in the opsim db file. min_mag: float [10] Minimum value of the star magnitude at 500nm to include. minsource: int [100] Minimum number of objects for phosim.py to simulate a chip. proper_motion: bool [True] Flag to enable application of proper motion to stars. protoDC2_ra: float [0] Desired RA (J2000 degrees) of protoDC2 center. protoDC2_dec: float [0] Desired Dec (J2000 degrees) of protoDC2 center. star_db_name: str [None] Filename of the database containing stellar sources sed_lookup_dir: str [None] Directory where the SED lookup tables reside. agn_db_name: str [None] Filename of the agn parameter sqlite db file. agn_threads: int [1] Number of threads to use when simulating AGN variability sn_db_name: str [None] Filename of the supernova parameter sqlite db file. sprinkler: bool [False] Flag to enable the Sprinkler. host_image_dir: string The location of the FITS images of lensed AGN/SNe hosts produced by generate_lensed_hosts_***.py host_data_dir: string Location of csv file of lensed host data created by the sprinkler gzip_threads: int The number of gzip jobs that can be started in parallel after catalogs are written (default=3) objects_to_skip: set-like or list-like [()] Collection of object types to skip, e.g., stars, knots, bulges, disks, sne, agn """ self.t_start = time.time() if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) self.gzip_threads = gzip_threads # load the data for the parametrized light # curve stellar variability model into a # global cache plc = ParametrizedLightCurveMixin() plc.load_parametrized_light_curves() self.config_dict = config_dict if config_dict is not None else {} self.descqa_catalog = descqa_catalog self.dither = dither self.min_mag = min_mag self.minsource = minsource self.proper_motion = proper_motion self.protoDC2_ra = protoDC2_ra self.protoDC2_dec = protoDC2_dec self.phot_params = PhotometricParameters(nexp=1, exptime=30) self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() self.obs_gen = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') if star_db_name is None: raise IOError("Need to specify star_db_name") if not os.path.isfile(star_db_name): raise IOError("%s is not a file\n" % star_db_name + "(This is what you specified for star_db_name") self.star_db = DC2StarObj(database=star_db_name, driver='sqlite') self.sprinkler = sprinkler if self.sprinkler and not HAS_TWINKLES: raise RuntimeError("You are trying to enable the sprinkler; " "but Twinkles cannot be imported") if not os.path.isdir(sed_lookup_dir): raise IOError("\n%s\nis not a dir" % sed_lookup_dir) self.sed_lookup_dir = sed_lookup_dir self._agn_threads = agn_threads if agn_db_name is not None: if os.path.exists(agn_db_name): self.agn_db_name = agn_db_name else: raise IOError("Path to Proto DC2 AGN database does not exist.") else: self.agn_db_name = None self.sn_db_name = None if sn_db_name is not None: if os.path.isfile(sn_db_name): self.sn_db_name = sn_db_name else: raise IOError("%s is not a file" % sn_db_name) if host_image_dir is None and self.sprinkler is not False: raise IOError( "Need to specify the name of the host image directory.") elif self.sprinkler is not False: if os.path.exists(host_image_dir): self.host_image_dir = host_image_dir else: raise IOError("Path to host image directory" + "\n\n%s\n\n" % host_image_dir + "does not exist.") if host_data_dir is None and self.sprinkler is not False: raise IOError( "Need to specify the name of the host data directory.") elif self.sprinkler is not False: if os.path.exists(host_data_dir): self.host_data_dir = host_data_dir else: raise IOError( "Path to host data directory does not exist.\n\n", "%s\n\n" % host_data_dir) self.instcats = get_instance_catalogs() object_types = 'stars knots bulges disks sprinkled hosts sne agn'.split( ) if any([_ not in object_types for _ in objects_to_skip]): raise RuntimeError(f'objects_to_skip ({objects_to_skip}) ' 'contains invalid object types') self.do_obj_type = {_: _ not in objects_to_skip for _ in object_types}
def generatePhosimInput(mode='a', runobsHistID=None): if mode == 'a': filewrite = 'append' elif mode == 'c': filewrite = 'clobber' opsimDB = os.path.join('.','kraken_1042_sqlite.db') logfilename = 'run.log' if os.path.isfile(logfilename): if filewrite =='append': pass elif filewrite == 'clobber': with open('run.log', 'w') as f: f.write('obsHistID,status,timestamp\n') else: print('file exists and mode uncertain') exit() else: with open('run.log', 'w') as f: f.write('obsHistID,status,time\n') #you need to provide ObservationMetaDataGenerator with the connection #string to an OpSim output database. This is the connection string #to a test database that comes when you install CatSim. generator = ObservationMetaDataGenerator(database=opsimDB, driver='sqlite') obsHistIDList = numpy.genfromtxt('FirstSet_obsHistIDs.csv', delimiter=',', usecols=0) obsMetaDataResults = [] # Change the slicing in this line for the range of visits for obsHistID in obsHistIDList[600:700]: if runobsHistID is not None: obsHistID = runobsHistID obsMetaDataResults.append(generator.getObservationMetaData(obsHistID=obsHistID, fieldRA=(53, 54), fieldDec=(-29, -27), boundLength=0.3)[0]) starObjNames = ['msstars', 'bhbstars', 'wdstars', 'rrlystars', 'cepheidstars'] snmodel = SNObj() for obs_metadata in obsMetaDataResults: filename = "InstanceCatalogs / phosim_input_%s.txt" \ %(obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) obs_metadata.phoSimMetaData['SIM_NSNAP'] = (1, numpy.dtype(int)) obs_metadata.phoSimMetaData['SIM_VISTIME'] = (30, numpy.dtype(float)) print('Starting Visit: ', obs_metadata.phoSimMetaData['Opsim_obshistid'][0]) compoundStarDBList = [MsStarObj, BhbStarObj, WdStarObj, RRLyStarObj, CepheidStarObj] compoundGalDBList = [GalaxyBulgeObj, GalaxyDiskObj, GalaxyAgnObj] compoundStarICList = [PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint, PhoSimCatalogPoint] compoundGalICList = [PhoSimCatalogSersic2D, PhoSimCatalogSersic2D, TwinklesCatalogZPoint] snphosim = PhoSimCatalogSN(db_obj=snmodel, obs_metadata=obs_metadata, column_outputs=['EBV']) snphosim.writeSedFile = True snphosim.suppressDimSN = True snphosim.prefix = 'spectra_files/' while True: try: starCat = CompoundInstanceCatalog(compoundStarICList, compoundStarDBList, obs_metadata=obs_metadata, constraint='gmag > 11.', compoundDBclass=sprinklerCompound) starCat.write_catalog(filename, chunk_size=10000) galCat = CompoundInstanceCatalog(compoundGalICList, compoundGalDBList, obs_metadata=obs_metadata, # constraint='g_ab > 11.', compoundDBclass=sprinklerCompound) galCat.write_catalog(filename, write_mode='a', write_header=False, chunk_size=10000) snphosim.write_catalog(filename, write_header=False, write_mode='a', chunk_size=10000) if runobsHistID is not None: print('Done doing requested obsHistID') sys.exit() with open(logfilename, 'a') as f: f.write('{0:d},DONE,{1:3.6f}\n'.format(obs_metadata.phoSimMetaData['Opsim_obshistid'][0], time.time())) break except RuntimeError: continue print("Finished Writing Visit: ", obs_metadata.phoSimMetaData['Opsim_obshistid'][0])
AperatureType = 'circle' AperatureRadius = 0.5 #FoV in degrees #AperatureRadius = 1.75 #LSST's Actual FoV in degrees SearchRegionRA = (-30.0,-20.0) SearchRegionDec = (-30.0,-20.0) SearchAirmass = (1.0,1.5) DesiredFilter = None import eups import os from lsst.sims.catUtils.utils import ObservationMetaDataGenerator from lsst.sims.catalogs.generation.db import ObservationMetaData #help(ObservationMetaDataGenerator) opsimdb = os.path.join(eups.productDir('sims_data'),'OpSimData','enigma_1189_sqlite.db') gen = ObservationMetaDataGenerator(driver='sqlite', database=opsimdb) SimObData = gen.getObservationMetaData(boundType=AperatureType, boundLength=AperatureRadius, fieldRA=SearchRegionRA, fieldDec=SearchRegionDec, airmass=SearchAirmass, telescopeFilter=DesiredFilter) NumOfObservations = len(SimObData) #print SimsObData[0].__dict__ from prettytable import PrettyTable UniquePointings = list({(o.unrefractedRA,o.unrefractedDec) for o in SimObData}) NumOfPointings = len(UniquePointings) print 'Number of Unique Pointings:', NumOfPointings,'\n' table2 = PrettyTable(["Pointing RA","Pointing Dec"]) for x in UniquePointings: table2.add_row([x[0], x[1]]) print table2
def testQueryExactValues(self): """ Test that ObservationMetaData returned by a query demanding an exact value do, in fact, adhere to that requirement. """ gen = ObservationMetaDataGenerator() bounds = [ ('obsHistID',5973), ('expDate',1220779), ('fieldRA',numpy.degrees(1.370916)), ('fieldDec',numpy.degrees(-0.456238)), ('moonRA',numpy.degrees(2.914132)), ('moonDec',numpy.degrees(0.06305)), ('rotSkyPos',numpy.degrees(3.116656)), ('telescopeFilter','i'), ('rawSeeing',0.728562), ('seeing', 0.88911899999999999), ('sunAlt',numpy.degrees(-0.522905)), ('moonAlt',numpy.degrees(0.099096)), ('dist2Moon',numpy.degrees(1.570307)), ('moonPhase',52.2325), ('expMJD',49367.129396), ('altitude',numpy.degrees(0.781015)), ('azimuth',numpy.degrees(3.470077)), ('visitExpTime',30.0), ('airmass',1.420459), ('m5',22.815249), ('skyBrightness',19.017605)] for ii in range(len(bounds)): tag = bounds[ii][0] if tag != 'telescopeFilter' and tag != 'visitExpTime': name = gen.columnMapping[ii][2] args = {} args[tag] = bounds[ii][1] results = gen.getObservationMetaData(**args) if gen.columnMapping[ii][4] is not None: value = gen.columnMapping[ii][4](bounds[ii][1]) else: value = bounds[ii][1] if name is not None: ct = 0 for obs_metadata in results: self.assertAlmostEqual(value, obs_metadata.phoSimMetaData[name][0],10) ct += 1 #Make sure that we did not choose a value which returns zero ObservationMetaData self.assertGreater(ct, 0) elif tag == 'm5': ct = 0 for obs_metadata in results: self.assertAlmostEqual(value, obs_metadata.m5.values()[0]) ct += 1 self.assertGreater(ct, 0) elif tag == 'seeing': ct = 0 for obs_metadata in results: self.assertAlmostEqual(value, obs_metadata.seeing.values()[0]) ct += 1 self.assertGreater(ct, 0)
help='field of view radius in degrees') args = parser.parse_args() obshistid_list = args.id opsimdb = args.db out_dir = args.out dither_switch = True if args.dither.lower()[0] == 'f': dither_switch = False from lsst.sims.catUtils.utils import ObservationMetaDataGenerator if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) obs_generator = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogSersic2D from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogZPoint from lsst.sims.catUtils.exampleCatalogDefinitions import DefaultPhoSimHeaderMap from lsst.sims.catUtils.baseCatalogModels import StarObj from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj, GalaxyDiskObj from lsst.sims.catUtils.baseCatalogModels import GalaxyAgnObj from lsst.sims.utils import _getRotSkyPos import copy star_db = StarObj(database='LSSTCATSIM', host='fatboy.phys.washington.edu', port=1433, driver='mssql+pymssql') bulge_db = GalaxyBulgeObj(connection=star_db.connection) disk_db = GalaxyDiskObj(connection=star_db.connection)
def setUpClass(cls): print('setting up %s' % sims_clean_up.targets) # These represent the dimmest magnitudes at which objects # are considered visible in each of the LSST filters # (taken from Table 2 of the overview paper) cls.obs_mag_cutoff = (23.68, 24.89, 24.43, 24.0, 24.45, 22.60) cls.opsim_db = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'opsimblitz1_1133_sqlite.db') rng = np.random.RandomState(8123) obs_gen = ObservationMetaDataGenerator(database=cls.opsim_db) cls.obs_list = obs_gen.getObservationMetaData(night=(0, 2)) cls.obs_list = rng.choice(cls.obs_list, 10, replace=False) fieldid_list = [] for obs in cls.obs_list: fieldid_list.append(obs.OpsimMetaData['fieldID']) # make sure we have selected observations such that the # same field is revisited more than once assert len(np.unique(fieldid_list)) < len(fieldid_list) cls.input_dir = tempfile.mkdtemp(prefix='alertDataGen', dir=ROOT) cls.star_db_name = tempfile.mktemp(prefix='alertDataGen_star_db', dir=cls.input_dir, suffix='.db') conn = sqlite3.connect(cls.star_db_name) cursor = conn.cursor() cursor.execute('''CREATE TABLE stars (simobjid int, htmid int, ra real, dec real, umag real, gmag real, rmag real, imag real, zmag real, ymag real, px real, pmra real, pmdec real, vrad real, varParamStr text)''') conn.commit() n_stars = 10 cls.ra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.dec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) u_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) g_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) r_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) i_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) z_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) y_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.px_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.pmra_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.pmdec_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.vrad_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.amp_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) cls.period_truth = np.zeros(n_stars*len(cls.obs_list), dtype=float) id_offset = -n_stars for obs in cls.obs_list: id_offset += n_stars ra_0 = obs.pointingRA dec_0 = obs.pointingDec rr = rng.random_sample(n_stars) theta = rng.random_sample(n_stars)*2.0*np.pi ra = ra_0 + rr*np.cos(theta) dec = dec_0 + rr*np.sin(theta) var_period = rng.random_sample(n_stars)*0.25 var_amp = rng.random_sample(n_stars)*1.0 + 0.01 subset = rng.randint(0, high=len(var_amp)-1, size=3) var_amp[subset[:2]] = 0.0 var_amp[subset[-1]] = -1.0 umag = rng.random_sample(n_stars)*5.0 + 15.0 gmag = rng.random_sample(n_stars)*5.0 + 15.0 rmag = rng.random_sample(n_stars)*5.0 + 15.0 imag = rng.random_sample(n_stars)*5.0 + 15.0 zmag = rng.random_sample(n_stars)*5.0 + 15.0 ymag = rng.random_sample(n_stars)*5.0 + 15.0 px = rng.random_sample(n_stars)*0.1 # say it is arcsec pmra = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr pmdec = rng.random_sample(n_stars)*50.0+100.0 # say it is arcsec/yr vrad = rng.random_sample(n_stars)*600.0 - 300.0 subset = rng.randint(0, high=n_stars-1, size=3) umag[subset] = 40.0 gmag[subset] = 40.0 rmag[subset] = 40.0 imag[subset] = 40.0 zmag[subset] = 40.0 ymag[subset] = 40.0 cls.ra_truth[id_offset:id_offset+n_stars] = np.round(ra, decimals=6) cls.dec_truth[id_offset:id_offset+n_stars] = np.round(dec, decimals=6) u_truth[id_offset:id_offset+n_stars] = np.round(umag, decimals=4) g_truth[id_offset:id_offset+n_stars] = np.round(gmag, decimals=4) r_truth[id_offset:id_offset+n_stars] = np.round(rmag, decimals=4) i_truth[id_offset:id_offset+n_stars] = np.round(imag, decimals=4) z_truth[id_offset:id_offset+n_stars] = np.round(zmag, decimals=4) y_truth[id_offset:id_offset+n_stars] = np.round(ymag, decimals=4) cls.px_truth[id_offset:id_offset+n_stars] = np.round(px, decimals=4) cls.pmra_truth[id_offset:id_offset+n_stars] = np.round(pmra, decimals=4) cls.pmdec_truth[id_offset:id_offset+n_stars] = np.round(pmdec, decimals=4) cls.vrad_truth[id_offset:id_offset+n_stars] = np.round(vrad, decimals=4) cls.amp_truth[id_offset:id_offset+n_stars] = np.round(var_amp, decimals=4) cls.period_truth[id_offset:id_offset+n_stars] = np.round(var_period, decimals=4) cls.max_str_len = -1 for i_star in range(n_stars): if var_amp[i_star] >= -0.1: varParamStr = ('{"m":"alert_test", "p":{"amp":%.4f, "per": %.4f}}' % (var_amp[i_star], var_period[i_star])) else: varParamStr = 'None' if len(varParamStr) > cls.max_str_len: cls.max_str_len = len(varParamStr) htmid = findHtmid(ra[i_star], dec[i_star], 21) query = ('''INSERT INTO stars VALUES(%d, %d, %.6f, %.6f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, %.4f, '%s')''' % (i_star+id_offset+1, htmid, ra[i_star], dec[i_star], umag[i_star], gmag[i_star], rmag[i_star], imag[i_star], zmag[i_star], ymag[i_star], px[i_star], pmra[i_star], pmdec[i_star], vrad[i_star], varParamStr)) cursor.execute(query) conn.commit() conn.close() cls.output_dir = tempfile.mkdtemp(dir=ROOT, prefix='alert_gen_output') cls.mag0_truth_dict = {} cls.mag0_truth_dict[0] = u_truth cls.mag0_truth_dict[1] = g_truth cls.mag0_truth_dict[2] = r_truth cls.mag0_truth_dict[3] = i_truth cls.mag0_truth_dict[4] = z_truth cls.mag0_truth_dict[5] = y_truth
# You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. # # import numpy as np import pandas as pd from lsst.sims.catUtils.utils import ObservationMetaDataGenerator from lsst.sims.utils import angularSeparation from collections import OrderedDict as Odict dbname = '/global/projecta/projectdirs/lsst/groups/SSim/DC2/minion_1016_desc_dithered_v4.db' ObsMetaData = ObservationMetaDataGenerator(database=dbname) def main(ramax=58, ramin=56, decmin=-32, decmax=-31, t0=59215, tm=59945): res = ObsMetaData.getObservationMetaData(boundLength=2, boundType='circle', fieldRA=(ramin-3, ramax+3), fieldDec=(decmin-3, decmax+3), expMJD=(t0, tm)) parsed = [Odict(obsmd.summary['OpsimMetaData']) for obsmd in res \ if obsmd.bandpass in ("g", "r", "i", "z", "y")] df = pd.DataFrame(parsed) X = df[['obsHistID', 'filter', 'FWHMeff', 'descDitheredRA', 'descDitheredDec', 'airmass', 'fiveSigmaDepth', 'expMJD']].copy() X.descDitheredRA = np.degrees(X.descDitheredRA) X.descDitheredDec = np.degrees(X.descDitheredDec)
def setUpClass(cls): # Set directory where scratch work will be done cls.madeScratchDir = False cls.scratchDir = 'scratchSpace' # Setup a directory in which test data will be made if not os.path.exists(cls.scratchDir): os.makedirs(cls.scratchDir) cls.madeScratchDir = True # ObsMetaData instance with spatial window within which we will # put galaxies in a fake galaxy catalog cls.obsMetaDataforCat = ObservationMetaData(boundType='circle', boundLength=np.degrees(0.25), pointingRA=np.degrees(0.13), pointingDec=np.degrees(-1.2), bandpassName=['r'], mjd=49350.) # Randomly generate self.size Galaxy positions within the spatial window # of obsMetaDataforCat cls.dbname = os.path.join(cls.scratchDir, 'galcat.db') cls.size = 1000 cls.GalaxyPositionSamps = sample_obsmetadata( obsmetadata=cls.obsMetaDataforCat, size=cls.size) # Create a galaxy Table overlapping with the obsMetaData Spatial Bounds # using positions from the samples above and a database name given by # self.dbname vals = cls._createFakeGalaxyDB() with open('valsFromTest.dat', 'w') as f: for i, v in enumerate(vals[0]): f.write(str(np.radians(vals[0][i])) + ' ' + str(np.radians(vals[1][i])) + '\n') # fig, ax = plt.subplots() # ax.plot(vals[0][:1000], vals[1][: 1000], '.') # ax.plot([0.13], [-1.2], 'rs', markersize=8) # fig.savefig(os.path.join(cls.scratchDir, 'match_galDBPosns.pdf')) # Read it into a CatalogDBObject galDB class MyGalaxyCatalog(CatalogDBObject): ''' Create a like CatalogDBObject connecting to a local sqlite database ''' objid = 'mytestgals' tableid = 'gals' idColKey = 'id' objectTypeId = 0 appendint = 10000 database = cls.dbname # dbAddress = './testData/galcat.db' raColName = 'raJ2000' decColName = 'decJ2000' driver = 'sqlite' # columns required to convert the ra, dec values in degrees # to radians again columns = [('id', 'id', int), ('raJ2000','raJ2000 * PI()/ 180. '), ('decJ2000','decJ2000 * PI()/ 180.'), ('redshift', 'redshift')] # class galCopy(InstanceCatalog): # column_outputs = ['id', 'raJ2000', 'decJ2000', 'redshift'] # override_formats = {'raJ2000': '%8e', 'decJ2000': '%8e'} cls.galDB = MyGalaxyCatalog(database=cls.dbname) # cls.galphot = galCopy(db_obj=cls.galDB, # obs_metadata=cls.obsMetaDataforCat) # cls.galPhotFname = os.path.join(cls.scratchDir, 'gals.dat') # cls.galphot.write_catalog(cls.galPhotFname) # Generate a set of Observation MetaData Outputs that overlap # the galaxies in space opsimPath = os.path.join(eups.productDir('sims_data'),'OpSimData') opsimDB = os.path.join(opsimPath,'opsimblitz1_1133_sqlite.db') generator = ObservationMetaDataGenerator() cls.obsMetaDataResults = generator.getObservationMetaData(limit=100, fieldRA=(5.0, 8.0), fieldDec=(-85.,-60.), expMJD=(49300., 49400.), boundLength=0.15, boundType='circle') # cls.obsMetaDataResults has obsMetaData corresponding to 15 pointings # This is tested in test_obsMetaDataGeneration # v = zip(*map(cls.coords, cls.obsMetaDataResults)) # fig2, ax2 = plt.subplots() # ax2.plot(vals[0][:1000], vals[1][: 1000], '.') # ax2.plot(v[0], v[1], 'ko', markersize=8) # ax2.axhline(-np.pi, color='k', lw=2) # ax2.axhline(np.pi, color='k', lw=2) # ax2.axvline(0., color='k', lw=2.) # ax2.axvline(2. * np.pi, color='k', lw=2.) # fig2.savefig(os.path.join(cls.scratchDir, 'matchPointings.pdf')) #print 'cls.obsMetaDataforCat' #print cls.obsMetaDataforCat.summary #print 'obsMetaDataResults' # obsMetaDataList = [] # for obsMetaData in tmpobsMetaDataResults: # obsMetaDataList.append(ObservationMetaData(boundType='circle', # boundLength=np.degrees(0.05), # unrefractedRA=np.degrees(0.13), # unrefractedDec=np.degrees(-1.2), # bandpassName=obsMetaData.bandpass, # mjd=obsMetaData.mjd)) # cls.obsMetaDataResults = tmpobsMetaDataResults# pobsMetaDataList # self.catalogList = self._writeManySNCatalogs() sncatalog = SNIaCatalog(db_obj=cls.galDB, obs_metadata=cls.obsMetaDataResults[12], column_outputs=['t0', 'flux_u', 'flux_g', \ 'flux_r', 'flux_i', 'flux_z',\ 'flux_y', 'mag_u', 'mag_g',\ 'mag_r', 'mag_i', 'mag_z', \ 'mag_y', 'adu_u', 'adu_g',\ 'adu_r', 'adu_i', 'adu_z', \ 'adu_y','mwebv']) sncatalog.suppressDimSN = True sncatalog.midSurveyTime = sncatalog.mjdobs - 20. sncatalog.snFrequency = 1.0 cls.fullCatalog = cls.scratchDir + '/testSNCatalogTest.dat' sncatalog.write_catalog(cls.fullCatalog) # Create a SNCatalog based on GalDB, and having times of explosions # overlapping the times in obsMetaData cls.fnameList = cls._writeManySNCatalogs(cls.obsMetaDataResults)
help='Path to OpSim database used for survey cadence') args = parser.parse_args() if args.out_dir is None: raise RuntimeError('must specify out_dir') if args.log_file is None: raise RuntimeError('must specify log file') if os.path.exists(args.log_file): raise RuntimeError('%s already exists' % args.log_file) if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) # get the list of ObservationMetaData to simulate obs_gen = ObservationMetaDataGenerator(args.opsim_db, driver='sqlite') obs_list = obs_gen.getObservationMetaData(night=(args.night0, args.night1)) del obs_gen sims_clean_up() gc.collect() # get the list of trixel htmids to simulate alert_gen = AlertDataGenerator() alert_gen.subdivide_obs(obs_list, htmid_level=6) n_tot_obs = 0 for htmid in alert_gen.htmid_list: n_tot_obs += alert_gen.n_obs(htmid) with open(args.log_file, 'a') as out_file:
def testQueryOnRanges(self): """ Test that ObservationMetaData objects returned by queries of the form min < value < max are, in fact, within that range. Test when querying on both a single and two columns. """ gen = ObservationMetaDataGenerator() #An list containing the bounds of our queries. #The order of the tuples must correspond to the order of #self.columnMapping in ObservationMetaDataGenerator. #This was generated with a separate script which printed #the median and maximum values of all of the quantities #in our test opsim database bounds = [ ('obsHistID',(5973, 11080)), ('expDate',(1220779, 1831593)), ('fieldRA',(numpy.degrees(1.370916), numpy.degrees(1.5348635))), ('fieldDec',(numpy.degrees(-0.456238), numpy.degrees(0.0597905))), ('moonRA',(numpy.degrees(2.914132), numpy.degrees(4.5716525))), ('moonDec',(numpy.degrees(0.06305), numpy.degrees(0.2216745))), ('rotSkyPos',(numpy.degrees(3.116656), numpy.degrees(4.6974265))), ('telescopeFilter',('i','i')), ('rawSeeing',(0.728562, 1.040495)), ('seeing', (0.7, 0.9)), ('sunAlt',(numpy.degrees(-0.522905), numpy.degrees(-0.366073))), ('moonAlt',(numpy.degrees(0.099096), numpy.degrees(0.5495415))), ('dist2Moon',(numpy.degrees(1.570307), numpy.degrees(2.347868))), ('moonPhase',(52.2325, 76.0149785)), ('expMJD',(49367.129396, 49374.1990025)), ('altitude',(numpy.degrees(0.781015), numpy.degrees(1.1433785))), ('azimuth',(numpy.degrees(3.470077), numpy.degrees(4.8765995))), ('visitExpTime',(30.0,30.0)), ('airmass',(1.420459, 2.0048075)), ('m5',(22.815249, 24.0047695)), ('skyBrightness',(19.017605, 20.512553))] #test querying on a single column for (ii,line) in enumerate(bounds): tag = line[0] if tag != 'telescopeFilter' and tag != 'visitExpTime': args = {} args[tag] = line[1] results = gen.getObservationMetaData(**args) if tag == 'skyBrightness': ct = 0 for obs_metadata in results: self.assertTrue(obs_metadata.skyBrightness<line[1][1]) self.assertTrue(obs_metadata.skyBrightness>line[1][0]) ct += 1 self.assertTrue(ct>0) elif tag == 'm5': ct = 0 for obs_metadata in results: self.assertTrue(obs_metadata.m5[obs_metadata.bandpass]<line[1][1]) self.assertTrue(obs_metadata.m5[obs_metadata.bandpass]>line[1][0]) ct += 1 self.assertTrue(ct>0) name = gen.columnMapping[ii][2] if name is not None: if gen.columnMapping[ii][4] is not None: xmin = gen.columnMapping[ii][4](line[1][0]) xmax = gen.columnMapping[ii][4](line[1][1]) else: xmin = line[1][0] xmax = line[1][1] ct = 0 for obs_metadata in results: ct += 1 self.assertTrue(obs_metadata.phoSimMetaData[name][0]<xmax) self.assertTrue(obs_metadata.phoSimMetaData[name][0]>xmin) #make sure that we did not accidentally choose values such that #no ObservationMetaData were ever returned self.assertTrue(ct>0) #test querying on two columns at once ct = 0 for ii in range(len(bounds)): tag1 = bounds[ii][0] if tag1 != 'telescopeFilter' and tag1 != 'visitExpTime': name1 = gen.columnMapping[ii][2] if gen.columnMapping[ii][4] is not None: xmin = gen.columnMapping[ii][4](bounds[ii][1][0]) xmax = gen.columnMapping[ii][4](bounds[ii][1][1]) else: xmin = bounds[ii][1][0] xmax = bounds[ii][1][1] for jj in range(ii+1, len(bounds)): tag2 = bounds[jj][0] if tag2 != 'telescopeFilter' and tag2 != 'visitExpTime': name2 = gen.columnMapping[jj][2] if gen.columnMapping[jj][4] is not None: ymin = gen.columnMapping[jj][4](bounds[jj][1][0]) ymax = gen.columnMapping[jj][4](bounds[jj][1][1]) else: ymin = bounds[jj][1][0] ymax = bounds[jj][1][1] args = {} args[tag1] = bounds[ii][1] args[tag2] = bounds[jj][1] results = gen.getObservationMetaData(**args) if name1 is not None or name2 is not None: for obs_metadata in results: ct += 1 if name1 is not None: self.assertTrue(obs_metadata.phoSimMetaData[name1][0]>xmin) self.assertTrue(obs_metadata.phoSimMetaData[name1][0]<xmax) if name2 is not None: self.assertTrue(obs_metadata.phoSimMetaData[name2][0]>ymin) self.assertTrue(obs_metadata.phoSimMetaData[name2][0]<ymax) #Make sure that we didn't choose values such that no ObservationMetaData were #ever returned self.assertTrue(ct>0)