def create_star_cache(db=None):
"""
Read the stars from a database and cache them in
$TWINKLES_DIR/data/star_cache.db
in the table 'star_cache_table'
Params
------
db is a CatalogDBObject connecting to the database from which to read
the data. If None, this will use the CatSim class StarObj(), which
connects to the star table on fatboy.
"""
star_dtype = np.dtype([('simobjid', int), ('ra', float), ('decl', float),
('magNorm', float), ('mura', float),
('mudecl', float), ('parallax', float),
('ebv', float), ('vrad', float),
('varParamStr', str, 256), ('sedfilename', str, 40),
('gmag', float)])
col_names = list(star_dtype.names)
star_cache_name = os.path.join(getPackageDir('twinkles'), 'data',
'twinkles_star_cache.txt')
star_db_name = os.path.join(getPackageDir('twinkles'), 'data',
'star_cache.db')
if db is None:
db = StarObj()
result_iterator = db.query_columns(colnames=col_names,
chunk_size=100000,
obs_metadata=_obs)
with open(star_cache_name, 'w') as output_file:
output_file.write('# ')
for name in col_names:
output_file.write('%s ' % name)
output_file.write('\n')
for chunk in result_iterator:
for line in chunk:
output_file.write((
'%d;%.17g;%.17g;%.17g;%.17g;%.17g;%.17g;%.17g;%.17g;%s;%s;%.17g\n'
% (line[1], line[2], line[3], line[4], line[5], line[6],
line[7], line[8], line[9], str(line[10]), str(line[11]),
line[12])).replace('nan',
'NULL').replace('None', 'NULL'))
if os.path.exists(star_db_name):
os.unlink(star_db_name)
dbo = fileDBObject(star_cache_name,
driver='sqlite',
runtable='star_cache_table',
database=star_db_name,
dtype=star_dtype,
delimiter=';',
idColKey='simobjid')
if os.path.exists(star_cache_name):
os.unlink(star_cache_name)
def demoLoadFromFile():
"""Demonstrate loading database connection config from config file.
This method can be used to change default connection at runtime,
but only before instantiating a DbObject.
"""
config = BaseCatalogConfig()
configFilename = os.path.join(getPackageDir("sims_catUtils"), "config",
"db.py")
config.load(configFilename)
starDB = StarObj(**config.toDict())
def demoGlobalConfig():
"""Demonstrate changing database connection in script.
This method can be used to change default connections at runtime,
but only before instantiating a DbObject
"""
CONFIG = BaseCatalogConfig(driver='mssql+pymssql',
port='51433',
host='localhost',
database='LSSTCATSIM')
#Can pass to the constructor
starDB = StarObj(**CONFIG.toDict())
def testGalaxyAndStarCatalog(self):
"""
Test GalaxyTileCompoundObj by creating a catalog of galaxy bulges, disks,
agns, and stars using both the 'old fashioned way' (one catalog at a time), and
using CompoundInstanceCatalog
"""
controlFileName = os.path.join(self.baseDir, 'galStar_compound_control.txt')
testFileName = os.path.join(self.baseDir, 'galStar_compound_test.txt')
if os.path.exists(controlFileName):
os.unlink(controlFileName)
if os.path.exists(testFileName):
os.unlink(testFileName)
obs = ObservationMetaData(pointingRA=25.0, pointingDec=-45.0,
boundType='circle', boundLength=0.05)
dbBulge = GalaxyBulgeObj()
dbDisk = GalaxyDiskObj()
dbAgn = GalaxyAgnObj()
dbStar = StarObj()
catBulge = BulgeDiskCatalog(dbBulge, obs_metadata=obs)
catDisk = BulgeDiskCatalog(dbDisk, obs_metadata=obs)
catAgn = AgnCatalog(dbAgn, obs_metadata=obs)
catStar = StarCatalog(dbStar, obs_metadata=obs)
catBulge.write_catalog(controlFileName, write_header=False, chunk_size=10000)
catDisk.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
catAgn.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
catStar.write_catalog(controlFileName, write_mode='a', write_header=False, chunk_size=10000)
totalCat = CompoundInstanceCatalog([BulgeDiskCatalog, BulgeDiskCatalog, StarCatalog, AgnCatalog],
[GalaxyBulgeObj, GalaxyDiskObj, StarObj, GalaxyAgnObj],
obs_metadata=obs,
compoundDBclass=GalaxyTileCompoundObj)
totalCat.write_catalog(testFileName, write_header=False, chunk_size=10000)
with open(controlFileName, 'r') as controlFile:
control = controlFile.readlines()
with open(testFileName, 'r') as testFile:
test = testFile.readlines()
for line in control:
self.assertIn(line, test)
for line in test:
self.assertIn(line, control)
def __init__(self, opsimdb, descqa_catalog, dither=True,
min_mag=10, minsource=100, proper_motion=False,
imsim_catalog=False):
"""
Parameters
----------
obsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
imsim_catalog: bool [False]
Flag to write an imsim-style object catalog.
"""
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.imsim_catalog = imsim_catalog
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
self.star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433, driver='mssql+pymssql')
self.instcats = get_instance_catalogs(imsim_catalog)
print line
import numpy
from lsst.sims.catalogs.measures.instance import InstanceCatalog
class simpleStarCatalog(InstanceCatalog):
column_outputs = ['raJ2000', 'decJ2000', 'sedFilename']
transformations = {'raJ2000': numpy.degrees, 'decJ2000': numpy.degrees}
from lsst.sims.catalogs.generation.db import ObservationMetaData
myObsMetadata = ObservationMetaData(unrefractedRA=45.0,
unrefractedDec=-10.0,
boundType='circle',
boundLength=0.02)
from lsst.sims.catUtils.baseCatalogModels import StarObj
starTableConnection = StarObj()
myCatalog = simpleStarCatalog(starTableConnection, obs_metadata=myObsMetadata)
myCatalog.write_catalog('test_catalog.txt')
readCatalog = open('test_catalog.txt', 'r').readlines()
printCatalogToScreen('test_catalog.txt')
def __init__(self,
opsimdb,
descqa_catalog,
dither=True,
min_mag=10,
minsource=100,
proper_motion=False,
imsim_catalog=False,
protoDC2_ra=0,
protoDC2_dec=0,
agn_db_name=None,
sprinkler=False):
"""
Parameters
----------
obsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
imsim_catalog: bool [False]
Flag to write an imsim-style object catalog.
protoDC2_ra: float [0]
Desired RA (J2000 degrees) of protoDC2 center.
protoDC2_dec: float [0]
Desired Dec (J2000 degrees) of protoDC2 center.
agn_db_name: str [None]
Filename of the agn parameter sqlite db file.
sprinkler: bool [False]
Flag to enable the Sprinkler.
"""
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
# load the data for the parametrized light
# curve stellar variability model into a
# global cache
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves()
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.imsim_catalog = imsim_catalog
self.protoDC2_ra = protoDC2_ra
self.protoDC2_dec = protoDC2_dec
self.phot_params = PhotometricParameters(nexp=1, exptime=30)
self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
self.star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
if agn_db_name is None:
raise IOError("Need to specify an Proto DC2 AGN database.")
else:
if os.path.exists(agn_db_name):
self.agn_db_name = agn_db_name
else:
raise IOError("Path to Proto DC2 AGN database does not exist.")
self.sprinkler = sprinkler
self.instcats = get_instance_catalogs(imsim_catalog)
# find the RA, Dec of the north Galactic pole
ra, dec = equatorialFromGalactic(0.0, 90.0)
# define a field of view that is a circle of radius 1 degree
# around that point
obs = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
boundType='circle',
boundLength=1.0)
# connect to our database of simulated stars
# (if you want to do this from off campus, talk to me;
# there are more authentication steps you need to go through)
star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
# define the columns you want to get from the database
# (note: the query automatically adds the 'simobjid'
# column to the front; that is just an integer
# uniquely identifying each star)
column_names = ['ra', 'decl', 'sedFilename', 'rmag']
results = star_db.query_columns(colnames=column_names, obs_metadata=obs)
# results is now an iterator over chunks of stars
# to get numpy arrays of teff, metallicity, logg, and rmag
# use
teff = []
metallicity = []
def test_make_instcat_header(self):
star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
cat = make_instcat_header(star_db, self.obs_md, self.outfile)
import sqlite3
import numpy as np
from lsst.sims.utils import ObservationMetaData
from lsst.sims.utils import htmModule as htm
from lsst.sims.catUtils.baseCatalogModels import StarObj
in_db = StarObj(database='LSSTCATSIM', host='fatboy.phys.washington.edu',
port=1433, driver='mssql+pymssql')
# full DC2 field of view
dc2_obs = ObservationMetaData(pointingRA=55.064, pointingDec=-29.783,
boundType='circle', boundLength=23.0)
# file to write
out_db_name = 'dc2_stellar_db.db'
with sqlite3.connect(out_db_name) as out_conn:
out_c = out_conn.cursor()
creation_query = '''CREATE TABLE stars (
simobjid int, htmid_6 int, ra real, decl real,
gal_l real, gal_b real, magNorm real,
mura real, mudecl real, parallax real,
ebv real, radialVelocity real, varParamStr text,
sedFilename text,
umag real, gmag real, rmag real, imag real,
zmag real, ymag real)'''
out_c.execute(creation_query)
out_conn.commit()
colnames = ['simobjid', 'ra', 'decl', 'gal_l', 'gal_b',
raise RuntimeError('%s does not exist' % opsimdb)
obs_generator = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogSersic2D
from lsst.sims.catUtils.exampleCatalogDefinitions import PhoSimCatalogZPoint
from lsst.sims.catUtils.exampleCatalogDefinitions import DefaultPhoSimHeaderMap
from lsst.sims.catUtils.baseCatalogModels import StarObj
from lsst.sims.catUtils.baseCatalogModels import GalaxyBulgeObj, GalaxyDiskObj
from lsst.sims.catUtils.baseCatalogModels import GalaxyAgnObj
from lsst.sims.utils import _getRotSkyPos
import copy
star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
bulge_db = GalaxyBulgeObj(connection=star_db.connection)
disk_db = GalaxyDiskObj(connection=star_db.connection)
agn_db = GalaxyAgnObj(connection=star_db.connection)
if not os.path.exists(out_dir):
os.mkdir(out_dir)
phosim_header_map = copy.deepcopy(DefaultPhoSimHeaderMap)
phosim_header_map['nsnap'] = 1
phosim_header_map['vistime'] = 30.0
phosim_header_map['camconfig'] = 1
for obshistid in obshistid_list:
def write_stars_to_truth(output=None,
n_procs=10, n_side=2048, clobber=False):
"""
Write static star truth to the truth catalog
Parameters
----------
output is the path to the output database
n_procs is the number of Multiprocessing processes to use when
calculating magnitudes
n_side is the nside parameter for calculating healpix locations
clobber is a boolean. If True, delete any already existing databases
with the same file name as output (default=False)
Returns
-------
None
Just writes to the database
"""
if output is None:
raise RuntimeError("Must specify output database")
if os.path.isfile(output):
if clobber:
os.unlink(output)
# the survey area
obs = ObservationMetaData(pointingRA=55.064,
pointingDec=-29.783,
mjd=59580.0,
boundType='circle',
boundLength=4.0)
db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
chunk_size = 10000
p_list = []
mgr = mp.Manager()
mag_dict = mgr.dict()
position_dict = {}
t_start = time.time()
row_ct = 0
iteration = 0
with sqlite3.connect(output) as out_conn:
out_cursor = out_conn.cursor()
data_iter = db.query_columns(colnames=['simobjid', 'sedFilename',
'magNorm', 'ra', 'decl'],
obs_metadata=obs,
chunk_size = 10000)
for star_chunk in data_iter:
proc = mp.Process(target=calculate_mags,
args=(star_chunk['sedFilename'],
star_chunk['magNorm'],
mag_dict))
proc.start()
p_list.append(proc)
# find healpix positions of the stars
hp_arr = hp.ang2pix(n_side,
star_chunk['ra'],
star_chunk['decl'],
lonlat=True,
nest=True)
position_dict[proc.pid] = {}
position_dict[proc.pid]['healpix'] = hp_arr
position_dict[proc.pid]['id'] = star_chunk['simobjid']
position_dict[proc.pid]['ra'] = star_chunk['ra']
position_dict[proc.pid]['dec'] = star_chunk['decl']
if len(p_list) >= n_procs:
for p in p_list:
p.join()
row_ct += write_results(out_conn, out_cursor,
mag_dict, position_dict)
p_list = []
position_dict = {}
mag_dict = mgr.dict()
iteration += 1
duration = (time.time()-t_start)/3600.0
predicted = 1.0e7*duration/row_ct
print('output %d in %.2e hrs; 10 million in %.2e' %
(row_ct, duration, predicted))
if len(p_list) > 0:
for p in p_list:
p.join()
write_results(out_conn, out_cursor,
mag_dict, position_dict)
