def readOpsim(self,
opsimfile,
constraint=None,
dbcols=None,
expMJDCol='expMJD'):
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
if dbcols is None:
dbcols = []
# Be sure the minimum columns that we need are in place.
# reqcols = ['expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
# 'visitExpTime', 'finSeeing', 'fiveSigmaDepth', 'solarElong']
reqcols = [
expMJDCol, 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
'visitExpTime', 'FWHMeff', 'FWHMgeom', 'fiveSigmaDepth',
'solarElong'
]
for col in reqcols:
if col not in dbcols:
dbcols.append(col)
simdata = opsdb.fetchMetricData(dbcols, sqlconstraint=constraint)
print("Queried data from opsim %s, fetched %d visits." %
(opsimfile, len(simdata[expMJDCol])),
file=self.logfile)
return simdata
def runMoObs(orbitfile,
outfileName,
opsimfile,
dbcols=None,
tstep=2. / 24.,
sqlconstraint='',
rFov=np.radians(1.75),
useCamera=True):
from lsst.sims.maf.db import OpsimDatabase
# Read orbits.
moogen = MoObs()
moogen.readOrbits(orbitfile)
print "Read orbit information from %s" % (orbitfile)
# Check rfov/camera choices.
if useCamera:
print "Using camera footprint"
else:
print "Not using camera footprint; using circular fov with %f degrees radius" % (
np.degrees(rFov))
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
if dbcols is None:
dbcols = []
# Be sure the columns that we need are in place.
reqcols = [
'expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
'visitExpTime', 'finSeeing', 'fiveSigmaDepth', 'solarElong'
]
for col in reqcols:
if col not in dbcols:
dbcols.append(col)
simdata = opsdb.fetchMetricData(dbcols, sqlconstraint=sqlconstraint)
print "Queried data from opsim %s, fetched %d visits." % (
opsimfile, len(simdata['expMJD']))
moogen.setTimesRange(timeStep=tstep,
timeStart=simdata['expMJD'].min(),
timeEnd=simdata['expMJD'].max())
print "Will generate ephemerides on grid of %f day timesteps, then extrapolate to opsim times." % (
tstep)
moogen.setupOorb()
for i, sso in moogen.orbits.iterrows():
ephs = moogen.generateEphs(sso)
interpfuncs = moogen.interpolateEphs(ephs)
idxObs = moogen.ssoInFov(interpfuncs,
simdata,
rFov=rFov,
useCamera=useCamera)
moogen.writeObs(sso['objId'],
interpfuncs,
simdata,
idxObs,
sedname=sso['sed_filename'],
outfileName=outfileName)
print "Wrote output observations to file %s" % (outfileName)
def runMoObs(orbitfile, outfileName, opsimfile,
dbcols=None, tstep=2./24., nyears=None,
rFov=np.radians(1.75), useCamera=True):
from lsst.sims.maf.db import OpsimDatabase
# Read orbits.
moogen = MoObs()
moogen.readOrbits(orbitfile)
print "Read orbit information from %s" %(orbitfile)
# Check rfov/camera choices.
if useCamera:
print "Using camera footprint"
else:
print "Not using camera footprint; using circular fov with %f degrees radius" %(np.degrees(rFov))
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
if dbcols is None:
dbcols = ['expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
'finSeeing', 'fiveSigmaDepth', 'visitExpTime', 'solarElong']
else:
reqcols = ['expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
'visitExpTime', 'finSeeing', 'fiveSigmaDepth']
for col in reqcols:
if col not in dbcols:
dbcols.append(col)
if nyears is not None:
ndays = nyears * 365
sqlconstraint = 'night<%d' %(ndays)
else:
ndays = 3650.0
sqlconstraint = ''
simdata = opsdb.fetchMetricData(dbcols, sqlconstraint=sqlconstraint)
print "Queried data from opsim %s, fetched %f years worth of visits." %(opsimfile, ndays/365.)
moogen.setTimes(timestep=tstep, ndays=ndays, timestart=simdata['expMJD'].min())
print "Will generate ephemerides on grid of %f day timesteps, then extrapolate to opsim times." %(tstep)
moogen.setupOorb()
for i, sso in moogen.orbits.iterrows():
ephs = moogen.generateEphs(sso)
interpfuncs = moogen.interpolateEphs(ephs)
idxObs = moogen.ssoInFov(interpfuncs, simdata, rFov=rFov, useCamera=useCamera)
moogen.writeObs(sso['objId'], interpfuncs, simdata, idxObs, outfileName=outfileName)
print "Wrote output observations to file %s" %(outfileName)
def readOpsim(opsimfile, constraint=None, dbcols=None):
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
if dbcols is None:
dbcols = []
colmap = getColMap(opsdb)
reqcols = [
colmap['mjd'], colmap['ra'], colmap['dec'], colmap['filter'],
colmap['exptime'], colmap['seeingEff'], colmap['seeingGeom'],
colmap['fiveSigmaDepth']
]
degreesIn = colmap['raDecDeg']
for col in reqcols:
if col not in dbcols:
dbcols.append(col)
simdata = opsdb.fetchMetricData(dbcols, sqlconstraint=constraint)
opsdb.close()
print("Queried data from opsim %s, fetched %d visits." %
(opsimfile, len(simdata)))
simdata = fixObsData(simdata, degreesIn=degreesIn)
return simdata, colmap
def queryFields(database):
# Read the fields from the field table in the database.
opsdb = OpsimDatabase(database)
fields = opsdb.fetchFieldsFromFieldTable()
# Add galactic and equatorial values to these field locations.
gall, galb = galacticFromEquatorial(fields['fieldRA'], fields['fieldDec'])
ecll = np.zeros(len(gall), float)
eclb = np.zeros(len(galb), float)
for i, (ra, dec) in enumerate(zip(fields['fieldRA'], fields['fieldDec'])):
coord = ephem.Equatorial(ra, dec, epoch=2000)
ecl = ephem.Ecliptic(coord)
ecll[i] = ecl.lon
eclb[i] = ecl.lat
newfields = np.empty((len(gall)), dtype=[('ra', float), ('dec', float),
('galL', float), ('galB', float),
('eclL', float), ('eclB', float)])
newfields['ra'] = fields['fieldRA']
newfields['dec'] = fields['fieldDec']
newfields['galL'] = gall
newfields['galB'] = galb
newfields['eclL'] = ecll
newfields['eclB'] = eclb
return newfields
def queryFields(database):
# Read the fields from the field table in the database.
opsdb = OpsimDatabase(database)
fields = opsdb.fetchFieldsFromFieldTable()
# Add galactic and equatorial values to these field locations.
gall, galb = galacticFromEquatorial(fields['fieldRA'], fields['fieldDec'])
ecll = np.zeros(len(gall), float)
eclb = np.zeros(len(galb), float)
for i, (ra, dec) in enumerate(zip(fields['fieldRA'], fields['fieldDec'])):
coord = ephem.Equatorial(ra, dec, epoch=2000)
ecl = ephem.Ecliptic(coord)
ecll[i] = ecl.lon
eclb[i] = ecl.lat
newfields = np.empty((len(gall)),
dtype=[('ra', float), ('dec', float), ('galL', float),
('galB', float), ('eclL', float),
('eclB', float)])
newfields['ra'] = fields['fieldRA']
newfields['dec'] = fields['fieldDec']
newfields['galL'] = gall
newfields['galB'] = galb
newfields['eclL'] = ecll
newfields['eclB'] = eclb
return newfields
# Define metrics, stackers and slicer so that we can see what columns we need from database.
metricList, plotDictList = setupMetrics(opsimName,
metadata,
0,
0,
plotlabel='',
cumulative=args.cumulative)
# Define any non-default stackers to be used.
stackerList = setupStackers(args)
# Define the slicer to be used at each step of the movie slicer.
subslicer = setupHealpixSlicer(args)
if not args.skipComp:
# Connect to database.
opsDb = OpsimDatabase(os.path.join(args.dbDir, args.opsimDb))
# Get data from database.
dbcols = [
'observationStartMJD', 'fiveSigmaDepth', args.raCol, args.decCol
]
simdata = utils.getSimData(opsDb,
sqlconstraint,
dbcols,
stackers=stackerList)
# Generate the bins for the movie slicer.
start_date = simdata['observationStartMJD'].min()
end_date = simdata['observationStartMJD'].max()
bins = np.arange(start_date, end_date + args.movieStepsize / 2.0,
args.movieStepsize, float)
# Run the movie slicer (and at each step, healpix slicer and calculate metrics).
gm = runSlices(opsimName, metadata, simdata, bins, args)
# Calculate m5 without airmass
m5_x1 = (
Cm[visitFilter]
+ dCm
+ 0.50 * (filtsky - 21.0)
+ 2.5 * np.log10(0.7 / FWHMeff)
+ 1.25 * np.log10(expTime / 30.0)
)
return m5, m5_x1
if __name__ == "__main__":
# Replace this with the filename for your sqlite database.
dbFile = "/Users/lynnej/opsim/db/ewok_1004_sqlite.db"
opsdb = OpsimDatabase(dbFile)
hardware, system, darksky = setup_photUtils()
cols = ["filter", "filtSkyBrightness", "FWHMeff", "visitExpTime", "airmass", "fiveSigmaDepth"]
# Loop through each bandpass and examine all m5 values at low airmass. (just using low airmass because
# the atmosphere we use in the system bandpass is for X=1.0, so high airmass would be unfair comparison).
for f in ("u", "g", "r", "i", "z", "y"):
sqlconstraint = 'airmass<1.05 and filter="%s"' % (f)
data = opsdb.fetchMetricData(cols, sqlconstraint)
m5_phot = calc_m5_photUtils(
hardware,
system,
darksky,
data["filter"],
data["filtSkyBrightness"],
def variedParameters(self, paramNameLike=None, dbDir=None):
"""
Query the opsim configuration table for a set of user defined
configuration parameters for a set of runs.
Parameters
----------
paramNameLike : list, opt
A list of of opsim configuration parameters to pull out of the
configuration table.
Results
-------
pandas DataFrame
A pandas dataframe containing a column for each of the configuration
parameters given in paramName like. The resulting dataframe is
indexed the name of the opsim runs.
runName parameter1 parameter2
<run_123> <parameterValues1> <parameterValues1>
Notes
-----
This method will use the sqlite 'like' function to query the
configuration table. Below is an example of how the items in
paramNameLike need to be formatted:
["%WideFastDeep%hour_angle_bonus%", "%WideFastDeep%airmass_bonus%"].
"""
if paramNameLike is None:
paramNameLike = ["%WideFastDeep%airmass_bonus%",
"%WideFastDeep%hour_angle_bonus%"]
sqlconstraints = []
parameterNames = []
for p in paramNameLike:
name = p.rstrip('%').lstrip('%').replace('%', ' ')
parameterNames.append(name)
sql = 'paramName like "%s"' % (p)
sqlconstraints.append(sql)
# Connect to original databases and grab configuration parameters.
opsdb = {}
for r in self.runlist:
# Check if file exists XXXX
if dbDir is None:
opsdb[r] = OpsimDatabase(os.path.join(r, 'data', r + '.db'))
else:
opsdb[r] = OpsimDatabase(os.path.join(dbDir, r + '.db'))
# try also sqlite.db
parameterValues = {}
for i, r in enumerate(self.runlist):
parameterValues[r] = {}
for pName, sql in zip(parameterNames, sqlconstraints):
val = opsdb[r].query_columns('Config', colnames=['paramValue'],
sqlconstraint=sql)
if len(val) > 1.0:
warnings.warn(sql + ' returned more than one value.' +
' Add additional information such as the proposal name' +
'\n' + 'Example: ' + '%WideFastDeep%hour_angle_bonus%')
parameterValues[r][pName] = -666
else:
parameterValues[r][pName] = val['paramValue'][0]
if self.verbose:
print('Queried Config Parameters with: ' + sql +
'\n' + 'found value: ' + str(parameterValues[r][pName]))
tempDFList = []
for r in self.runlist:
tempDFList.append(pd.DataFrame(parameterValues[r], index=[r]))
# Concatenate dataframes for each run.
if self.parameters is None:
self.parameters = pd.concat(tempDFList)
else:
self.parameters = self.parameters.join(tempDFList)
vel = vel / 24.0 # "/s
# See https://listserv.lsstcorp.org/mailman/private/lsst-solarsystem/2006-December/24.html
# should grab simObs['expTime'] ..
t = 30.0 # seconds
teff = t/(1+1.6*vel*t/simdata[idxs]['finSeeing'])
dmagTrailing = 1.25*np.log10(t/teff)
# Convert to recarray.
dmags = np.rec.fromarrays([dmagColor, dmagTrailing], names=['dmagColor', 'dmagTrailing'])
# Join arrays.
ssoObs = merge_arrays([ephs, simdata[idxs], dmags], flatten=True, asrecarray=True)
return ssoObs
###
dbAddress = 'sqlite:///enigma_1189_sqlite.db'
ops = OpsimDatabase(dbAddress)
dbcols = ['expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter', 'finSeeing', 'fiveSigmaDepth']
simdata = ops.fetchMetricData(dbcols, sqlconstraint='')
orbits = pandas.read_table('pha20141031.des', sep=' ')
orbits = orbits.to_records()
rephs = pandas.read_table('phas_obs_all.txt', sep=' ')
rephs = rephs.to_records()
ssoObs = joinObs(rephs, simdata)
# write out joint data.
outfile = open('phas_allobs_all.txt', 'w')
outnames = ['!!ObjID', 'time', 'ra', 'dec', 'dradt', 'ddecdt', 'dist', 'magV', 'phaseangle', 'solarelon',
'expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter', 'finSeeing', 'fiveSigmaDepth',
raise Exception(
'OpsimDB should be just the filename of the sqlite file (not %s). Use --dbDir.'
% (args.opsimDb))
# Define metrics.
metricList = setupMetrics(args, 0, 0)
# Define any non-default stackers to be used.
stackerList = setupStackers(args)
# Define the slicer to be used at each step of the movie slicer.
subslicer = setupHealpixSlicer(args)
if not args.skipComp:
# Get db connection info, and connect to database.
opsimName = args.opsimDb.replace('_sqlite.db', '')
dbAddress = 'sqlite:///' + os.path.join(args.dbDir, args.opsimDb)
opsDb = OpsimDatabase(dbAddress)
sqlconstraint = args.sqlConstraint
metadata = sqlconstraint.replace('=', '').replace(
'filter', '').replace("'", '').replace('"', '').replace('/', '.')
# Find the columns we need to query from the database.
sm = sliceMetrics.RunSliceMetric(useResultsDb=False,
outDir=args.outDir)
sm.setMetricsSlicerStackers(metricList, subslicer, stackerList)
dbcols = sm.findDataCols()
# Get data from database.
simdata = utils.getSimData(opsDb,
sqlconstraint,
dbcols,
stackers=stackerList)
# Generate the bins for the movie slicer.
start_date = simdata['expMJD'].min()
1 + (10**(0.8 * dCm_infinity[visitFilter]) - 1) / Tscale)
# Calculate fiducial m5
m5 = (Cm[visitFilter] + dCm + 0.50 * (filtsky - 21.0) +
2.5 * np.log10(0.7 / FWHMeff) + 1.25 * np.log10(expTime / 30.0) -
kAtm[visitFilter] * (airmass - 1.0) + 1.1 * tauCloud)
# Calculate m5 without airmass
m5_x1 = (Cm[visitFilter] + dCm + 0.50 * (filtsky - 21.0) +
2.5 * np.log10(0.7 / FWHMeff) + 1.25 * np.log10(expTime / 30.0))
return m5, m5_x1
if __name__ == '__main__':
# Replace this with the filename for your sqlite database.
dbFile = '/Users/lynnej/opsim/db/ewok_1004_sqlite.db'
opsdb = OpsimDatabase(dbFile)
hardware, system, darksky = setup_photUtils()
cols = [
'filter', 'filtSkyBrightness', 'FWHMeff', 'visitExpTime', 'airmass',
'fiveSigmaDepth'
]
# Loop through each bandpass and examine all m5 values at low airmass. (just using low airmass because
# the atmosphere we use in the system bandpass is for X=1.0, so high airmass would be unfair comparison).
for f in ('u', 'g', 'r', 'i', 'z', 'y'):
sqlconstraint = 'airmass<1.05 and filter="%s"' % (f)
data = opsdb.fetchMetricData(cols, sqlconstraint)
m5_phot = calc_m5_photUtils(hardware, system, darksky, data['filter'],
data['filtSkyBrightness'], data['FWHMeff'],
data['visitExpTime'], data['airmass'])
def queryFields(database):
# Read the fields from the field table in the database.
opsdb = OpsimDatabase('enigma_1189_sqlite.db')
fields = opsdb.fetchFieldsFromFieldTable()
return fields
def runLinearObs(orbitfile,
outfileName,
opsimfile,
dbcols=None,
tstep=2. / 24.,
sqlconstraint='',
rFov=1.75,
useCamera=True,
obscode=807,
expMJDCol='expMJD'):
from lsst.sims.maf.db import OpsimDatabase
# Read orbits.
lObs = LinearObs(orbitfile, obscode=obscode, timescale='TAI')
print("Read orbit information from %s" % (orbitfile))
# Check rfov/camera choices.
if useCamera:
print("Using camera footprint")
else:
print(
"Not using camera footprint; using circular fov with %f degrees radius"
% (rFov))
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
if dbcols is None:
dbcols = []
# Be sure the columns that we need are in place.
#reqcols = ['expMJD', 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
# 'visitExpTime', 'finSeeing', 'fiveSigmaDepth', 'solarElong']
reqcols = [
expMJDCol, 'night', 'fieldRA', 'fieldDec', 'rotSkyPos', 'filter',
'visitExpTime', 'FWHMeff', 'FWHMgeom', 'fiveSigmaDepth', 'solarElong'
]
for col in reqcols:
if col not in dbcols:
dbcols.append(col)
simdata = opsdb.fetchMetricData(dbcols, sqlconstraint=sqlconstraint)
print("Queried data from opsim %s, fetched %d visits." %
(opsimfile, len(simdata[expMJDCol])))
lObs.setTimesRange(timeStep=tstep,
timeStart=simdata[expMJDCol].min(),
timeEnd=simdata[expMJDCol].max())
print(
"Will generate ephemerides on grid of %f day timesteps, then extrapolate to opsim times."
% (tstep))
for sso in lObs:
objid = sso.orbits['objId'].iloc[0]
sedname = sso.orbits['sed_filename'].iloc[0]
ephs = lObs.generateEphs(sso)
interpfuncs = lObs.interpolateEphs(ephs)
idxObs = lObs.ssoInFov(interpfuncs,
simdata,
rFov=rFov,
useCamera=useCamera)
lObs.writeObs(objid,
interpfuncs,
simdata,
idxObs,
sedname=sedname,
outfileName=outfileName)
print("Wrote output observations to file %s" % (outfileName))
def readOpsim(opsimfile, constraint=None, footprint='camera', dbcols=None):
"""Read the opsim database.
Parameters
----------
opsimfile: str
Name (& path) of the opsim database file.
constraint: str, opt
Optional SQL constraint (minus 'where') on the opsim data to read from db.
Default is None.
footprint: str, opt
Footprint option for the final matching of object against OpSim FOV.
Default 'camera' means that 'rotSkyPos' must be fetched from the db.
Any other value will not require rotSkyPos.
dbcols: list of str, opt
List of additional columns to query from the db and add to the output observations.
Default None.
Returns
-------
np.ndarray, dictionary
The OpSim data read from the database, and the dictionary mapping the column names to the data.
"""
# Read opsim database.
opsdb = OpsimDatabase(opsimfile)
colmap = getColMap(opsdb)
if 'rotSkyPos' not in colmap:
colmap['rotSkyPos'] = 'rotSkyPos'
# Set the minimum required columns.
min_cols = [
colmap['mjd'], colmap['night'], colmap['ra'], colmap['dec'],
colmap['filter'], colmap['exptime'], colmap['seeingGeom'],
colmap['fiveSigmaDepth']
]
if footprint == 'camera':
min_cols.append(colmap['rotSkyPos'])
if dbcols is not None:
min_cols += dbcols
min_cols = list(set(min_cols))
# Check if these minimum required columns are in the database.
simdata = opsdb.query_columns(tablename=opsdb.defaultTable,
colnames=min_cols,
sqlconstraint=constraint,
numLimit=1)
# If that was successful, there are some additional columns that can be useful:
more_cols = [colmap['rotSkyPos'], colmap['seeingEff'], 'solarElong']
failed_cols = []
for col in more_cols:
try:
simdata = opsdb.query_columns(tablename=opsdb.defaultTable,
colnames=[col],
sqlconstraint=constraint,
numLimit=1)
except ValueError:
failed_cols.append(col)
for col in failed_cols:
more_cols.remove(col)
cols = min_cols + more_cols
cols = list(set(cols))
logging.info('Querying for columns:\n %s' % (cols))
# Go ahead and query for all of the observations.
simdata = opsdb.fetchMetricData(cols, sqlconstraint=constraint)
opsdb.close()
logging.info("Queried data from opsim %s, fetched %d visits." %
(opsimfile, len(simdata)))
return simdata, colmap
