def test_OpSimOutput_fromOmSimDB(fname, opsimversion, tableName, expected):
fname = os.path.join(oss.example_data, fname)
opsout = OpSimOutput.fromOpSimDB(fname, opsimversion=opsimversion,
tableNames=(tableName, 'Proposal'))
# Check the length of the summary table
assert len(opsout.summary) == expected[0]
# Check the length of the summary table
assert len(opsout.proposalTable) == expected[1]
# Check that 'wfd', and 'ddf' are in the proposal table
propNameCol = opsout.opsimVars['propName']
proposalNames = opsout.proposalTable[propNameCol].values
assert 'wfd' in proposalNames
assert 'ddf' in proposalNames
pointingRA = opsout.opsimVars['pointingRA']
pointingDec = opsout.opsimVars['pointingDec']
# Check that ra are in the right units
assert opsout.opsimVars['angleUnit'] == expected[2]
if opsout.opsimVars['angleUnit'] == 'degrees':
assert opsout.summary[pointingRA].max() > 10.
elif opsout.opsimVars['angleUnit'] == 'radians':
assert opsout.summary[pointingDec].max() < 2.* np.pi + 1.0e-5
def test_get_propIDDict(fname, opsimversion, expected):
fname = os.path.join(oss.example_data, fname)
engine = create_engine('sqlite:///' + fname)
df = pd.read_sql_table(con=engine, table_name='Proposal')
mydict = OpSimOutput.get_propIDDict(df, opsimversion=opsimversion)
assert mydict['ddf'] == expected[1]
assert mydict['wfd'] == expected[0]
def test_pointingTree(fname, opsimversion, tableNames, angleUnit):
"""test that `PointingTree` is set up correctly to handle
pointings provided by `OpSimOutput.summary` by checking that
distance calculations for the 10 nearest ones match
"""
fname = os.path.join(oss.example_data, fname)
ops_out = OpSimOutput.fromOpSimDB(fname,
opsimversion=opsimversion,
tableNames=tableNames)
pointings = ops_out.summary.iloc[:100]
# Find distances to a specific location ra=54.0 deg, dec=-27.5 deg
radeg = 54.0
decdeg = -27.5
ra = np.radians(radeg)
dec = np.radians(decdeg)
ptree = PointingTree(pointings, raCol='_ra', decCol='_dec')
# Find the 10 nearest pointings
dists, idxs = ptree.tree.query([(dec, ra)], k=10)
# Now calculate the distances more directly
# order the 10 pointings as above
obs = pointings.iloc[idxs[0]]
vec = hp.ang2vec(radeg, decdeg, lonlat=True)
vecs = hp.ang2vec(np.degrees(obs._ra), np.degrees(obs._dec), lonlat=True)
assert_allclose(np.dot(vecs, vec), np.cos(dists[0]))
def test_SynOpSim_init(fname, opsimversion, tableNames, angleUnit):
fname = os.path.join(oss.example_data, fname)
ops_out = OpSimOutput.fromOpSimDB(fname,
opsimversion=opsimversion,
tableNames=tableNames)
num_visits = len(ops_out.summary)
synopsim = SynOpSim(ops_out.summary)
assert synopsim.pointings.night.size == num_visits
def test_fromOpSimDB(fname, opsimversion, tableNames, angleUnit):
fname = os.path.join(oss.example_data, fname)
print(fname, opsimversion)
ops_out = OpSimOutput.fromOpSimDB(fname,
opsimversion=opsimversion,
ditherColumns=None,
add_dithers=False)
print('finished ops_out', len(ops_out.summary))
synopsim = SynOpSim(ops_out.summary)
synopsimd = SynOpSim.fromOpSimDB(fname,
opsimversion=opsimversion,
ditherColumns=None,
add_dithers=False)
assert synopsim.pointings.equals(synopsimd.pointings)
import sys
sys.stdout.flush()
# Read the opsim data base and start logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
print('Healpy version: ', hp.__version__)
print('numpy version: ', np.__version__)
print('healpy dir', getPackageDir('healpy'))
print('sims_skybrightness_dir', getPackageDir('sims_skybrightness'))
print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data'))
logger.info('Start reading opsim database')
minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
opsout = OpSimOutput.fromOpSimDB(minion_out,
zeroDDFDithers=True,
subset="unique_all")
print('reading done')
df = opsout.summary.copy()
logger.info('Finished reading database at {}'.format(time.time()))
# Bandpasses and LSST related stuff
def atmTransName(airmass):
"""
obtain the filename with the transmissions for files for an airmass value closest to the requested value
"""
l = np.arange(1.0, 2.51, 0.1)
idx = np.abs(l - airmass).argmin()
a = np.int(10 * l[idx])
baseline = getPackageDir('THROUGHPUTS')
# Read the opsim data base and start logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
tstart = time.time()
logger.info('Start Program at time {}'.format(tstart))
print('Healpy version: ', hp.__version__)
print('numpy version: ', np.__version__)
print('healpy dir', getPackageDir('healpy'))
print('sims_skybrightness_dir', getPackageDir('sims_skybrightness'))
print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data'))
print('obscond version', obscond.__version__)
logger.info('Start reading opsim database')
minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
#minion_out = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="unique_all")
# opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="ddf")
print('reading done\n')
df = opsout.summary.copy()
logger.info('Finished reading database at {}'.format(time.time()))
totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles()
photparams = PhotometricParameters()
# Split the entries in the opsim database for parallelization
splits = 1000
dfs = np.array_split(df, splits)
print('splitting dataframe of size {0} into {1} splits each of size {2}\n'.format(len(df), splits, len(dfs[0])))
calcdfs = []
availddfFileName = basename + "_DDF_avail.csv"
selectedddfFileName = basename + "_DDF_sel.csv"
print('output file names for DDF are {0}, {1}, {2}'.format(
ddf_simlibfilename, availddfFileName, selectedddfFileName))
numFields_DDF = args.numFields_DDF
numFields_WFD = args.numFields_WFD
print(args)
sys.stdout.flush()
# find ddf healpixels
if get_ddf_pixels:
print('Finding the DDF healpixels \n')
opsout_ddf = OpSimOutput.fromOpSimDB(dbname,
opsimversion=opsimversion,
subset='ddf',
dithercolumns=dithercolumns,
filterNull=filternulls)
if len(opsout_ddf.summary) > 0:
print("writing out ddf pixels\n")
simlib_ddf = Simlibs(opsout_ddf.summary,
opsimversion=opsimversion,
usePointingTree=True)
ddf_hid = set(simlib_ddf.observedVisitsinRegion().index.values)
else:
print("writing out null set of ddf pixels\n")
ddf_hid = set([])
print("written out null set of ddf pixels\n")
print('There are {} pixels in the ddf fields'.format(len(ddf_hid)))
# read the database into a `pd.DataFrame`
opsim_fname = 'feature_rolling_half_mask_10yrs.db'
simlib_fname = 'feature_rolling_half_mask_10yrs.simlib'
script_start = time.time()
log_str = 'Running script with opsimsummary version {}\n'.format(
oss.__version__)
log_val = 'Starting Calculation at {}\n'.format(script_start)
log_str += log_val
pkgDir = os.path.split(oss.__file__)[0]
dbname = os.path.join('/Users/rbiswas/data/', 'LSST/OpSimData', opsim_fname)
log_val = 'The OpSim DataBase used is {}\n'.format(dbname)
log_str += log_val
# read the database into a `pd.DataFrame`
opsout = OpSimOutput.fromOpSimDB(dbname,
opsimversion='lsstv4',
tableNames=('SummaryAllProps', 'Proposal'))
summary = opsout.summary
log_val = 'dataframe read in from database {}\n'.format(time.time())
log_str += log_val
simlibs = Simlibs(summary, opsimversion='lsstv4', usePointingTree=True)
rng = np.random.RandomState(1)
simlibs.randomSimlibs(numFields=50000, fname=simlib_fname) #, rng=rng)
log_val = 'Done'
log_str += log_val
log_val = 'Writing simlib for input to outfile {0} at time {1}\n'.format(
simlib_fname, time.time())
print(log_val)
log_str += log_val