def test_OpSimOutput_fromOmSimDB(fname, opsimversion, tableName, expected): fname = os.path.join(oss.example_data, fname) opsout = OpSimOutput.fromOpSimDB(fname, opsimversion=opsimversion, tableNames=(tableName, 'Proposal')) # Check the length of the summary table assert len(opsout.summary) == expected[0] # Check the length of the summary table assert len(opsout.proposalTable) == expected[1] # Check that 'wfd', and 'ddf' are in the proposal table propNameCol = opsout.opsimVars['propName'] proposalNames = opsout.proposalTable[propNameCol].values assert 'wfd' in proposalNames assert 'ddf' in proposalNames pointingRA = opsout.opsimVars['pointingRA'] pointingDec = opsout.opsimVars['pointingDec'] # Check that ra are in the right units assert opsout.opsimVars['angleUnit'] == expected[2] if opsout.opsimVars['angleUnit'] == 'degrees': assert opsout.summary[pointingRA].max() > 10. elif opsout.opsimVars['angleUnit'] == 'radians': assert opsout.summary[pointingDec].max() < 2.* np.pi + 1.0e-5
def test_get_propIDDict(fname, opsimversion, expected): fname = os.path.join(oss.example_data, fname) engine = create_engine('sqlite:///' + fname) df = pd.read_sql_table(con=engine, table_name='Proposal') mydict = OpSimOutput.get_propIDDict(df, opsimversion=opsimversion) assert mydict['ddf'] == expected[1] assert mydict['wfd'] == expected[0]
def test_pointingTree(fname, opsimversion, tableNames, angleUnit): """test that `PointingTree` is set up correctly to handle pointings provided by `OpSimOutput.summary` by checking that distance calculations for the 10 nearest ones match """ fname = os.path.join(oss.example_data, fname) ops_out = OpSimOutput.fromOpSimDB(fname, opsimversion=opsimversion, tableNames=tableNames) pointings = ops_out.summary.iloc[:100] # Find distances to a specific location ra=54.0 deg, dec=-27.5 deg radeg = 54.0 decdeg = -27.5 ra = np.radians(radeg) dec = np.radians(decdeg) ptree = PointingTree(pointings, raCol='_ra', decCol='_dec') # Find the 10 nearest pointings dists, idxs = ptree.tree.query([(dec, ra)], k=10) # Now calculate the distances more directly # order the 10 pointings as above obs = pointings.iloc[idxs[0]] vec = hp.ang2vec(radeg, decdeg, lonlat=True) vecs = hp.ang2vec(np.degrees(obs._ra), np.degrees(obs._dec), lonlat=True) assert_allclose(np.dot(vecs, vec), np.cos(dists[0]))
def test_SynOpSim_init(fname, opsimversion, tableNames, angleUnit): fname = os.path.join(oss.example_data, fname) ops_out = OpSimOutput.fromOpSimDB(fname, opsimversion=opsimversion, tableNames=tableNames) num_visits = len(ops_out.summary) synopsim = SynOpSim(ops_out.summary) assert synopsim.pointings.night.size == num_visits
def test_fromOpSimDB(fname, opsimversion, tableNames, angleUnit): fname = os.path.join(oss.example_data, fname) print(fname, opsimversion) ops_out = OpSimOutput.fromOpSimDB(fname, opsimversion=opsimversion, ditherColumns=None, add_dithers=False) print('finished ops_out', len(ops_out.summary)) synopsim = SynOpSim(ops_out.summary) synopsimd = SynOpSim.fromOpSimDB(fname, opsimversion=opsimversion, ditherColumns=None, add_dithers=False) assert synopsim.pointings.equals(synopsimd.pointings)
import sys sys.stdout.flush() # Read the opsim data base and start logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) print('Healpy version: ', hp.__version__) print('numpy version: ', np.__version__) print('healpy dir', getPackageDir('healpy')) print('sims_skybrightness_dir', getPackageDir('sims_skybrightness')) print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data')) logger.info('Start reading opsim database') minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db' opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="unique_all") print('reading done') df = opsout.summary.copy() logger.info('Finished reading database at {}'.format(time.time())) # Bandpasses and LSST related stuff def atmTransName(airmass): """ obtain the filename with the transmissions for files for an airmass value closest to the requested value """ l = np.arange(1.0, 2.51, 0.1) idx = np.abs(l - airmass).argmin() a = np.int(10 * l[idx]) baseline = getPackageDir('THROUGHPUTS')
# Read the opsim data base and start logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) tstart = time.time() logger.info('Start Program at time {}'.format(tstart)) print('Healpy version: ', hp.__version__) print('numpy version: ', np.__version__) print('healpy dir', getPackageDir('healpy')) print('sims_skybrightness_dir', getPackageDir('sims_skybrightness')) print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data')) print('obscond version', obscond.__version__) logger.info('Start reading opsim database') minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db' #minion_out = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db' opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="unique_all") # opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="ddf") print('reading done\n') df = opsout.summary.copy() logger.info('Finished reading database at {}'.format(time.time())) totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles() photparams = PhotometricParameters() # Split the entries in the opsim database for parallelization splits = 1000 dfs = np.array_split(df, splits) print('splitting dataframe of size {0} into {1} splits each of size {2}\n'.format(len(df), splits, len(dfs[0]))) calcdfs = []
availddfFileName = basename + "_DDF_avail.csv" selectedddfFileName = basename + "_DDF_sel.csv" print('output file names for DDF are {0}, {1}, {2}'.format( ddf_simlibfilename, availddfFileName, selectedddfFileName)) numFields_DDF = args.numFields_DDF numFields_WFD = args.numFields_WFD print(args) sys.stdout.flush() # find ddf healpixels if get_ddf_pixels: print('Finding the DDF healpixels \n') opsout_ddf = OpSimOutput.fromOpSimDB(dbname, opsimversion=opsimversion, subset='ddf', dithercolumns=dithercolumns, filterNull=filternulls) if len(opsout_ddf.summary) > 0: print("writing out ddf pixels\n") simlib_ddf = Simlibs(opsout_ddf.summary, opsimversion=opsimversion, usePointingTree=True) ddf_hid = set(simlib_ddf.observedVisitsinRegion().index.values) else: print("writing out null set of ddf pixels\n") ddf_hid = set([]) print("written out null set of ddf pixels\n") print('There are {} pixels in the ddf fields'.format(len(ddf_hid))) # read the database into a `pd.DataFrame`
opsim_fname = 'feature_rolling_half_mask_10yrs.db' simlib_fname = 'feature_rolling_half_mask_10yrs.simlib' script_start = time.time() log_str = 'Running script with opsimsummary version {}\n'.format( oss.__version__) log_val = 'Starting Calculation at {}\n'.format(script_start) log_str += log_val pkgDir = os.path.split(oss.__file__)[0] dbname = os.path.join('/Users/rbiswas/data/', 'LSST/OpSimData', opsim_fname) log_val = 'The OpSim DataBase used is {}\n'.format(dbname) log_str += log_val # read the database into a `pd.DataFrame` opsout = OpSimOutput.fromOpSimDB(dbname, opsimversion='lsstv4', tableNames=('SummaryAllProps', 'Proposal')) summary = opsout.summary log_val = 'dataframe read in from database {}\n'.format(time.time()) log_str += log_val simlibs = Simlibs(summary, opsimversion='lsstv4', usePointingTree=True) rng = np.random.RandomState(1) simlibs.randomSimlibs(numFields=50000, fname=simlib_fname) #, rng=rng) log_val = 'Done' log_str += log_val log_val = 'Writing simlib for input to outfile {0} at time {1}\n'.format( simlib_fname, time.time()) print(log_val) log_str += log_val