def testHistogramMetric(self): metric = metrics.HistogramMetric(bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) good = np.where(mb.metricValues.mask[:, -1] == False)[0] expected = np.array([[self.n1, 0.], [0., self.n2]]) assert (np.array_equal(mb.metricValues.data[good, :], expected)) # Check that I can run a different statistic metric = metrics.HistogramMetric(col='fiveSigmaDepth', statistic='sum', bins=[0.5, 1.5, 2.5]) mb = metricBundle.MetricBundle(metric, slicer, sql) mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) expected = np.array([[self.m5_1 * self.n1, 0.], [0., self.m5_2 * self.n2]]) assert (np.array_equal(mb.metricValues.data[good, :], expected))
def __getOpSimMjd(self, opsim, ra, dec, fil): colmn = 'observationStartMJD' opsdb = db.OpsimDatabase(opsim) # Directory where tmp files are going to be stored TODO eliminate - this outDir = 'TmpDir' resultsDb = db.ResultsDb(outDir=outDir) metric = metrics.PassMetric(cols=[colmn, 'fiveSigmaDepth', 'filter']) slicer = slicers.UserPointsSlicer(ra=ra, dec=dec) sqlconstraint = 'filter = \'' + fil + '\'' bundle = mb.MetricBundle(metric, slicer, sqlconstraint, runName='name') bgroup = mb.MetricBundleGroup({0: bundle}, opsdb, outDir=outDir, resultsDb=resultsDb) bgroup.runAll() filters = np.unique(bundle.metricValues[0]['filter']) mv = bundle.metricValues[0] # Get dates mjd = mv[colmn] mjd = np.sort(mjd) print('Num of visits ' + str(len(mjd)) + ' ' + opsim) return mjd
def testHistogramM5Metric(self): metric = metrics.HistogramM5Metric(bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) # Clobber the stacker that gets auto-added mb.stackerList = [] mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) good = np.where((mb.metricValues.mask[:, 0] == False) | (mb.metricValues.mask[:, 1] == False))[0] checkMetric = metrics.Coaddm5Metric() tempSlice = np.zeros(self.n1, dtype=list(zip(['fiveSigmaDepth'], [float]))) tempSlice['fiveSigmaDepth'] += self.m5_1 val1 = checkMetric.run(tempSlice) tempSlice = np.zeros(self.n2, dtype=list(zip(['fiveSigmaDepth'], [float]))) tempSlice['fiveSigmaDepth'] += self.m5_2 val2 = checkMetric.run(tempSlice) expected = np.array([[val1, -666.], [-666., val2]]) assert (np.array_equal(mb.metricValues.data[good, :], expected))
def testOut(self): """ Check that the metric bundle can generate the expected output """ slicer = slicers.HealpixSlicer(nside=8) metric = metrics.MeanMetric(col='airmass') sql = 'filter="r"' metricB = metricBundles.MetricBundle(metric, slicer, sql) filepath = os.path.join(os.getenv('SIMS_MAF_DIR'), 'tests/') database = os.path.join(filepath, 'opsimblitz1_1133_sqlite.db') opsdb = db.OpsimDatabase(database=database) resultsDb = db.ResultsDb(outDir=self.outDir) bgroup = metricBundles.MetricBundleGroup({0: metricB}, opsdb, outDir=self.outDir, resultsDb=resultsDb) bgroup.runAll() bgroup.plotAll() bgroup.writeAll() outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*')) outNpz = glob.glob(os.path.join(self.outDir, '*.npz')) outPdf = glob.glob(os.path.join(self.outDir, '*.pdf')) # By default, make 3 plots for healpix assert (len(outThumbs) == 3) assert (len(outPdf) == 3) assert (len(outNpz) == 1)
def runGlance(dbfile, outDir='Glance', runName='runname'): conn = db.SimpleDatabase(dbfile, defaultTable='SummaryAllProps', defaultdbTables={ 'SummaryAllProps': ['SummaryAllProps', 'observationId'] }) colmap = { 'ra': 'RA', 'dec': 'dec', 'mjd': 'mjd', 'exptime': 'exptime', 'visittime': 'exptime', 'alt': 'alt', 'az': 'az', 'filter': 'filter', 'fiveSigmaDepth': 'fivesigmadepth', 'night': 'night', 'slewtime': 'slewtime', 'seeingGeom': 'FWHM_geometric' } gb = glanceBundle(colmap_dict=colmap) resultsDb = db.ResultsDb(outDir=outDir) group = metricBundles.MetricBundleGroup(gb, conn, outDir=outDir, resultsDb=resultsDb, runName=runName) group.runAll() group.plotAll()
def compute_metric(params): """Function to execute the metric calculation when code is called from the commandline""" obsdb = db.OpsimDatabase('../../tutorials/baseline2018a.db') outputDir = '/home/docmaf/' resultsDb = db.ResultsDb(outDir=outputDir) (propids, proptags) = obsdb.fetchPropInfo() surveyWhere = obsdb.createSQLWhere(params['survey'], proptags) obs_params = { 'filters': params['filters'], 'cadence': params['cadence'], 'start_date': params['start_date'], 'end_date': params['end_date'] } metric = CadenceOverVisibilityWindowMetric(**obs_params) slicer = slicers.HealpixSlicer(nside=64) sqlconstraint = surveyWhere bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint) bgroup = metricBundles.MetricBundleGroup({0: bundle}, obsdb, outDir='newmetric_test', resultsDb=resultsDb) bgroup.runAll()
def run(bdict, opsdb, colmap, args): resultsDb = db.ResultsDb(outDir=args.outDir) group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb) group.runAll() group.plotAll() resultsDb.close() mafUtils.writeConfigs(opsdb, args.outDir)
def load_and_run(): dbFile = 'baseline_nexp2_v1.7_10yrs.db' opsimdb = db.OpsimDatabase(dbFile) runName = dbFile.replace('.db', '') nside = 64 slicer = slicers.HealpixSlicer(nside=nside) metric = SNNSNMetric(verbose=False) #, zlim_coeff=0.98) bundleList = [] #sql = '' sql = '(note = "%s")' % ('DD:COSMOS') bundleList.append( metricBundles.MetricBundle(metric, slicer, sql, runName=runName)) outDir = 'temp' resultsDb = db.ResultsDb(outDir=outDir) bundleDict = metricBundles.makeBundlesDictFromList(bundleList) bgroup = metricBundles.MetricBundleGroup(bundleDict, opsimdb, outDir=outDir, resultsDb=resultsDb) bgroup.runAll() bgroup.plotAll()
def runChips(useCamera=False): import numpy as np import lsst.sims.maf.slicers as slicers import lsst.sims.maf.metrics as metrics import lsst.sims.maf.metricBundles as metricBundles import lsst.sims.maf.db as db from lsst.sims.maf.plots import PlotHandler import matplotlib.pylab as plt import healpy as hp print 'Camera setting = ', useCamera database = 'enigma_1189_sqlite.db' sqlWhere = 'filter = "r" and night < 800 and fieldRA < %f and fieldDec > %f and fieldDec < 0' % (np.radians(15), np.radians(-15)) opsdb = db.OpsimDatabase(database) outDir = 'Camera' resultsDb = db.ResultsDb(outDir=outDir) nside=512 tag = 'F' if useCamera: tag='T' metric = metrics.CountMetric('expMJD', metricName='chipgap_%s'%tag) slicer = slicers.HealpixSlicer(nside=nside, useCamera=useCamera) bundle1 = metricBundles.MetricBundle(metric,slicer,sqlWhere) bg = metricBundles.MetricBundleGroup({0:bundle1},opsdb, outDir=outDir, resultsDb=resultsDb) bg.runAll() hp.gnomview(bundle1.metricValues, xsize=800,ysize=800, rot=(7,-7,0), unit='Count', min=1) plt.savefig(outDir+'/fig'+tag+'.png')
def replot(bdict, opsdb, colmap, args): resultsDb = db.ResultsDb(outDir=args.outDir) group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb) group.readAll() group.plotAll() resultsDb.close()
def runSlew(opsdb, colmap, args): resultsDb = db.ResultsDb(outDir=args.outDir) bdict = batches.slewBasics(colmap, args.runName) dbTable = None group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb, dbTable=dbTable) group.runAll() group.plotAll() if 'slewStatesTable' in colmap: bdict = batches.slewAngles(colmap, args.runName) dbTable = colmap['slewStatesTable'] group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb, dbTable=dbTable) group.runAll() group.plotAll() if 'slewSpeedsTable' in colmap: bdict = batches.slewSpeeds(colmap, args.runName) dbTable = colmap['slewSpeedsTable'] group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb, dbTable=dbTable) group.runAll() group.plotAll() if 'slewActivitiesTable' in colmap: nslews = opsdb.fetchTotalSlewN() bdict = batches.slewActivities(colmap, args.runName, totalSlewN=nslews) dbTable = colmap['slewActivitiesTable'] group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb, dbTable=dbTable) group.runAll() group.plotAll() resultsDb.close() mafUtils.writeConfigs(opsdb, args.outDir)
def run_maf(dbFile, ra, dec): """Retrive min inter_night gap, and observation history with the input of database file name and arrays of RA and DEC. Note: the observing cadence returned are not ordered by date!! """ # establish connection to sqllite database file. opsimdb = db.OpsimDatabase(dbFile) # While we're in transition between opsim v3 and v4, this may be helpful: print("{dbFile} is an opsim version {version} database".format(dbFile=dbFile, version=opsimdb.opsimVersion)) if opsimdb.opsimVersion == "V3": # For v3 databases: mjdcol = 'expMJD' degrees = False cols = ['filter', 'fiveSigmaDepth', mjdcol, 'expDate'] stackerList = [] else: # For v4 and alternate scheduler databases. mjdcol = 'observationStartMJD' degrees = True cols = ['filter', 'fiveSigmaDepth', mjdcol] stackerList = [expDateStacker()] # IntraNightGapsMetric returns the gap (in days) between observations within the same night custom reduceFunc to find min gaps metric = metrics.cadenceMetrics.IntraNightGapsMetric(reduceFunc=np.amin, mjdCol=mjdcol) # PassMetric just pass all values metric_pass = metrics.simpleMetrics.PassMetric(cols=cols) # slicer for slicing pointing history slicer = slicers.UserPointsSlicer(ra, dec, lonCol='fieldRA', latCol='fieldDec', latLonDeg=degrees) # sql constrains, 3 for baseline2018a, 1 for rolling m2045 sql = '' # bundles to combine metric, slicer and sql constrain together bundle = metricBundles.MetricBundle(metric, slicer, sql) date_bundle = metricBundles.MetricBundle(metric_pass, slicer, sql, stackerList=stackerList) # create metric bundle group and returns bg = metricBundles.MetricBundleGroup( { 'sep': bundle, 'cadence': date_bundle }, opsimdb, outDir=outDir, resultsDb=resultsDb) bg.runAll() opsimdb.close() return bg
def replotSlew(opsdb, colmap, args): print('Only replots slew basics batch.') bdict = batches.slewBasics(colmap, args.runName) resultsDb = db.ResultsDb(outDir=args.outDir) group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb) group.readAll() group.plotAll() resultsDb.close()
def testOpsim2dSlicer(self): metric = metrics.AccumulateCountMetric(bins=[0.5, 1.5, 2.5]) slicer = slicers.OpsimFieldSlicer() sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.fieldData = self.fieldData mbg.runCurrent('', simData=self.simData) expected = np.array([[self.n1, self.n1], [-666., self.n2]]) assert (np.array_equal(mb.metricValues.data, expected))
def runGlance(dbfile, outDir='Glance', runName='runname', camera='LSST'): conn = db.Database(dbfile, defaultTable='observations') colmap = { 'ra': 'RA', 'dec': 'dec', 'mjd': 'mjd', 'exptime': 'exptime', 'visittime': 'exptime', 'alt': 'alt', 'az': 'az', 'filter': 'filter', 'fiveSigmaDepth': 'fivesigmadepth', 'night': 'night', 'slewtime': 'slewtime', 'seeingGeom': 'FWHM_geometric', 'rotSkyPos': 'rotSkyPos', 'raDecDeg': True, 'slewdist': None, 'note': 'note' } #del colmap['note'] gb = glanceBatch(colmap=colmap, slicer_camera=camera, runName=runName) resultsDb = db.ResultsDb(outDir=outDir) group = metricBundles.MetricBundleGroup(gb, conn, outDir=outDir, resultsDb=resultsDb) group.runAll() group.plotAll() fb = fOBatch(colmap=colmap, runName=runName) group = metricBundles.MetricBundleGroup(fb, conn, outDir=outDir, resultsDb=resultsDb) group.runAll() group.plotAll()
def testHealpix2dSlicer(self): metric = metrics.AccumulateCountMetric(bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) good = np.where(mb.metricValues.mask[:, -1] == False)[0] expected = np.array([[self.n1, self.n1], [-666., self.n2]]) assert (np.array_equal(mb.metricValues.data[good, :], expected))
def run(bdict, opsdb, colmap, args, save=True): resultsDb = db.ResultsDb(outDir=args.outDir) group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb, saveEarly=False) if save: group.runAll() group.plotAll() else: group.runAll(clearMemory=True, plotNow=True) resultsDb.close() mafUtils.writeConfigs(opsdb, args.outDir)
def testAccumulateMetric(self): metric = metrics.AccumulateMetric(col='fiveSigmaDepth', bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) # Clobber the stacker that gets auto-added mb.stackerList = [] mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) good = np.where(mb.metricValues.mask[:, -1] == False)[0] expected = np.array([[self.n1*self.m5_1, self.n1*self.m5_1], [-666., self.n2 * self.m5_2]]) assert(np.array_equal(mb.metricValues.data[good, :], expected))
def testRunRegularToo(self): """ Test that a binned slicer and a regular slicer can run together """ bundleList = [] metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' bundleList.append(metricBundle.MetricBundle(metric, slicer, sql)) metric = metrics.Coaddm5Metric() slicer = slicers.HealpixSlicer(nside=16) bundleList.append(metricBundle.MetricBundle(metric, slicer, sql)) bd = metricBundle.makeBundlesDictFromList(bundleList) mbg = metricBundle.MetricBundleGroup(bd, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) assert (np.array_equal(bundleList[0].metricValues[:, 1].compressed(), bundleList[1].metricValues.compressed()))
def testOut(self): """ Check that the metric bundle can generate the expected output """ nside = 8 slicer = slicers.HealpixSlicer(nside=nside) metric = metrics.MeanMetric(col='airmass') sql = 'filter="r"' stacker1 = stackers.RandomDitherFieldPerVisitStacker() stacker2 = stackers.GalacticStacker() map1 = maps.GalCoordsMap() map2 = maps.StellarDensityMap() metricB = metricBundles.MetricBundle(metric, slicer, sql, stackerList=[stacker1, stacker2], mapsList=[map1, map2]) database = os.path.join(getPackageDir('sims_data'), 'OpSimData', 'astro-lsst-01_2014.db') opsdb = db.OpsimDatabaseV4(database=database) resultsDb = db.ResultsDb(outDir=self.outDir) bgroup = metricBundles.MetricBundleGroup({0: metricB}, opsdb, outDir=self.outDir, resultsDb=resultsDb) bgroup.runAll() bgroup.plotAll() bgroup.writeAll() opsdb.close() outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*')) outNpz = glob.glob(os.path.join(self.outDir, '*.npz')) outPdf = glob.glob(os.path.join(self.outDir, '*.pdf')) # By default, make 3 plots for healpix assert (len(outThumbs) == 3) assert (len(outPdf) == 3) assert (len(outNpz) == 1)
def testAccumulateM5Metric(self): metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5]) slicer = slicers.HealpixSlicer(nside=16) sql = '' mb = metricBundle.MetricBundle(metric, slicer, sql) mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False) mbg.setCurrent('') mbg.runCurrent('', simData=self.simData) good = np.where(mb.metricValues.mask[:, -1] == False)[0] checkMetric = metrics.Coaddm5Metric() tempSlice = np.zeros(self.n1, dtype=zip(['fiveSigmaDepth'], [float])) tempSlice['fiveSigmaDepth'] += self.m5_1 val1 = checkMetric.run(tempSlice) tempSlice = np.zeros(self.n2, dtype=zip(['fiveSigmaDepth'], [float])) tempSlice['fiveSigmaDepth'] += self.m5_2 val2 = checkMetric.run(tempSlice) expected = np.array([[val1, val1], [-666., val2]]) assert (np.array_equal(mb.metricValues.data[good, :], expected))
def get_cadence(ra, dec, b, snrLimit, nPtsLimit, filters, outDir, opsimdb, resultsDb): # The pass metric just passes data straight through. metric = metrics.PassMetric(cols=['filter', 'fiveSigmaDepth', 'expMJD']) slicer = slicers.UserPointsSlicer(ra, dec, lonCol='ditheredRA', latCol='ditheredDec') sql = '' bundle = metricBundles.MetricBundle(metric, slicer, sql) bg = metricBundles.MetricBundleGroup({0: bundle}, opsimdb, outDir=outDir, resultsDb=resultsDb) bg.runAll() bundle.metricValues.data[0]['filter'] print("Plotting...") colors = {'u': 'cyan', 'g': 'g', 'r': 'y', 'i': 'r', 'z': 'm', 'y': 'k'} dayZero = bundle.metricValues.data[0]['expMJD'].min() times = [] depths = [] plt.clf() for fname in filters: good = np.where(bundle.metricValues.data[0]['filter'] == fname) times.append(bundle.metricValues.data[0]['expMJD'][good] - dayZero) depths.append(bundle.metricValues.data[0]['fiveSigmaDepth'][good]) plt.scatter(bundle.metricValues.data[0]['expMJD'][good] - dayZero, bundle.metricValues.data[0]['fiveSigmaDepth'][good], c=colors[fname], label=fname) plt.xlabel('Day') plt.ylabel('5$\sigma$ depth') plt.legend(scatterpoints=1, loc="upper left", bbox_to_anchor=(1, 1)) plt.savefig("l45b{0}_cadence.pdf".format(int(b))) return times, depths
def testOut(self): """ Check that the metric bundle can generate the expected output """ nside = 8 slicer = slicers.HealpixSlicer(nside=nside) metric = metrics.MeanMetric(col='airmass') sql = 'filter="r"' stacker1 = stackers.RandomDitherFieldPerVisitStacker() stacker2 = stackers.GalacticStacker() map1 = maps.GalCoordsMap() map2 = maps.StellarDensityMap() metricB = metricBundles.MetricBundle(metric, slicer, sql, stackerList=[stacker1, stacker2]) filepath = os.path.join(os.getenv('SIMS_MAF_DIR'), 'tests/') database = os.path.join(filepath, 'opsimblitz1_1133_sqlite.db') opsdb = db.OpsimDatabase(database=database) resultsDb = db.ResultsDb(outDir=self.outDir) bgroup = metricBundles.MetricBundleGroup({0: metricB}, opsdb, outDir=self.outDir, resultsDb=resultsDb) bgroup.runAll() bgroup.plotAll() bgroup.writeAll() outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*')) outNpz = glob.glob(os.path.join(self.outDir, '*.npz')) outPdf = glob.glob(os.path.join(self.outDir, '*.pdf')) # By default, make 3 plots for healpix assert (len(outThumbs) == 3) assert (len(outPdf) == 3) assert (len(outNpz) == 1)
'assuming WFD == area with 750 or more visits.') parser.add_argument('dbfile', type=str, help='sqlite file of observations (full path).') args = parser.parse_args() # Pull out the visits. We'll reuse these for the footprint calculation below. # Note that visits is a dataframe. visits = get_visits(args.dbfile) # Instead of always figuring out the WFD footprint from what we expected, let's define it based on # what we got .. and define the "WFD" as the area with at least 750 visits per pointing. runName = os.path.split(args.dbfile)[-1].replace('.db', '') nside = 64 # Find the WFD footprint m = CountMetric(col='observationStartMJD') s = HealpixSlicer(nside) simdata = visits.query('visitExposureTime > 11') simdata = simdata.query('not note.str.startswith("DD")', engine='python').to_records() bundle = mb.MetricBundle(m, s, 'long notDD', runName=runName) g = mb.MetricBundleGroup({f'{runName}': bundle}, None) g.setCurrent('long notDD') g.runCurrent('long notDD', simData=simdata) wfd_footprint = bundle.metricValues.filled(0) wfd_footprint = np.where(wfd_footprint > 750, 1, 0) visits, propTags, propId = label_visits(visits, wfd_footprint) update_database(args.dbfile, visits, propTags, propId)
def ResultadosNtotBolV2(FBS, mod): # ========================================================== # mod = "A" # FBS = "1.5" # modo = "A" # filtros_considerados = ["u","g"] # f1,f2 tq f2 mas rojo que f1 # ========================================================== #validacion(filtros_considerados) #f1,f2 = filtros_considerados # g_modA_LookupT_extension.pk # lookup_table = "{}_mod{}_LookupT_extension.pkl".format(f2, modo) # debe estar en la carpeta de /lookuptables en /essentials # f2 porque ese se ocupa , el f1 es para potencial lyman pbreak nomas #filtros_modo = "{}_mod{}".format("".join(filtros_considerados),modo) print("FBS usado:", FBS) print("mod:", mod) ##################################################################################### ################################## 3 BUNDLES ######################################## ##################################################################################### metric = NtotMetricV2(mod, f1f2diff=2) # ========================= WFD ================================= constraint1 = "note NOT LIKE '%DD%'" wfd_standard = schedUtils.WFD_no_gp_healpixels( 64) #, dec_max=2.5, dec_min=-62.5) slicer1 = slicers.HealpixSubsetSlicer( 64, np.where(wfd_standard == 1)[0] ) #nside = 64, hpid = The subset of healpix id's to use to calculate the metric. bundle1 = mb.MetricBundle(metric, slicer1, constraint1) # ========================= DDF ================================= constraint2 = "note LIKE '%DD%'" slicer2 = slicers.HealpixSlicer(nside=64) bundle2 = mb.MetricBundle(metric, slicer2, constraint2) print("==============================================") print("constraint WFD:" + constraint1) print("constraint DDF:" + constraint2) ##################################################################################### ################################# DIRECTORIOS ####################################### ##################################################################################### #Please enter your SciServer username between the single quotes below! # your_username = '******' # Check avaiable database directoies show_fbs_dirs() # if your_username == '': # do NOT put your username here, put it in the cell at the top of the notebook. # raise Exception('Please provide your username! See the top of the notebook.') dbDir = './lsst_cadence/FBS_{}/'.format(FBS) outDir = '/data/agonzalez/output_FBS_{}/bolNtot_mod{}_FINAL/'.format( FBS, mod) if not os.path.exists(os.path.abspath(outDir)): os.makedirs(os.path.abspath(outDir), exist_ok=True) opSimDbs, resultDbs = connect_dbs(dbDir, outDir) metricDataPath = '/data/agonzalez/output_FBS_{}/bolNtot_mod{}_FINAL/MetricData/'.format( FBS, mod) if not os.path.exists(os.path.abspath(metricDataPath)): os.makedirs(os.path.abspath(metricDataPath), exist_ok=True) print("===================================================") print("dbDir :", dbDir) print("outDir :", outDir) print("metricDataPath :", metricDataPath) print("===================================================") ##################################################################################### ################################# BUNDLE GROUP ###################################### ##################################################################################### dbRuns = show_opsims(dbDir) print(dbRuns) dbRuns = [x for x in dbRuns if "noddf" not in x] #archivo70plus = open("jhu70plus{}.txt".format(FBS),"r") #dbRuns = [x.rstrip() for x in list(archivo70plus)] #archivo70plus.close() for run in dbRuns: #[70:]: bDict = {"WFD": bundle1, "DDF": bundle2} bundle1.setRunName(run) bundle2.setRunName(run) bgroup = mb.MetricBundleGroup(bDict, opSimDbs[run], metricDataPath, resultDbs[run]) bgroup.runAll()
args, extras = parser.parse_known_args() # Build metric bundles. (bundleDict, mergedHistDict, noSaveBundleDict) = makeBundleList(args.dbFile, nside=args.nside, lonCol=args.lonCol, latCol=args.latCol, benchmark=args.benchmark, seeingCol=args.seeingCol) # Set up / connect to resultsDb. resultsDb = db.ResultsDb(outDir=args.outDir) # Connect to opsimdb. opsdb = db.OpsimDatabaseV4(args.dbFile) if args.runNoSave: group = metricBundles.MetricBundleGroup(noSaveBundleDict, opsdb, saveEarly=False, outDir=args.outDir, resultsDb=resultsDb) group.runAll(clearMemory=True, plotNow=True) del group, noSaveBundleDict # Set up metricBundleGroup. group = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=args.outDir, resultsDb=resultsDb) # Read or run to get metric values. if args.plotOnly: group.readAll() else: group.runAll() # Make plots. group.plotAll() # Make merged plots. for key in mergedHistDict:
def runSlices(opsimName, metadata, simdata, bins, args, verbose=False): """ Set up and run the movie slicer, and at each step, setup the healpix slicer and run the metrics, creating and saving the plots. """ # Set up movie slicer movieslicer = setupMovieSlicer(simdata, binsize=args.movieStepsize, cumulative=args.cumulative) start_date = movieslicer[0]['slicePoint']['binLeft'] sliceformat = '%s0%dd' % ('%', int(np.log10(len(movieslicer))) + 1) # Run through the movie slicer slicePoints: for i, movieslice in enumerate(movieslicer): t = time.time() slicenumber = sliceformat % (i) """ # Set up plot label. timeInterval = '%.2f to %.2f' %(movieslice['slicePoint']['binLeft']-start_date, movieslice['slicePoint']['binRight']-start_date) """ # Or add simple view of time to plot label. times_from_start = movieslice['slicePoint']['binRight'] - ( int(bins[0]) + 0.16 - 0.5) # Opsim years are 365 days (not 365.25) years = int(times_from_start / 365) days = times_from_start - years * 365 plotlabel = 'Year %d Day %.4f' % (years, days) metricList, plotDictList = setupMetrics( opsimName, metadata, start_date, movieslice['slicePoint']['binRight'], cumulative=args.cumulative, plotlabel=plotlabel, verbose=verbose) # Identify the subset of simdata in the movieslicer 'data slice' simdatasubset = simdata[movieslice['idxs']] # Set up healpix slicer on subset of simdata provided by movieslicer hs = setupHealpixSlicer(args) bundles = [] for metric, plotDict in zip(metricList, plotDictList): bundles.append( mB.MetricBundle(metric, hs, sqlconstraint=args.sqlConstraint, metadata=metadata, runName=opsimName, plotDict=plotDict, plotFuncs=[plots.HealpixSkyMap()])) # Remove (default) stackers from bundles, because we've already run them above on the original data. for mb in bundles: mb.stackerList = [] bundledict = mB.makeBundlesDictFromList(bundles) # Set up metricBundleGroup to handle metrics calculation + plotting bg = mB.MetricBundleGroup(bundledict, opsDb, outDir=args.outDir, resultsDb=None, saveEarly=False) # Calculate metric data values for simdatasubset (this also sets up indexing in the slicer) bg.setCurrent(args.sqlConstraint) bg.runCurrent(constraint=args.sqlConstraint, simData=simdatasubset) # Plot data for this slice of the movie, adding slicenumber as a suffix for output plots bg.plotAll(outfileSuffix=slicenumber, closefigs=True, dpi=72, thumbnail=False, figformat='png') # Write the data -- uncomment if you want to do this. # sm.writeAll(outfileSuffix=slicenumber) if verbose: dt, t = dtime(t) print('Ran and plotted slice %s of movieslicer in %f s' % (slicenumber, dt))
if args.db is None: if os.path.isfile('trackingDb_sqlite.db'): os.remove('trackingDb_sqlite.db') db_files = glob.glob('*.db') else: db_files = [args.db] run_names = [ os.path.basename(name).replace('.db', '') for name in db_files ] for filename, name in zip(db_files, run_names): if os.path.isdir(name + '_glance'): shutil.rmtree(name + '_glance') opsdb = db.OpsimDatabaseV4(filename) colmap = batches.ColMapDict() bdict = {} bdict.update(batches.glanceBatch(colmap, name)) bdict.update(batches.fOBatch(colmap, name)) resultsDb = db.ResultsDb(outDir=name + '_glance') group = mb.MetricBundleGroup(bdict, opsdb, outDir=name + '_glance', resultsDb=resultsDb, saveEarly=False) group.runAll(clearMemory=True, plotNow=True) resultsDb.close() opsdb.close() db.addRunToDatabase(name + '_glance', 'trackingDb_sqlite.db', None, name, '', '', name + '.db')
" Default=fieldRA.") parser.add_argument( "--latCol", type=str, default='fieldDec', help="Column to use for Dec values (can be a stacker dither column)." + " Default=fieldDec.") parser.add_argument('--night', type=int, default=1) parser.set_defaults() args, extras = parser.parse_known_args() bundleDict = makeBundleList(args.dbFile, nside=args.nside, lonCol=args.lonCol, latCol=args.latCol, night=args.night) # Set up / connect to resultsDb. resultsDb = db.ResultsDb(outDir=args.outDir) # Connect to opsimdb. opsdb = db.OpsimDatabaseV3(args.dbFile) # Set up metricBundleGroup. group = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=args.outDir, resultsDb=resultsDb) group.runAll() group.plotAll()
def go(nside=64, rmag=21., SedTemplate='flat', DoRun=False, LFilters = [], \ LNightMax=[], nightMax=1e4, \ CustomPlotLimits=True, \ RunOne=False, MaxRuns=1e3, \ SpatialClip=95., \ seeingCol='FWHMeff', \ sCmap='cubehelix_r', \ checkCorrKind=False, \ wfdPlane=True, \ useGRIZ=False): # Go to the directory where the sqlite databases are held... # cd /Users/clarkson/Data/LSST/OpSimRuns/opsim20160411 # WIC 2015-12-29 - set up for a master-run with all cases, this time with plotting limits # Break the specifications across lines to make subdivision easier # Subsets by time first, then by filter, finally the whole shebang # 2016-04-23 - replaced enigma_1189 --> minion_1016 # 2016-04-23 - replaced ops2_1092 --> minion_1020 # (Yes the inversion of the first two is deliberate.) runNames = ['minion_1016', 'minion_1020', 'minion_1020', 'minion_1016', \ 'minion_1020', 'minion_1016', 'minion_1020', 'minion_1016', \ 'minion_1020', 'minion_1016'] LFilters = ['', '', '', '', \ 'u', 'u', 'y', 'y', \ '', ''] LNightMax = [365, 365, 730, 730, \ 1e4, 1e4, 1e4, 1e4, \ 1e4, 1e4] # WIC try again, this time on the new astro_lsst_01_1004 only if wfdPlane: LFilters = ['', '', '', 'u', 'y'] LNightMax = [365, 730, 1e4, 1e4, 1e4] runNames = ['astro_lsst_01_1004' for i in range (len(LFilters)) ] # WIC 2016-05-01 check correlation if checkCorrKind: LFilters = ['', ''] LNightMax = [365, 365] runNames = ['minion_1016', 'minion_1016'] # Type of correlation used for HA Degen # checkCorrKind = True useSpearmanR = [False, True] if useGRIZ: runNames=['minion_1016','astro_lsst_01_1004', 'minion_1020'] LFilters = ['griz' for iRun in range(len(runNames)) ] #LNightMax = [1e4 for iRun in range(len(runNames)) ] #LNightMax = [730 for iRun in range(len(runNames)) ] LNightMax = [365 for iRun in range(len(runNames)) ] # List of upper limits to parallax and proper motion error. For parallax, 3.0 mas is probably good LUpperParallax = [] LUpperPropmotion = [] if CustomPlotLimits: LUpperParallax = [10, 10, 10, 10, \ 10, 10, 40, 40, \ 3.0, 3.0 ] # For proper motion, it's a little tricky to say because the # regular case is so pathological for the field. Try the following: LUpperPropmotion = [40, 40, 5, 20, \ 3.5, 20, 3.5, 20, \ 0.5, 5] if len(runNames) < 2: LUpperPropmotion = [100 for i in range(len(runNames))] print "runAstrom.go INFO - will run the following:" for iSho in range(len(runNames)): sFilThis = '' # print iSho, len(LFilters) if iSho <= len(LFilters): sFilThis = sqlFromFilterString(LFilters[iSho]) print "%i: %-12s, %1s, %i, sqlFilter -- %s" % (iSho, runNames[iSho], LFilters[iSho], LNightMax[iSho], sFilThis) print "===========================" print "mag max = %.2f" % (rmag) print "---------------------------" # print runNames # if not DoRun: # print "Set DoRun=True to actually run this." # print len(LFilters), len(runNames), len(LFilters) == len(runNames) # return #'kraken_1038', 'kraken_1034', 'ops2_1098'] # nside = 64 slicer = slicers.HealpixSlicer(nside=nside) # Make it so we don't bother with the silly power spectra plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()] # WIC - back up the plotting arguments with a default value plotFuncsPristine = copy.deepcopy(plotFuncs) # WIC - the only way this will make sense to me is if I make a # dictionary of plot arguments. Let's try it... DPlotArgs = {} for plotArg in ['parallax', 'propmotion', 'coverage', 'HAdegen']: DPlotArgs[plotArg] = copy.deepcopy(plotFuncs) if CustomPlotLimits: # Use the same color map for all the metrics for plotMetric in DPlotArgs.keys(): DPlotArgs[plotMetric][0].defaultPlotDict['cmap'] = sCmap # Apply spatial clipping for all but the HADegen, for which we # have other limits... for plotMetric in ['parallax', 'propmotion', 'coverage']: DPlotArgs[plotMetric][0].defaultPlotDict['percentileClip'] = SpatialClip # Some limits common to spatial maps and histograms for iPl in range(0,2): DPlotArgs['propmotion'][iPl].defaultPlotDict['logScale'] = True # NOT a loop because we might want to separate out the behavior # Standardized range for the histograms for new parallax metrics DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0. DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1. DPlotArgs['HAdegen'][1].defaultPlotDict['xMin'] = -1. DPlotArgs['HAdegen'][1].defaultPlotDict['xMax'] = 1. # Standardize the sky map for the HAdegen as well. DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0. DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1. DPlotArgs['HAdegen'][0].defaultPlotDict['xMin'] = -1. DPlotArgs['HAdegen'][0].defaultPlotDict['xMax'] = 1. # Standardize at least the lower bound of the histogram in # both the proper motion and parallax errors. Upper limit we # can customize with a loop. DPlotArgs['propmotion'][1].defaultPlotDict['xMin'] = 1e-2 # should not be zero if log scale!! DPlotArgs['parallax'][1].defaultPlotDict['xMin'] = 0. # WIC - try changing the plot dictionary if not DoRun: plotFuncs[0].defaultPlotDict['logScale'] = True print DPlotArgs['propmotion'][0].defaultPlotDict print DPlotArgs['propmotion'][1].defaultPlotDict return # The old runs have the seeing in finSeeing #seeingCol = 'finSeeing' ### UPDATE THE SEEING COLUMN #seeingCol = 'FWHMeff' ## Moved up to a command-line argument # Use all the observations. Can change if you want a different # time span # sqlconstraint = '' # list of sqlconstraints now used, which gets handled within the loop. # run some summary stats on everything summaryMetrics = [metrics.MedianMetric()] tStart = time.time() # Running one, or the whole lot? RunMax = len(runNames) # allow user to set a different number (say, 2) if MaxRuns < RunMax and MaxRuns > 0: RunMax = int(MaxRuns) # the following keyword overrides if RunOne: RunMax = 1 print "Starting runs. RunMax = %i" % (RunMax) for iRun in range(RunMax): run = runNames[iRun][:] # for run in runNames: # Open the OpSim database timeStartIteration = time.time() # Some syntax added to test for existence of the database dbFil = run+'_sqlite.db' if not os.access(dbFil, os.R_OK): print "runAstrom.go FATAL - cannot acces db file %s" % (dbFil) print "runAstrom.go FATAL - skipping run %s" % (run) continue else: deltaT = time.time()-tStart print "runAstrom.go INFO - ##################################" print "runAstrom.go INFO - starting run %s with nside=%i after %.2f minutes" \ % (run, nside, deltaT/60.) opsdb = db.OpsimDatabase(run+'_sqlite.db') # Set SQL constraint appropriate for each filter in the # list. If we supplied a list of filters, use it for sqlconstraint = '' ThisFilter = 'ugrizy' if len(LFilters) == len(runNames): # Only change the filter if one was actually supplied! if len(LFilters[iRun]) > 0: ThisFilter = LFilters[iRun] sqlconstraint = sqlFromFilterString(ThisFilter) ### sqlconstraint = 'filter = "%s"' % (ThisFilter) # If nightmax was supplied, use it ThisNightMax = int(nightMax) # copy not view if len(LNightMax) == len(runNames): # Only update nightmax if one was given try: ThisNightMax = int(LNightMax[iRun]) # This might be redundant with the fmt statement below. if len(sqlconstraint) < 1: sqlconstraint = 'night < %i' % (ThisNightMax) else: sqlconstraint = '%s and night < %i' % (sqlconstraint, ThisNightMax) except: print "runAstrom.go WARN - run %i problem with NightMax" % (iRun) dumdum = 1. # Set where the output should go - include the filter!! sMag = '%.1f' % (rmag) sMag = sMag.replace(".","p") outDir = './metricEvals/%s_nside%i_%s_n%i_r%s' % (run, nside, ThisFilter, ThisNightMax, sMag) # Ensure we'll be able to find this later on... if CustomPlotLimits: outDir = '%s_lims' % (outDir) # if we are testing the kind of correlation used, include that # in the output here. if checkCorrKind: if useSpearmanR[iRun]: sCorr = 'spearmanR' else: sCorr = 'pearsonR' outDir = '%s_%s' % (outDir, sCorr) # From this point onwards, stuff actually gets run. This is # the place to output what will actually happen next. print "runAstrom.go INFO - about to run:" print "runAstrom.go INFO - sqlconstraint: %s ; run name %s ; nside %i" % (sqlconstraint, run, nside) print "runAstrom.go INFO - output directory will be %s" % (outDir) if not DoRun: continue # ensure the output directory actually exists... if not os.access(outDir, os.R_OK): print "runAstrom.go INFO - creating output directory %s" % (outDir) os.makedirs(outDir) resultsDb = db.ResultsDb(outDir=outDir) bundleList = [] # WIC - to make this at least somewhat uniform, build the plot # functions including arguments out of our copies above. plotFuncsPropmotion = copy.deepcopy(DPlotArgs['propmotion']) plotFuncsParallax = copy.deepcopy(DPlotArgs['parallax']) plotFuncsCoverage = copy.deepcopy(DPlotArgs['coverage']) plotFuncsHAdegen = copy.deepcopy(DPlotArgs['HAdegen']) # if using custom plot limits, will want to include the limits # for proper motion and parallax too... programming a bit defensively # here, including an extra check (rather than just the length of the lists # above). if CustomPlotLimits: if len(LUpperParallax) == len(runNames): plotFuncsParallax[1].defaultPlotDict['xMax'] = float(LUpperParallax[iRun]) if len(LUpperPropmotion) == len(runNames): plotFuncsPropmotion[1].defaultPlotDict['xMax'] = float(LUpperPropmotion[iRun]) # Configure the metrics metric = metrics.ParallaxMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate) bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run, # plotFuncs=plotFuncs, \ plotFuncs = plotFuncsParallax, \ summaryMetrics=summaryMetrics) bundleList.append(bundle) metric=metrics.ProperMotionMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate) bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run, # plotFuncs=plotFuncs, \ plotFuncs=plotFuncsPropmotion, \ summaryMetrics=summaryMetrics) bundleList.append(bundle) metric = calibrationMetrics.ParallaxCoverageMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate) bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run, # plotFuncs=plotFuncs, \ plotFuncs=plotFuncsCoverage, \ summaryMetrics=summaryMetrics) bundleList.append(bundle) # Now for the HA Degen metric. If testing the type of # correlation, call the metric differently here. Since the # argument to actually do this is only part of my github fork # at the moment, we use a different call. Running with default # arguments (checkCorrKind=False) should then work without # difficulty. metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate) if checkCorrKind: metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate, useSpearmanR=useSpearmanR[iRun]) print "TESTING CORRELATION KIND -- useSpearmanR", useSpearmanR[iRun] bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run, # plotFuncs=plotFuncs, \ plotFuncs=plotFuncsHAdegen, \ summaryMetrics=summaryMetrics) bundleList.append(bundle) # Run everything and make plots bundleDict = metricBundles.makeBundlesDictFromList(bundleList) bgroup = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=outDir, resultsDb=resultsDb) # try: bgroup.runAll() print "runAstrom.go INFO - bundles took %.2f minutes" \ % ((time.time() - timeStartIteration) / 60.) # except KeyboardInterrupt: # print "runAstrom.go FATAL - keyboard interrupt detected. Halting." # return bgroup.plotAll() print "runAstrom.go INFO - bundles + plotting took %.2f minutes" \ % ((time.time() - timeStartIteration) / 60.) print "Finished entire set. %i runs took %.2f minutes." % (iRun + 1, (time.time()-tStart)/60.)