def testHistogramMetric(self):
metric = metrics.HistogramMetric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
good = np.where(mb.metricValues.mask[:, -1] == False)[0]
expected = np.array([[self.n1, 0.], [0., self.n2]])
assert (np.array_equal(mb.metricValues.data[good, :], expected))
# Check that I can run a different statistic
metric = metrics.HistogramMetric(col='fiveSigmaDepth',
statistic='sum',
bins=[0.5, 1.5, 2.5])
mb = metricBundle.MetricBundle(metric, slicer, sql)
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
expected = np.array([[self.m5_1 * self.n1, 0.],
[0., self.m5_2 * self.n2]])
assert (np.array_equal(mb.metricValues.data[good, :], expected))
def __getOpSimMjd(self, opsim, ra, dec, fil):
colmn = 'observationStartMJD'
opsdb = db.OpsimDatabase(opsim)
# Directory where tmp files are going to be stored TODO eliminate - this
outDir = 'TmpDir'
resultsDb = db.ResultsDb(outDir=outDir)
metric = metrics.PassMetric(cols=[colmn, 'fiveSigmaDepth', 'filter'])
slicer = slicers.UserPointsSlicer(ra=ra, dec=dec)
sqlconstraint = 'filter = \'' + fil + '\''
bundle = mb.MetricBundle(metric, slicer, sqlconstraint, runName='name')
bgroup = mb.MetricBundleGroup({0: bundle},
opsdb,
outDir=outDir,
resultsDb=resultsDb)
bgroup.runAll()
filters = np.unique(bundle.metricValues[0]['filter'])
mv = bundle.metricValues[0]
# Get dates
mjd = mv[colmn]
mjd = np.sort(mjd)
print('Num of visits ' + str(len(mjd)) + ' ' + opsim)
return mjd
def testHistogramM5Metric(self):
metric = metrics.HistogramM5Metric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
# Clobber the stacker that gets auto-added
mb.stackerList = []
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
good = np.where((mb.metricValues.mask[:, 0] == False)
| (mb.metricValues.mask[:, 1] == False))[0]
checkMetric = metrics.Coaddm5Metric()
tempSlice = np.zeros(self.n1,
dtype=list(zip(['fiveSigmaDepth'], [float])))
tempSlice['fiveSigmaDepth'] += self.m5_1
val1 = checkMetric.run(tempSlice)
tempSlice = np.zeros(self.n2,
dtype=list(zip(['fiveSigmaDepth'], [float])))
tempSlice['fiveSigmaDepth'] += self.m5_2
val2 = checkMetric.run(tempSlice)
expected = np.array([[val1, -666.], [-666., val2]])
assert (np.array_equal(mb.metricValues.data[good, :], expected))
def testOut(self):
"""
Check that the metric bundle can generate the expected output
"""
slicer = slicers.HealpixSlicer(nside=8)
metric = metrics.MeanMetric(col='airmass')
sql = 'filter="r"'
metricB = metricBundles.MetricBundle(metric, slicer, sql)
filepath = os.path.join(os.getenv('SIMS_MAF_DIR'), 'tests/')
database = os.path.join(filepath, 'opsimblitz1_1133_sqlite.db')
opsdb = db.OpsimDatabase(database=database)
resultsDb = db.ResultsDb(outDir=self.outDir)
bgroup = metricBundles.MetricBundleGroup({0: metricB},
opsdb,
outDir=self.outDir,
resultsDb=resultsDb)
bgroup.runAll()
bgroup.plotAll()
bgroup.writeAll()
outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*'))
outNpz = glob.glob(os.path.join(self.outDir, '*.npz'))
outPdf = glob.glob(os.path.join(self.outDir, '*.pdf'))
# By default, make 3 plots for healpix
assert (len(outThumbs) == 3)
assert (len(outPdf) == 3)
assert (len(outNpz) == 1)
def runGlance(dbfile, outDir='Glance', runName='runname'):
conn = db.SimpleDatabase(dbfile,
defaultTable='SummaryAllProps',
defaultdbTables={
'SummaryAllProps':
['SummaryAllProps', 'observationId']
})
colmap = {
'ra': 'RA',
'dec': 'dec',
'mjd': 'mjd',
'exptime': 'exptime',
'visittime': 'exptime',
'alt': 'alt',
'az': 'az',
'filter': 'filter',
'fiveSigmaDepth': 'fivesigmadepth',
'night': 'night',
'slewtime': 'slewtime',
'seeingGeom': 'FWHM_geometric'
}
gb = glanceBundle(colmap_dict=colmap)
resultsDb = db.ResultsDb(outDir=outDir)
group = metricBundles.MetricBundleGroup(gb,
conn,
outDir=outDir,
resultsDb=resultsDb,
runName=runName)
group.runAll()
group.plotAll()
def compute_metric(params):
"""Function to execute the metric calculation when code is called from
the commandline"""
obsdb = db.OpsimDatabase('../../tutorials/baseline2018a.db')
outputDir = '/home/docmaf/'
resultsDb = db.ResultsDb(outDir=outputDir)
(propids, proptags) = obsdb.fetchPropInfo()
surveyWhere = obsdb.createSQLWhere(params['survey'], proptags)
obs_params = {
'filters': params['filters'],
'cadence': params['cadence'],
'start_date': params['start_date'],
'end_date': params['end_date']
}
metric = CadenceOverVisibilityWindowMetric(**obs_params)
slicer = slicers.HealpixSlicer(nside=64)
sqlconstraint = surveyWhere
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint)
bgroup = metricBundles.MetricBundleGroup({0: bundle},
obsdb,
outDir='newmetric_test',
resultsDb=resultsDb)
bgroup.runAll()
def run(bdict, opsdb, colmap, args):
resultsDb = db.ResultsDb(outDir=args.outDir)
group = mb.MetricBundleGroup(bdict, opsdb, outDir=args.outDir, resultsDb=resultsDb)
group.runAll()
group.plotAll()
resultsDb.close()
mafUtils.writeConfigs(opsdb, args.outDir)
def load_and_run():
dbFile = 'baseline_nexp2_v1.7_10yrs.db'
opsimdb = db.OpsimDatabase(dbFile)
runName = dbFile.replace('.db', '')
nside = 64
slicer = slicers.HealpixSlicer(nside=nside)
metric = SNNSNMetric(verbose=False) #, zlim_coeff=0.98)
bundleList = []
#sql = ''
sql = '(note = "%s")' % ('DD:COSMOS')
bundleList.append(
metricBundles.MetricBundle(metric, slicer, sql, runName=runName))
outDir = 'temp'
resultsDb = db.ResultsDb(outDir=outDir)
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict,
opsimdb,
outDir=outDir,
resultsDb=resultsDb)
bgroup.runAll()
bgroup.plotAll()
def runChips(useCamera=False):
import numpy as np
import lsst.sims.maf.slicers as slicers
import lsst.sims.maf.metrics as metrics
import lsst.sims.maf.metricBundles as metricBundles
import lsst.sims.maf.db as db
from lsst.sims.maf.plots import PlotHandler
import matplotlib.pylab as plt
import healpy as hp
print 'Camera setting = ', useCamera
database = 'enigma_1189_sqlite.db'
sqlWhere = 'filter = "r" and night < 800 and fieldRA < %f and fieldDec > %f and fieldDec < 0' % (np.radians(15), np.radians(-15))
opsdb = db.OpsimDatabase(database)
outDir = 'Camera'
resultsDb = db.ResultsDb(outDir=outDir)
nside=512
tag = 'F'
if useCamera:
tag='T'
metric = metrics.CountMetric('expMJD', metricName='chipgap_%s'%tag)
slicer = slicers.HealpixSlicer(nside=nside, useCamera=useCamera)
bundle1 = metricBundles.MetricBundle(metric,slicer,sqlWhere)
bg = metricBundles.MetricBundleGroup({0:bundle1},opsdb, outDir=outDir, resultsDb=resultsDb)
bg.runAll()
hp.gnomview(bundle1.metricValues, xsize=800,ysize=800, rot=(7,-7,0), unit='Count', min=1)
plt.savefig(outDir+'/fig'+tag+'.png')
def replot(bdict, opsdb, colmap, args):
resultsDb = db.ResultsDb(outDir=args.outDir)
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb)
group.readAll()
group.plotAll()
resultsDb.close()
def runSlew(opsdb, colmap, args):
resultsDb = db.ResultsDb(outDir=args.outDir)
bdict = batches.slewBasics(colmap, args.runName)
dbTable = None
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb,
dbTable=dbTable)
group.runAll()
group.plotAll()
if 'slewStatesTable' in colmap:
bdict = batches.slewAngles(colmap, args.runName)
dbTable = colmap['slewStatesTable']
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb,
dbTable=dbTable)
group.runAll()
group.plotAll()
if 'slewSpeedsTable' in colmap:
bdict = batches.slewSpeeds(colmap, args.runName)
dbTable = colmap['slewSpeedsTable']
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb,
dbTable=dbTable)
group.runAll()
group.plotAll()
if 'slewActivitiesTable' in colmap:
nslews = opsdb.fetchTotalSlewN()
bdict = batches.slewActivities(colmap, args.runName, totalSlewN=nslews)
dbTable = colmap['slewActivitiesTable']
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb,
dbTable=dbTable)
group.runAll()
group.plotAll()
resultsDb.close()
mafUtils.writeConfigs(opsdb, args.outDir)
def run_maf(dbFile, ra, dec):
"""Retrive min inter_night gap, and observation history with the input of database file name and arrays of RA and DEC.
Note: the observing cadence returned are not ordered by date!!
"""
# establish connection to sqllite database file.
opsimdb = db.OpsimDatabase(dbFile)
# While we're in transition between opsim v3 and v4, this may be helpful: print("{dbFile} is an opsim version {version} database".format(dbFile=dbFile, version=opsimdb.opsimVersion))
if opsimdb.opsimVersion == "V3":
# For v3 databases:
mjdcol = 'expMJD'
degrees = False
cols = ['filter', 'fiveSigmaDepth', mjdcol, 'expDate']
stackerList = []
else:
# For v4 and alternate scheduler databases.
mjdcol = 'observationStartMJD'
degrees = True
cols = ['filter', 'fiveSigmaDepth', mjdcol]
stackerList = [expDateStacker()]
# IntraNightGapsMetric returns the gap (in days) between observations within the same night custom reduceFunc to find min gaps
metric = metrics.cadenceMetrics.IntraNightGapsMetric(reduceFunc=np.amin,
mjdCol=mjdcol)
# PassMetric just pass all values
metric_pass = metrics.simpleMetrics.PassMetric(cols=cols)
# slicer for slicing pointing history
slicer = slicers.UserPointsSlicer(ra,
dec,
lonCol='fieldRA',
latCol='fieldDec',
latLonDeg=degrees)
# sql constrains, 3 for baseline2018a, 1 for rolling m2045
sql = ''
# bundles to combine metric, slicer and sql constrain together
bundle = metricBundles.MetricBundle(metric, slicer, sql)
date_bundle = metricBundles.MetricBundle(metric_pass,
slicer,
sql,
stackerList=stackerList)
# create metric bundle group and returns
bg = metricBundles.MetricBundleGroup(
{
'sep': bundle,
'cadence': date_bundle
},
opsimdb,
outDir=outDir,
resultsDb=resultsDb)
bg.runAll()
opsimdb.close()
return bg
def replotSlew(opsdb, colmap, args):
print('Only replots slew basics batch.')
bdict = batches.slewBasics(colmap, args.runName)
resultsDb = db.ResultsDb(outDir=args.outDir)
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb)
group.readAll()
group.plotAll()
resultsDb.close()
def testOpsim2dSlicer(self):
metric = metrics.AccumulateCountMetric(bins=[0.5, 1.5, 2.5])
slicer = slicers.OpsimFieldSlicer()
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.fieldData = self.fieldData
mbg.runCurrent('', simData=self.simData)
expected = np.array([[self.n1, self.n1], [-666., self.n2]])
assert (np.array_equal(mb.metricValues.data, expected))
def runGlance(dbfile, outDir='Glance', runName='runname', camera='LSST'):
conn = db.Database(dbfile, defaultTable='observations')
colmap = {
'ra': 'RA',
'dec': 'dec',
'mjd': 'mjd',
'exptime': 'exptime',
'visittime': 'exptime',
'alt': 'alt',
'az': 'az',
'filter': 'filter',
'fiveSigmaDepth': 'fivesigmadepth',
'night': 'night',
'slewtime': 'slewtime',
'seeingGeom': 'FWHM_geometric',
'rotSkyPos': 'rotSkyPos',
'raDecDeg': True,
'slewdist': None,
'note': 'note'
}
#del colmap['note']
gb = glanceBatch(colmap=colmap, slicer_camera=camera, runName=runName)
resultsDb = db.ResultsDb(outDir=outDir)
group = metricBundles.MetricBundleGroup(gb,
conn,
outDir=outDir,
resultsDb=resultsDb)
group.runAll()
group.plotAll()
fb = fOBatch(colmap=colmap, runName=runName)
group = metricBundles.MetricBundleGroup(fb,
conn,
outDir=outDir,
resultsDb=resultsDb)
group.runAll()
group.plotAll()
def testHealpix2dSlicer(self):
metric = metrics.AccumulateCountMetric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
good = np.where(mb.metricValues.mask[:, -1] == False)[0]
expected = np.array([[self.n1, self.n1], [-666., self.n2]])
assert (np.array_equal(mb.metricValues.data[good, :], expected))
def run(bdict, opsdb, colmap, args, save=True):
resultsDb = db.ResultsDb(outDir=args.outDir)
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb,
saveEarly=False)
if save:
group.runAll()
group.plotAll()
else:
group.runAll(clearMemory=True, plotNow=True)
resultsDb.close()
mafUtils.writeConfigs(opsdb, args.outDir)
def testAccumulateMetric(self):
metric = metrics.AccumulateMetric(col='fiveSigmaDepth', bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
# Clobber the stacker that gets auto-added
mb.stackerList = []
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
good = np.where(mb.metricValues.mask[:, -1] == False)[0]
expected = np.array([[self.n1*self.m5_1, self.n1*self.m5_1],
[-666., self.n2 * self.m5_2]])
assert(np.array_equal(mb.metricValues.data[good, :], expected))
def testRunRegularToo(self):
"""
Test that a binned slicer and a regular slicer can run together
"""
bundleList = []
metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
metric = metrics.Coaddm5Metric()
slicer = slicers.HealpixSlicer(nside=16)
bundleList.append(metricBundle.MetricBundle(metric, slicer, sql))
bd = metricBundle.makeBundlesDictFromList(bundleList)
mbg = metricBundle.MetricBundleGroup(bd, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
assert (np.array_equal(bundleList[0].metricValues[:, 1].compressed(),
bundleList[1].metricValues.compressed()))
def testOut(self):
"""
Check that the metric bundle can generate the expected output
"""
nside = 8
slicer = slicers.HealpixSlicer(nside=nside)
metric = metrics.MeanMetric(col='airmass')
sql = 'filter="r"'
stacker1 = stackers.RandomDitherFieldPerVisitStacker()
stacker2 = stackers.GalacticStacker()
map1 = maps.GalCoordsMap()
map2 = maps.StellarDensityMap()
metricB = metricBundles.MetricBundle(metric,
slicer,
sql,
stackerList=[stacker1, stacker2],
mapsList=[map1, map2])
database = os.path.join(getPackageDir('sims_data'), 'OpSimData',
'astro-lsst-01_2014.db')
opsdb = db.OpsimDatabaseV4(database=database)
resultsDb = db.ResultsDb(outDir=self.outDir)
bgroup = metricBundles.MetricBundleGroup({0: metricB},
opsdb,
outDir=self.outDir,
resultsDb=resultsDb)
bgroup.runAll()
bgroup.plotAll()
bgroup.writeAll()
opsdb.close()
outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*'))
outNpz = glob.glob(os.path.join(self.outDir, '*.npz'))
outPdf = glob.glob(os.path.join(self.outDir, '*.pdf'))
# By default, make 3 plots for healpix
assert (len(outThumbs) == 3)
assert (len(outPdf) == 3)
assert (len(outNpz) == 1)
def testAccumulateM5Metric(self):
metric = metrics.AccumulateM5Metric(bins=[0.5, 1.5, 2.5])
slicer = slicers.HealpixSlicer(nside=16)
sql = ''
mb = metricBundle.MetricBundle(metric, slicer, sql)
mbg = metricBundle.MetricBundleGroup({0: mb}, None, saveEarly=False)
mbg.setCurrent('')
mbg.runCurrent('', simData=self.simData)
good = np.where(mb.metricValues.mask[:, -1] == False)[0]
checkMetric = metrics.Coaddm5Metric()
tempSlice = np.zeros(self.n1, dtype=zip(['fiveSigmaDepth'], [float]))
tempSlice['fiveSigmaDepth'] += self.m5_1
val1 = checkMetric.run(tempSlice)
tempSlice = np.zeros(self.n2, dtype=zip(['fiveSigmaDepth'], [float]))
tempSlice['fiveSigmaDepth'] += self.m5_2
val2 = checkMetric.run(tempSlice)
expected = np.array([[val1, val1], [-666., val2]])
assert (np.array_equal(mb.metricValues.data[good, :], expected))
def get_cadence(ra, dec, b, snrLimit, nPtsLimit, filters, outDir, opsimdb,
resultsDb):
# The pass metric just passes data straight through.
metric = metrics.PassMetric(cols=['filter', 'fiveSigmaDepth', 'expMJD'])
slicer = slicers.UserPointsSlicer(ra,
dec,
lonCol='ditheredRA',
latCol='ditheredDec')
sql = ''
bundle = metricBundles.MetricBundle(metric, slicer, sql)
bg = metricBundles.MetricBundleGroup({0: bundle},
opsimdb,
outDir=outDir,
resultsDb=resultsDb)
bg.runAll()
bundle.metricValues.data[0]['filter']
print("Plotting...")
colors = {'u': 'cyan', 'g': 'g', 'r': 'y', 'i': 'r', 'z': 'm', 'y': 'k'}
dayZero = bundle.metricValues.data[0]['expMJD'].min()
times = []
depths = []
plt.clf()
for fname in filters:
good = np.where(bundle.metricValues.data[0]['filter'] == fname)
times.append(bundle.metricValues.data[0]['expMJD'][good] - dayZero)
depths.append(bundle.metricValues.data[0]['fiveSigmaDepth'][good])
plt.scatter(bundle.metricValues.data[0]['expMJD'][good] - dayZero,
bundle.metricValues.data[0]['fiveSigmaDepth'][good],
c=colors[fname],
label=fname)
plt.xlabel('Day')
plt.ylabel('5$\sigma$ depth')
plt.legend(scatterpoints=1, loc="upper left", bbox_to_anchor=(1, 1))
plt.savefig("l45b{0}_cadence.pdf".format(int(b)))
return times, depths
def testOut(self):
"""
Check that the metric bundle can generate the expected output
"""
nside = 8
slicer = slicers.HealpixSlicer(nside=nside)
metric = metrics.MeanMetric(col='airmass')
sql = 'filter="r"'
stacker1 = stackers.RandomDitherFieldPerVisitStacker()
stacker2 = stackers.GalacticStacker()
map1 = maps.GalCoordsMap()
map2 = maps.StellarDensityMap()
metricB = metricBundles.MetricBundle(metric,
slicer,
sql,
stackerList=[stacker1, stacker2])
filepath = os.path.join(os.getenv('SIMS_MAF_DIR'), 'tests/')
database = os.path.join(filepath, 'opsimblitz1_1133_sqlite.db')
opsdb = db.OpsimDatabase(database=database)
resultsDb = db.ResultsDb(outDir=self.outDir)
bgroup = metricBundles.MetricBundleGroup({0: metricB},
opsdb,
outDir=self.outDir,
resultsDb=resultsDb)
bgroup.runAll()
bgroup.plotAll()
bgroup.writeAll()
outThumbs = glob.glob(os.path.join(self.outDir, 'thumb*'))
outNpz = glob.glob(os.path.join(self.outDir, '*.npz'))
outPdf = glob.glob(os.path.join(self.outDir, '*.pdf'))
# By default, make 3 plots for healpix
assert (len(outThumbs) == 3)
assert (len(outPdf) == 3)
assert (len(outNpz) == 1)
'assuming WFD == area with 750 or more visits.')
parser.add_argument('dbfile',
type=str,
help='sqlite file of observations (full path).')
args = parser.parse_args()
# Pull out the visits. We'll reuse these for the footprint calculation below.
# Note that visits is a dataframe.
visits = get_visits(args.dbfile)
# Instead of always figuring out the WFD footprint from what we expected, let's define it based on
# what we got .. and define the "WFD" as the area with at least 750 visits per pointing.
runName = os.path.split(args.dbfile)[-1].replace('.db', '')
nside = 64
# Find the WFD footprint
m = CountMetric(col='observationStartMJD')
s = HealpixSlicer(nside)
simdata = visits.query('visitExposureTime > 11')
simdata = simdata.query('not note.str.startswith("DD")',
engine='python').to_records()
bundle = mb.MetricBundle(m, s, 'long notDD', runName=runName)
g = mb.MetricBundleGroup({f'{runName}': bundle}, None)
g.setCurrent('long notDD')
g.runCurrent('long notDD', simData=simdata)
wfd_footprint = bundle.metricValues.filled(0)
wfd_footprint = np.where(wfd_footprint > 750, 1, 0)
visits, propTags, propId = label_visits(visits, wfd_footprint)
update_database(args.dbfile, visits, propTags, propId)
def ResultadosNtotBolV2(FBS, mod):
# ==========================================================
# mod = "A"
# FBS = "1.5"
# modo = "A"
# filtros_considerados = ["u","g"] # f1,f2 tq f2 mas rojo que f1
# ==========================================================
#validacion(filtros_considerados)
#f1,f2 = filtros_considerados
# g_modA_LookupT_extension.pk
# lookup_table = "{}_mod{}_LookupT_extension.pkl".format(f2, modo) # debe estar en la carpeta de /lookuptables en /essentials
# f2 porque ese se ocupa , el f1 es para potencial lyman pbreak nomas
#filtros_modo = "{}_mod{}".format("".join(filtros_considerados),modo)
print("FBS usado:", FBS)
print("mod:", mod)
#####################################################################################
################################## 3 BUNDLES ########################################
#####################################################################################
metric = NtotMetricV2(mod, f1f2diff=2)
# ========================= WFD =================================
constraint1 = "note NOT LIKE '%DD%'"
wfd_standard = schedUtils.WFD_no_gp_healpixels(
64) #, dec_max=2.5, dec_min=-62.5)
slicer1 = slicers.HealpixSubsetSlicer(
64,
np.where(wfd_standard == 1)[0]
) #nside = 64, hpid = The subset of healpix id's to use to calculate the metric.
bundle1 = mb.MetricBundle(metric, slicer1, constraint1)
# ========================= DDF =================================
constraint2 = "note LIKE '%DD%'"
slicer2 = slicers.HealpixSlicer(nside=64)
bundle2 = mb.MetricBundle(metric, slicer2, constraint2)
print("==============================================")
print("constraint WFD:" + constraint1)
print("constraint DDF:" + constraint2)
#####################################################################################
################################# DIRECTORIOS #######################################
#####################################################################################
#Please enter your SciServer username between the single quotes below!
# your_username = '******'
# Check avaiable database directoies
show_fbs_dirs()
# if your_username == '': # do NOT put your username here, put it in the cell at the top of the notebook.
# raise Exception('Please provide your username! See the top of the notebook.')
dbDir = './lsst_cadence/FBS_{}/'.format(FBS)
outDir = '/data/agonzalez/output_FBS_{}/bolNtot_mod{}_FINAL/'.format(
FBS, mod)
if not os.path.exists(os.path.abspath(outDir)):
os.makedirs(os.path.abspath(outDir), exist_ok=True)
opSimDbs, resultDbs = connect_dbs(dbDir, outDir)
metricDataPath = '/data/agonzalez/output_FBS_{}/bolNtot_mod{}_FINAL/MetricData/'.format(
FBS, mod)
if not os.path.exists(os.path.abspath(metricDataPath)):
os.makedirs(os.path.abspath(metricDataPath), exist_ok=True)
print("===================================================")
print("dbDir :", dbDir)
print("outDir :", outDir)
print("metricDataPath :", metricDataPath)
print("===================================================")
#####################################################################################
################################# BUNDLE GROUP ######################################
#####################################################################################
dbRuns = show_opsims(dbDir)
print(dbRuns)
dbRuns = [x for x in dbRuns if "noddf" not in x]
#archivo70plus = open("jhu70plus{}.txt".format(FBS),"r")
#dbRuns = [x.rstrip() for x in list(archivo70plus)]
#archivo70plus.close()
for run in dbRuns: #[70:]:
bDict = {"WFD": bundle1, "DDF": bundle2}
bundle1.setRunName(run)
bundle2.setRunName(run)
bgroup = mb.MetricBundleGroup(bDict, opSimDbs[run], metricDataPath,
resultDbs[run])
bgroup.runAll()
args, extras = parser.parse_known_args()
# Build metric bundles.
(bundleDict, mergedHistDict, noSaveBundleDict) = makeBundleList(args.dbFile, nside=args.nside,
lonCol=args.lonCol, latCol=args.latCol,
benchmark=args.benchmark,
seeingCol=args.seeingCol)
# Set up / connect to resultsDb.
resultsDb = db.ResultsDb(outDir=args.outDir)
# Connect to opsimdb.
opsdb = db.OpsimDatabaseV4(args.dbFile)
if args.runNoSave:
group = metricBundles.MetricBundleGroup(noSaveBundleDict, opsdb, saveEarly=False,
outDir=args.outDir, resultsDb=resultsDb)
group.runAll(clearMemory=True, plotNow=True)
del group, noSaveBundleDict
# Set up metricBundleGroup.
group = metricBundles.MetricBundleGroup(bundleDict, opsdb,
outDir=args.outDir, resultsDb=resultsDb)
# Read or run to get metric values.
if args.plotOnly:
group.readAll()
else:
group.runAll()
# Make plots.
group.plotAll()
# Make merged plots.
for key in mergedHistDict:
def runSlices(opsimName, metadata, simdata, bins, args, verbose=False):
"""
Set up and run the movie slicer, and at each step,
setup the healpix slicer and run the metrics, creating and saving the plots.
"""
# Set up movie slicer
movieslicer = setupMovieSlicer(simdata,
binsize=args.movieStepsize,
cumulative=args.cumulative)
start_date = movieslicer[0]['slicePoint']['binLeft']
sliceformat = '%s0%dd' % ('%', int(np.log10(len(movieslicer))) + 1)
# Run through the movie slicer slicePoints:
for i, movieslice in enumerate(movieslicer):
t = time.time()
slicenumber = sliceformat % (i)
"""
# Set up plot label.
timeInterval = '%.2f to %.2f' %(movieslice['slicePoint']['binLeft']-start_date,
movieslice['slicePoint']['binRight']-start_date)
"""
# Or add simple view of time to plot label.
times_from_start = movieslice['slicePoint']['binRight'] - (
int(bins[0]) + 0.16 - 0.5)
# Opsim years are 365 days (not 365.25)
years = int(times_from_start / 365)
days = times_from_start - years * 365
plotlabel = 'Year %d Day %.4f' % (years, days)
metricList, plotDictList = setupMetrics(
opsimName,
metadata,
start_date,
movieslice['slicePoint']['binRight'],
cumulative=args.cumulative,
plotlabel=plotlabel,
verbose=verbose)
# Identify the subset of simdata in the movieslicer 'data slice'
simdatasubset = simdata[movieslice['idxs']]
# Set up healpix slicer on subset of simdata provided by movieslicer
hs = setupHealpixSlicer(args)
bundles = []
for metric, plotDict in zip(metricList, plotDictList):
bundles.append(
mB.MetricBundle(metric,
hs,
sqlconstraint=args.sqlConstraint,
metadata=metadata,
runName=opsimName,
plotDict=plotDict,
plotFuncs=[plots.HealpixSkyMap()]))
# Remove (default) stackers from bundles, because we've already run them above on the original data.
for mb in bundles:
mb.stackerList = []
bundledict = mB.makeBundlesDictFromList(bundles)
# Set up metricBundleGroup to handle metrics calculation + plotting
bg = mB.MetricBundleGroup(bundledict,
opsDb,
outDir=args.outDir,
resultsDb=None,
saveEarly=False)
# Calculate metric data values for simdatasubset (this also sets up indexing in the slicer)
bg.setCurrent(args.sqlConstraint)
bg.runCurrent(constraint=args.sqlConstraint, simData=simdatasubset)
# Plot data for this slice of the movie, adding slicenumber as a suffix for output plots
bg.plotAll(outfileSuffix=slicenumber,
closefigs=True,
dpi=72,
thumbnail=False,
figformat='png')
# Write the data -- uncomment if you want to do this.
# sm.writeAll(outfileSuffix=slicenumber)
if verbose:
dt, t = dtime(t)
print('Ran and plotted slice %s of movieslicer in %f s' %
(slicenumber, dt))
if args.db is None:
if os.path.isfile('trackingDb_sqlite.db'):
os.remove('trackingDb_sqlite.db')
db_files = glob.glob('*.db')
else:
db_files = [args.db]
run_names = [
os.path.basename(name).replace('.db', '') for name in db_files
]
for filename, name in zip(db_files, run_names):
if os.path.isdir(name + '_glance'):
shutil.rmtree(name + '_glance')
opsdb = db.OpsimDatabaseV4(filename)
colmap = batches.ColMapDict()
bdict = {}
bdict.update(batches.glanceBatch(colmap, name))
bdict.update(batches.fOBatch(colmap, name))
resultsDb = db.ResultsDb(outDir=name + '_glance')
group = mb.MetricBundleGroup(bdict,
opsdb,
outDir=name + '_glance',
resultsDb=resultsDb,
saveEarly=False)
group.runAll(clearMemory=True, plotNow=True)
resultsDb.close()
opsdb.close()
db.addRunToDatabase(name + '_glance', 'trackingDb_sqlite.db', None,
name, '', '', name + '.db')
" Default=fieldRA.")
parser.add_argument(
"--latCol",
type=str,
default='fieldDec',
help="Column to use for Dec values (can be a stacker dither column)." +
" Default=fieldDec.")
parser.add_argument('--night', type=int, default=1)
parser.set_defaults()
args, extras = parser.parse_known_args()
bundleDict = makeBundleList(args.dbFile,
nside=args.nside,
lonCol=args.lonCol,
latCol=args.latCol,
night=args.night)
# Set up / connect to resultsDb.
resultsDb = db.ResultsDb(outDir=args.outDir)
# Connect to opsimdb.
opsdb = db.OpsimDatabaseV3(args.dbFile)
# Set up metricBundleGroup.
group = metricBundles.MetricBundleGroup(bundleDict,
opsdb,
outDir=args.outDir,
resultsDb=resultsDb)
group.runAll()
group.plotAll()
def go(nside=64, rmag=21., SedTemplate='flat', DoRun=False, LFilters = [], \
LNightMax=[], nightMax=1e4, \
CustomPlotLimits=True, \
RunOne=False, MaxRuns=1e3, \
SpatialClip=95., \
seeingCol='FWHMeff', \
sCmap='cubehelix_r', \
checkCorrKind=False, \
wfdPlane=True, \
useGRIZ=False):
# Go to the directory where the sqlite databases are held...
# cd /Users/clarkson/Data/LSST/OpSimRuns/opsim20160411
# WIC 2015-12-29 - set up for a master-run with all cases, this time with plotting limits
# Break the specifications across lines to make subdivision easier
# Subsets by time first, then by filter, finally the whole shebang
# 2016-04-23 - replaced enigma_1189 --> minion_1016
# 2016-04-23 - replaced ops2_1092 --> minion_1020
# (Yes the inversion of the first two is deliberate.)
runNames = ['minion_1016', 'minion_1020', 'minion_1020', 'minion_1016', \
'minion_1020', 'minion_1016', 'minion_1020', 'minion_1016', \
'minion_1020', 'minion_1016']
LFilters = ['', '', '', '', \
'u', 'u', 'y', 'y', \
'', '']
LNightMax = [365, 365, 730, 730, \
1e4, 1e4, 1e4, 1e4, \
1e4, 1e4]
# WIC try again, this time on the new astro_lsst_01_1004 only
if wfdPlane:
LFilters = ['', '', '', 'u', 'y']
LNightMax = [365, 730, 1e4, 1e4, 1e4]
runNames = ['astro_lsst_01_1004' for i in range (len(LFilters)) ]
# WIC 2016-05-01 check correlation
if checkCorrKind:
LFilters = ['', '']
LNightMax = [365, 365]
runNames = ['minion_1016', 'minion_1016']
# Type of correlation used for HA Degen
# checkCorrKind = True
useSpearmanR = [False, True]
if useGRIZ:
runNames=['minion_1016','astro_lsst_01_1004', 'minion_1020']
LFilters = ['griz' for iRun in range(len(runNames)) ]
#LNightMax = [1e4 for iRun in range(len(runNames)) ]
#LNightMax = [730 for iRun in range(len(runNames)) ]
LNightMax = [365 for iRun in range(len(runNames)) ]
# List of upper limits to parallax and proper motion error. For parallax, 3.0 mas is probably good
LUpperParallax = []
LUpperPropmotion = []
if CustomPlotLimits:
LUpperParallax = [10, 10, 10, 10, \
10, 10, 40, 40, \
3.0, 3.0 ]
# For proper motion, it's a little tricky to say because the
# regular case is so pathological for the field. Try the following:
LUpperPropmotion = [40, 40, 5, 20, \
3.5, 20, 3.5, 20, \
0.5, 5]
if len(runNames) < 2:
LUpperPropmotion = [100 for i in range(len(runNames))]
print "runAstrom.go INFO - will run the following:"
for iSho in range(len(runNames)):
sFilThis = ''
# print iSho, len(LFilters)
if iSho <= len(LFilters):
sFilThis = sqlFromFilterString(LFilters[iSho])
print "%i: %-12s, %1s, %i, sqlFilter -- %s" % (iSho, runNames[iSho], LFilters[iSho], LNightMax[iSho], sFilThis)
print "==========================="
print "mag max = %.2f" % (rmag)
print "---------------------------"
# print runNames
# if not DoRun:
# print "Set DoRun=True to actually run this."
# print len(LFilters), len(runNames), len(LFilters) == len(runNames)
# return
#'kraken_1038', 'kraken_1034', 'ops2_1098']
# nside = 64
slicer = slicers.HealpixSlicer(nside=nside)
# Make it so we don't bother with the silly power spectra
plotFuncs = [plots.HealpixSkyMap(), plots.HealpixHistogram()]
# WIC - back up the plotting arguments with a default value
plotFuncsPristine = copy.deepcopy(plotFuncs)
# WIC - the only way this will make sense to me is if I make a
# dictionary of plot arguments. Let's try it...
DPlotArgs = {}
for plotArg in ['parallax', 'propmotion', 'coverage', 'HAdegen']:
DPlotArgs[plotArg] = copy.deepcopy(plotFuncs)
if CustomPlotLimits:
# Use the same color map for all the metrics
for plotMetric in DPlotArgs.keys():
DPlotArgs[plotMetric][0].defaultPlotDict['cmap'] = sCmap
# Apply spatial clipping for all but the HADegen, for which we
# have other limits...
for plotMetric in ['parallax', 'propmotion', 'coverage']:
DPlotArgs[plotMetric][0].defaultPlotDict['percentileClip'] = SpatialClip
# Some limits common to spatial maps and histograms
for iPl in range(0,2):
DPlotArgs['propmotion'][iPl].defaultPlotDict['logScale'] = True
# NOT a loop because we might want to separate out the behavior
# Standardized range for the histograms for new parallax metrics
DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0.
DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMin'] = -1.
DPlotArgs['HAdegen'][1].defaultPlotDict['xMax'] = 1.
# Standardize the sky map for the HAdegen as well.
DPlotArgs['coverage'][1].defaultPlotDict['xMin'] = 0.
DPlotArgs['coverage'][1].defaultPlotDict['xMax'] = 1.
DPlotArgs['HAdegen'][0].defaultPlotDict['xMin'] = -1.
DPlotArgs['HAdegen'][0].defaultPlotDict['xMax'] = 1.
# Standardize at least the lower bound of the histogram in
# both the proper motion and parallax errors. Upper limit we
# can customize with a loop.
DPlotArgs['propmotion'][1].defaultPlotDict['xMin'] = 1e-2 # should not be zero if log scale!!
DPlotArgs['parallax'][1].defaultPlotDict['xMin'] = 0.
# WIC - try changing the plot dictionary
if not DoRun:
plotFuncs[0].defaultPlotDict['logScale'] = True
print DPlotArgs['propmotion'][0].defaultPlotDict
print DPlotArgs['propmotion'][1].defaultPlotDict
return
# The old runs have the seeing in finSeeing
#seeingCol = 'finSeeing'
### UPDATE THE SEEING COLUMN
#seeingCol = 'FWHMeff' ## Moved up to a command-line argument
# Use all the observations. Can change if you want a different
# time span
# sqlconstraint = ''
# list of sqlconstraints now used, which gets handled within the loop.
# run some summary stats on everything
summaryMetrics = [metrics.MedianMetric()]
tStart = time.time()
# Running one, or the whole lot?
RunMax = len(runNames)
# allow user to set a different number (say, 2)
if MaxRuns < RunMax and MaxRuns > 0:
RunMax = int(MaxRuns)
# the following keyword overrides
if RunOne:
RunMax = 1
print "Starting runs. RunMax = %i" % (RunMax)
for iRun in range(RunMax):
run = runNames[iRun][:]
# for run in runNames:
# Open the OpSim database
timeStartIteration = time.time()
# Some syntax added to test for existence of the database
dbFil = run+'_sqlite.db'
if not os.access(dbFil, os.R_OK):
print "runAstrom.go FATAL - cannot acces db file %s" % (dbFil)
print "runAstrom.go FATAL - skipping run %s" % (run)
continue
else:
deltaT = time.time()-tStart
print "runAstrom.go INFO - ##################################"
print "runAstrom.go INFO - starting run %s with nside=%i after %.2f minutes" \
% (run, nside, deltaT/60.)
opsdb = db.OpsimDatabase(run+'_sqlite.db')
# Set SQL constraint appropriate for each filter in the
# list. If we supplied a list of filters, use it for
sqlconstraint = ''
ThisFilter = 'ugrizy'
if len(LFilters) == len(runNames):
# Only change the filter if one was actually supplied!
if len(LFilters[iRun]) > 0:
ThisFilter = LFilters[iRun]
sqlconstraint = sqlFromFilterString(ThisFilter)
### sqlconstraint = 'filter = "%s"' % (ThisFilter)
# If nightmax was supplied, use it
ThisNightMax = int(nightMax) # copy not view
if len(LNightMax) == len(runNames):
# Only update nightmax if one was given
try:
ThisNightMax = int(LNightMax[iRun]) # This might be redundant with the fmt statement below.
if len(sqlconstraint) < 1:
sqlconstraint = 'night < %i' % (ThisNightMax)
else:
sqlconstraint = '%s and night < %i' % (sqlconstraint, ThisNightMax)
except:
print "runAstrom.go WARN - run %i problem with NightMax" % (iRun)
dumdum = 1.
# Set where the output should go - include the filter!!
sMag = '%.1f' % (rmag)
sMag = sMag.replace(".","p")
outDir = './metricEvals/%s_nside%i_%s_n%i_r%s' % (run, nside, ThisFilter, ThisNightMax, sMag)
# Ensure we'll be able to find this later on...
if CustomPlotLimits:
outDir = '%s_lims' % (outDir)
# if we are testing the kind of correlation used, include that
# in the output here.
if checkCorrKind:
if useSpearmanR[iRun]:
sCorr = 'spearmanR'
else:
sCorr = 'pearsonR'
outDir = '%s_%s' % (outDir, sCorr)
# From this point onwards, stuff actually gets run. This is
# the place to output what will actually happen next.
print "runAstrom.go INFO - about to run:"
print "runAstrom.go INFO - sqlconstraint: %s ; run name %s ; nside %i" % (sqlconstraint, run, nside)
print "runAstrom.go INFO - output directory will be %s" % (outDir)
if not DoRun:
continue
# ensure the output directory actually exists...
if not os.access(outDir, os.R_OK):
print "runAstrom.go INFO - creating output directory %s" % (outDir)
os.makedirs(outDir)
resultsDb = db.ResultsDb(outDir=outDir)
bundleList = []
# WIC - to make this at least somewhat uniform, build the plot
# functions including arguments out of our copies above.
plotFuncsPropmotion = copy.deepcopy(DPlotArgs['propmotion'])
plotFuncsParallax = copy.deepcopy(DPlotArgs['parallax'])
plotFuncsCoverage = copy.deepcopy(DPlotArgs['coverage'])
plotFuncsHAdegen = copy.deepcopy(DPlotArgs['HAdegen'])
# if using custom plot limits, will want to include the limits
# for proper motion and parallax too... programming a bit defensively
# here, including an extra check (rather than just the length of the lists
# above).
if CustomPlotLimits:
if len(LUpperParallax) == len(runNames):
plotFuncsParallax[1].defaultPlotDict['xMax'] = float(LUpperParallax[iRun])
if len(LUpperPropmotion) == len(runNames):
plotFuncsPropmotion[1].defaultPlotDict['xMax'] = float(LUpperPropmotion[iRun])
# Configure the metrics
metric = metrics.ParallaxMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs = plotFuncsParallax, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric=metrics.ProperMotionMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsPropmotion, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
metric = calibrationMetrics.ParallaxCoverageMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsCoverage, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Now for the HA Degen metric. If testing the type of
# correlation, call the metric differently here. Since the
# argument to actually do this is only part of my github fork
# at the moment, we use a different call. Running with default
# arguments (checkCorrKind=False) should then work without
# difficulty.
metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate)
if checkCorrKind:
metric = calibrationMetrics.ParallaxHADegenMetric(rmag=rmag, seeingCol=seeingCol, SedTemplate=SedTemplate, useSpearmanR=useSpearmanR[iRun])
print "TESTING CORRELATION KIND -- useSpearmanR", useSpearmanR[iRun]
bundle = metricBundles.MetricBundle(metric, slicer, sqlconstraint, runName=run,
# plotFuncs=plotFuncs, \
plotFuncs=plotFuncsHAdegen, \
summaryMetrics=summaryMetrics)
bundleList.append(bundle)
# Run everything and make plots
bundleDict = metricBundles.makeBundlesDictFromList(bundleList)
bgroup = metricBundles.MetricBundleGroup(bundleDict, opsdb, outDir=outDir, resultsDb=resultsDb)
# try:
bgroup.runAll()
print "runAstrom.go INFO - bundles took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
# except KeyboardInterrupt:
# print "runAstrom.go FATAL - keyboard interrupt detected. Halting."
# return
bgroup.plotAll()
print "runAstrom.go INFO - bundles + plotting took %.2f minutes" \
% ((time.time() - timeStartIteration) / 60.)
print "Finished entire set. %i runs took %.2f minutes." % (iRun + 1, (time.time()-tStart)/60.)
