def testNsecsDefault(self):
ts = DateTime(1192755506000000000)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
self.assertTrue(ts.isValid())
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
self.assertTrue(ts.isValid())
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
self.assertTrue(ts.isValid())
def testNsecsDefault(self):
ts = DateTime(1192755506000000000)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
self.assertTrue(ts.isValid())
def testPopMjdDate(self):
for timesys in (None, "UTC", "TAI"):
dataDict = {
"DATE1": 51943.1705801,
"DATE2": 44302.0433218,
"BADMJDDATE": "2001-02-03T04:05:06.123456789",
}
if timesys is not None:
dataDict["TIMESYS"] = timesys
length = len(dataDict)
for key, mjdDate in dataDict.items():
if not key.startswith("DATE"):
continue
lsstSys = dict(
UTC=DateTime.UTC,
TAI=DateTime.TAI,
).get(timesys, DateTime.UTC)
desDate = DateTime(mjdDate, DateTime.MJD, lsstSys)
md = getMetadata(dataDict)
date = self.makeRawVisitInfo.popMjdDate(md, key, timesys=timesys)
self.assertEqual(len(md.names()), length)
self.assertAlmostEqual(date.get(), desDate.get())
badDate = self.makeRawVisitInfo.popMjdDate(md, "BADMJDDATE")
self.assertEqual(badDate, DateTime())
missingDate = self.makeRawVisitInfo.popMjdDate(md, "BADKEY")
self.assertEqual(missingDate, DateTime())
def testNsecsTT(self):
ts = DateTime(long(1192755538184000000), DateTime.TT)
self.assertEqual(ts.nsecs(DateTime.UTC), long(1192755473000000000))
self.assertEqual(ts.nsecs(DateTime.TAI), long(1192755506000000000))
self.assertEqual(ts.nsecs(), long(1192755506000000000))
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC),
54392.040196759262)
self.assertTrue(ts.isValid())
def testInvalid(self):
ts = DateTime()
self.assertFalse(ts.isValid())
for scale in self.timeScales:
self.assertEqual(ts.nsecs(scale), DateTime.invalid_nsecs)
for system in self.dateSystems:
with self.assertRaises(pexExcept.RuntimeError):
ts.get(system, scale)
with self.assertRaises(pexExcept.RuntimeError):
ts.gmtime(scale)
with self.assertRaises(pexExcept.RuntimeError):
ts.timespec(scale)
with self.assertRaises(pexExcept.RuntimeError):
ts.timeval(scale)
with self.assertRaises(pexExcept.RuntimeError):
ts.toString(scale)
self.assertEqual(repr(ts), "DateTime()")
def testInvalid(self):
ts = DateTime()
self.assertFalse(ts.isValid())
for scale in self.timeScales:
self.assertEqual(ts.nsecs(scale), DateTime.invalid_nsecs)
for system in self.dateSystems:
with self.assertRaises(RuntimeError):
ts.get(system, scale)
with self.assertRaises(RuntimeError):
ts.gmtime(scale)
with self.assertRaises(RuntimeError):
ts.timespec(scale)
with self.assertRaises(RuntimeError):
ts.timeval(scale)
with self.assertRaises(RuntimeError):
ts.toString(scale)
self.assertEqual(repr(ts), "DateTime()")
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000L)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000L)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
# Following interface is deprecated
self.assertAlmostEqual(ts.mjd(DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.mjd(DateTime.TAI), 45205.125 + 21.0/86400.0)
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def tearDown(self):
del self.butler
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=85471048, snap=0, raft='0,3', sensor='0,1', channel='1,0',
immediate=True)
self.assertEqual(raw.getWidth(), 513)
self.assertEqual(raw.getHeight(), 2001)
self.assertEqual(raw.getFilter().getFilterProperty().getName(), "y")
self.assertEqual(raw.getDetector().getName(), "R:0,3 S:0,1")
origin = raw.getWcs().getSkyOrigin()
self.assertAlmostEqual(
origin.getLongitude().asDegrees(), 0.0058520, 6)
self.assertAlmostEqual(
origin.getLatitude().asDegrees(), -2.3052624, 6)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
# Explicit test for NaN here, because phosim output may not have consistent alt/az/ra/dec/time
self.assertTrue(math.isnan(visitInfo.getEra()))
self.assertAlmostEqual(visitInfo.getExposureTime(), self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def tearDown(self):
del self.butler
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=85471048, snap=0, raft='0,3', sensor='0,1', channel='1,0',
immediate=True)
self.assertEqual(raw.getWidth(), 513)
self.assertEqual(raw.getHeight(), 2001)
self.assertEqual(raw.getFilter().getFilterProperty().getName(), "y")
self.assertEqual(raw.getDetector().getName(), "R:0,3 S:0,1")
origin = raw.getWcs().getSkyOrigin()
self.assertAlmostEqual(
origin.getLongitude().asDegrees(), 0.0058520, 6)
self.assertAlmostEqual(
origin.getLatitude().asDegrees(), -2.3052624, 6)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
# Explicit test for NaN here, because phosim output may not have consistent alt/az/ra/dec/time
self.assertTrue(math.isnan(visitInfo.getEra()))
self.assertAlmostEqual(visitInfo.getExposureTime(), self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def testNsecs(self):
ts = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
ts2 = ts
self.assertEqual(ts, ts2)
ts2 = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts, ts2)
ts2 = DateTime(1234567890000000000, DateTime.UTC)
self.assertNotEqual(ts, ts2)
def _makeDiaSources(self, sources: numpy.ndarray, indices: numpy.ndarray,
dt: DateTime, visit_id: int) -> pandas.DataFrame:
"""Generate catalog of DiaSources to store in a database
Parameters
----------
sources : `numpy.ndarray`
(x, y, z) coordinates of sources, array dimension is (N, 3)
indices : `numpy.array`
array of indices of sources, 1-dim ndarray, transient sources
have negative indices
dt : `DateTime`
Visit time.
visit_id : `int`
ID of the visit
Returns
-------
catalog : `pandas.DataFrame`
Catalog of DIASources.
"""
def polar(row: Any) -> pandas.Series:
v3d = Vector3d(row.x, row.y, row.z)
sp = SpherePoint(v3d)
return pandas.Series(
[sp.getRa().asDegrees(),
sp.getDec().asDegrees()],
index=["ra", "decl"])
midPointTai = dt.get(system=DateTime.MJD)
catalog = pandas.DataFrame(sources, columns=["x", "y", "z"])
catalog["diaObjectId"] = indices
catalog = cast(pandas.DataFrame,
catalog[catalog["diaObjectId"] != _OUTSIDER])
cat_polar = cast(pandas.DataFrame,
catalog.apply(polar, axis=1, result_type='expand'))
cat_polar["diaObjectId"] = catalog["diaObjectId"]
catalog = cat_polar
catalog["ccdVisitId"] = visit_id
catalog["parentDiaSourceId"] = 0
catalog["psFlux"] = 1.
catalog["psFluxErr"] = 0.01
catalog["midPointTai"] = midPointTai
catalog["flags"] = 0
nrows = catalog.shape[0]
catalog["diaSourceId"] = range(self.lastSourceId + 1,
self.lastSourceId + 1 + nrows)
self.lastSourceId += nrows
return catalog
def testNsecs(self):
ts = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
ts2 = ts
self.assertEqual(ts, ts2)
ts2 = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts, ts2)
ts2 = DateTime(1234567890000000000, DateTime.UTC)
self.assertNotEqual(ts, ts2)
def testGetDR7(self):
obsSdssDir = lsst.utils.getPackageDir('obs_sdss')
butler = dafPersist.Butler(
root=os.path.join(obsSdssDir, "tests", "data", "dr7", "runs"))
sub = butler.subset("fpC", run=5754, camcol=3, field=280, filter="r")
self.assertEqual(len(sub), 1)
for ref in sub:
im = ref.get("fpC")
w, h = im.getWidth(), im.getHeight()
self.assertEqual(im.__class__, lsst.afw.image.ImageU)
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
im_md = ref.get("fpC_md")
self.assertEqual(im_md.get("RUN"), 5754)
self.assertEqual(im_md.get("FRAME"), 280)
self.assertEqual(im_md.get("STRIPE"), 82)
msk = ref.get("fpM")
w, h = msk.getWidth(), msk.getHeight()
self.assertEqual(msk.__class__,
lsst.afw.image.Mask[lsst.afw.image.MaskPixel])
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
psf = ref.get("psField")
k = psf.getKernel()
w, h = k.getWidth(), k.getHeight()
self.assertEqual(psf.__class__, lsst.meas.algorithms.PcaPsf)
self.assertEqual(w, 31)
self.assertEqual(h, 31)
wcs = ref.get("asTrans")
self.assertIsInstance(wcs, SkyWcs)
self.assertFalse(wcs.isFlipped)
# comparison is to results from lsst.afw.image.TanWcs class
self.assertCoordsAlmostEqual(
wcs.pixelToSky(700, 1000),
IcrsCoord(343.6507738304687 * degrees,
-0.3509870420713227 * degrees))
tsField = ref.get("tsField")
self.assertAlmostEqual(tsField.gain, 4.72, 2)
self.assertAlmostEqual(tsField.airmass, 1.2815132857671601)
self.assertAlmostEqual(tsField.exptime, 53.907456)
predDateStart = DateTime(53664.226070589997, DateTime.MJD,
DateTime.TAI)
predDateAvg = DateTime(
predDateStart.nsecs(DateTime.TAI) +
int(0.5e9 * tsField.exptime), DateTime.TAI)
self.assertAlmostEqual(tsField.dateAvg.get(), predDateAvg.get())
def testGetDR7(self):
obsSdssDir = lsst.utils.getPackageDir('obs_sdss')
butler = dafPersist.Butler(
root=os.path.join(obsSdssDir, "tests", "data", "dr7", "runs"))
sub = butler.subset("fpC", run=5754, camcol=3, field=280, filter="r")
self.assertEqual(len(sub), 1)
for ref in sub:
im = ref.get("fpC")
w, h = im.getWidth(), im.getHeight()
self.assertEqual(im.__class__, lsst.afw.image.ExposureU)
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
im_md = ref.get("fpC_md")
self.assertEqual(im_md.getScalar("RUN"), 5754)
self.assertEqual(im_md.getScalar("FRAME"), 280)
self.assertEqual(im_md.getScalar("STRIPE"), 82)
msk = ref.get("fpM")
w, h = msk.getWidth(), msk.getHeight()
self.assertEqual(msk.__class__, lsst.afw.image.Mask[lsst.afw.image.MaskPixel])
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
psf = ref.get("psField")
k = psf.getKernel()
w, h = k.getWidth(), k.getHeight()
self.assertEqual(psf.__class__, lsst.meas.algorithms.PcaPsf)
self.assertEqual(w, 31)
self.assertEqual(h, 31)
wcs = ref.get("asTrans")
self.assertIsInstance(wcs, SkyWcs)
self.assertFalse(wcs.isFlipped)
# comparison is to results from lsst.afw.image.TanWcs class
self.assertSpherePointsAlmostEqual(wcs.pixelToSky(700, 1000),
SpherePoint(343.6507738304687, -0.3509870420713227, degrees))
tsField = ref.get("tsField")
self.assertAlmostEqual(tsField.gain, 4.72, 2)
self.assertAlmostEqual(tsField.airmass, 1.2815132857671601)
self.assertAlmostEqual(tsField.exptime, 53.907456)
predDateStart = DateTime(53664.226070589997, DateTime.MJD, DateTime.TAI)
predDateAvg = DateTime(predDateStart.nsecs(DateTime.TAI) + int(0.5e9*tsField.exptime),
DateTime.TAI)
self.assertAlmostEqual(tsField.dateAvg.get(), predDateAvg.get())
def testGetDR7(self):
obsSdssDir = lsst.utils.getPackageDir('obs_sdss')
butler = dafPersist.Butler(
root=os.path.join(obsSdssDir, "tests", "data", "dr7", "runs"))
sub = butler.subset("fpC", run=5754, camcol=3, field=280, filter="r")
self.assertEqual(len(sub), 1)
for ref in sub:
im = ref.get("fpC")
w, h = im.getWidth(), im.getHeight()
self.assertEqual(im.__class__, lsst.afw.image.ImageU)
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
im_md = ref.get("fpC_md")
self.assertEqual(im_md.get("RUN"), 5754)
self.assertEqual(im_md.get("FRAME"), 280)
self.assertEqual(im_md.get("STRIPE"), 82)
msk = ref.get("fpM")
w, h = msk.getWidth(), msk.getHeight()
self.assertEqual(msk.__class__, lsst.afw.image.Mask[lsst.afw.image.MaskPixel])
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
psf = ref.get("psField")
k = psf.getKernel()
w, h = k.getWidth(), k.getHeight()
self.assertEqual(psf.__class__, lsst.meas.algorithms.PcaPsf)
self.assertEqual(w, 31)
self.assertEqual(h, 31)
wcs = ref.get("asTrans")
self.assertEqual(wcs.__class__, lsst.afw.image.TanWcs)
self.assertFalse(wcs.isFlipped())
self.assertAlmostEqual(wcs.getFitsMetadata().get("CRPIX1"), 1.0, 5)
self.assertAlmostEqual(wcs.getFitsMetadata().get("CRPIX2"), 1.0, 5)
tsField = ref.get("tsField")
self.assertAlmostEqual(tsField.gain, 4.72, 2)
self.assertAlmostEqual(tsField.airmass, 1.2815132857671601)
self.assertAlmostEqual(tsField.exptime, 53.907456)
predDateStart = DateTime(53664.226070589997, DateTime.MJD, DateTime.TAI)
predDateAvg = DateTime(predDateStart.nsecs(DateTime.TAI) + int(0.5e9*tsField.exptime),
DateTime.TAI)
self.assertAlmostEqual(tsField.dateAvg.get(), predDateAvg.get())
class GetDataTestCase(lsst.utils.tests.TestCase):
"""Demonstrate retrieval of data from a repository."""
@unittest.skipIf(testDataDirectory is None, "%s is not available" % testDataPackage)
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
self.dateAvg = DateTime(56598.261654253474, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = -0.8
self.weath_airPressure = 62170.
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 2.3659577248977826*radians
def tearDown(self):
del self.butler
rmtree(self.repoPath)
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
ccd = raw.getDetector()
self.assertEqual(raw.getDimensions(), afwGeom.Extent2I(*self.rawSize))
self.assertEqual(raw.getFilter().getFilterProperty().getName(), self.filter)
self.assertEqual(ccd.getId(), self.ccdNum)
self.assertEqual(ccd.getBBox().getDimensions(), afwGeom.Extent2I(*self.ccdSize))
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
self.assertAnglesAlmostEqual(visitInfo.getEra(), self.era)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
self.assertEqual(visitInfo.getExposureTime(), self.exptime)
self.assertEqual(visitInfo.getDarkTime(), self.darktime)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure, places=4)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def testPupil(self):
"""Test retrieval of pupil (without checking value)"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
visitInfo = raw.getInfo().getVisitInfo()
camera = self.butler.get("camera", visit=self.visit, ccd=self.ccdNum)
size = 16.4
npix = 1024
pupilFactory = camera.getPupilFactory(visitInfo, size, npix)
self.assertIsInstance(pupilFactory, lsst.obs.hsc.hscPupil.HscPupilFactory)
pupil = pupilFactory.getPupil(afwGeom.Point2D(0.0, 0.0))
self.assertEqual(pupil.size, size)
self.assertEqual(pupil.scale, size/npix)
self.assertEqual(pupil.illuminated.shape, (npix, npix))
def testRawMetadata(self):
"""Test retrieval of raw image metadata"""
md = self.butler.get("raw_md", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(md.getScalar("DET-ID"), self.ccdNum)
def testBias(self):
"""Test retrieval of bias frame"""
bias = self.butler.get("bias", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(bias.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(bias.getDetector().getId(), self.ccdNum)
def testDark(self):
"""Test retrieval of dark frame"""
dark = self.butler.get("dark", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(dark.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(dark.getDetector().getId(), self.ccdNum)
self.assertEqual(dark.getInfo().getVisitInfo().getExposureTime(), 1.0)
def testFlat(self):
"""Test retrieval of flat"""
flat = self.butler.get("flat", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(flat.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(flat.getDetector().getId(), self.ccdNum)
def testLinearizer(self):
self.assertTrue(self.butler.get('linearizer', ccdnum=1))
def testNsecsTAI(self):
ts = DateTime(1192755506000000000L, DateTime.TAI)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000L)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000L)
self.assertEqual(ts.nsecs(), 1192755506000000000L)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
datadir = self.getTestDataDir()
self.repoPath = os.path.join(datadir, "DATA")
self.calibPath = os.path.join(datadir, "CALIB")
self.butler = dafPersist.Butler(root=self.repoPath,
calibRoot=self.calibPath)
self.size = (2112, 4644)
self.dataId = {'visit': 1038843}
self.filter = "i2"
self.exposureTime = 615.037
self.darkTime = 615.0
self.dateAvg = DateTime(54771.610881712964, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = SpherePoint(135.40941055, -2.39999432, degrees)
self.boresightAzAlt = SpherePoint(122.34, 52.02, degrees)
self.boresightAirmass = 1.269
self.boresightRotAngle = 0 * degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.468876 * degrees
self.obs_latitude = 19.825252 * degrees
self.obs_elevation = 4215
self.weath_airTemperature = 0.90
self.weath_airPressure = 617.65 * 100 # 100 Pascal/millibar
self.weath_humidity = 39.77
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 4.55388 * radians
def tearDown(self):
del self.butler
def assertExposure(self, exp, ccd, checkFilter=True):
print("dataId: ", self.dataId)
print("ccd: ", ccd)
print("width: ", exp.getWidth())
print("height: ", exp.getHeight())
print("detector name: ", exp.getDetector().getName())
print("filter name: ", exp.getFilter().getFilterProperty().getName())
self.assertEqual(exp.getWidth(), self.size[0])
self.assertEqual(exp.getHeight(), self.size[1])
self.assertEqual(exp.getDetector().getName(), "ccd%02d" % ccd)
if checkFilter:
self.assertEqual(exp.getFilter().getFilterProperty().getName(),
self.filter)
if display and ccd % 18 == 0:
global frame
frame += 1
ccd = exp.getDetector()
for amp in ccd:
amp = amp
print(ccd.getId(), amp.getId(),
amp.getDataSec().toString(),
amp.getBiasSec().toString(),
amp.getElectronicParams().getGain())
cameraGeomUtils.showCcd(ccd, ccdImage=exp, frame=frame)
def getTestDataDir(self):
try:
datadir = getPackageDir("testdata_cfht")
except pexExcept.NotFoundError as e:
warnings.warn(e.args[0])
raise unittest.SkipTest(
"Skipping test as testdata_cfht is not setup")
return datadir
def testRaw(self):
"""Test retrieval of raw image"""
if display:
global frame
frame += 1
cameraGeomUtils.showCamera(self.butler.mapper.camera, frame=frame)
for ccd in range(36):
raw = self.butler.get("raw", self.dataId, ccd=ccd, immediate=True)
self.assertExposure(raw, ccd)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(),
self.dateAvg.get())
self.assertAnglesAlmostEqual(visitInfo.getEra(),
self.era,
maxDiff=0.1 * degrees)
self.assertAlmostEqual(visitInfo.getExposureTime(),
self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(),
self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(),
self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(),
self.boresightAirmass)
self.assertTrue(visitInfo.getBoresightRotAngle(),
self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(),
self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(),
self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(),
self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(),
self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(),
self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def getDetrend(self, detrend):
"""Test retrieval of detrend image"""
for ccd in range(36):
flat = self.butler.get(detrend,
self.dataId,
ccd=ccd,
immediate=True)
self.assertExposure(flat, ccd, checkFilter=False)
def testFlat(self):
self.getDetrend("flat")
def testBias(self):
self.getDetrend("bias")
def testFringe(self):
self.getDetrend("fringe")
def testPackageName(self):
name = dafPersist.Butler.getMapperClass(root=self.repoPath).packageName
self.assertEqual(name, "obs_cfht")
class GetDataTestCase(lsst.utils.tests.TestCase):
"""Demonstrate retrieval of data from a repository."""
@unittest.skipIf(testDataDirectory is None, "%s is not available" % testDataPackage)
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
dateBeg = DateTime(56598.26106374757, DateTime.MJD, DateTime.UTC)
dateAvgNSec = dateBeg.nsecs() + int(0.5e9*self.exptime)
self.dateAvg = DateTime(dateAvgNSec, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = MakeRawVisitInfo.centigradeFromKelvin(272.35)
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(621.7)
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
lst = 340.189608333*degrees
self.era = lst - self.obs_longitude
def tearDown(self):
del self.butler
rmtree(self.repoPath)
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
ccd = raw.getDetector()
self.assertEqual(raw.getDimensions(), afwGeom.Extent2I(*self.rawSize))
self.assertEqual(raw.getFilter().getFilterProperty().getName(), self.filter)
self.assertEqual(ccd.getId(), self.ccdNum)
self.assertEqual(ccd.getBBox().getDimensions(), afwGeom.Extent2I(*self.ccdSize))
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
self.assertAnglesAlmostEqual(visitInfo.getEra(), self.era)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
self.assertEqual(visitInfo.getExposureTime(), self.exptime)
self.assertEqual(visitInfo.getDarkTime(), self.darktime)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def testPupil(self):
"""Test retrieval of pupil (without checking value)"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
visitInfo = raw.getInfo().getVisitInfo()
camera = self.butler.get("camera", visit=self.visit, ccd=self.ccdNum)
size = 16.4
npix = 1024
pupilFactory = camera.getPupilFactory(visitInfo, size, npix)
self.assertIsInstance(pupilFactory, lsst.obs.hsc.hscPupil.HscPupilFactory)
pupil = pupilFactory.getPupil(afwGeom.Point2D(0.0, 0.0))
self.assertEqual(pupil.size, size)
self.assertEqual(pupil.scale, size/npix)
self.assertEqual(pupil.illuminated.shape, (npix, npix))
def testRawMetadata(self):
"""Test retrieval of raw image metadata"""
md = self.butler.get("raw_md", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(md.get("DET-ID"), self.ccdNum)
def testBias(self):
"""Test retrieval of bias frame"""
bias = self.butler.get("bias", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(bias.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(bias.getDetector().getId(), self.ccdNum)
def testDark(self):
"""Test retrieval of dark frame"""
dark = self.butler.get("dark", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(dark.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(dark.getDetector().getId(), self.ccdNum)
self.assertEqual(dark.getInfo().getVisitInfo().getExposureTime(), 1.0)
def testFlat(self):
"""Test retrieval of flat"""
flat = self.butler.get("flat", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(flat.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(flat.getDetector().getId(), self.ccdNum)
def testLinearizer(self):
self.assertTrue(self.butler.get('linearizer', ccdnum=1))
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
datadir = self.getTestDataDir()
self.repoPath = os.path.join(datadir, "DATA")
self.calibPath = os.path.join(datadir, "CALIB")
self.butler = dafPersist.Butler(root=self.repoPath,
calibRoot=self.calibPath)
self.size = (2112, 4644)
self.dataId = {'visit': 1038843}
self.filter = "i2"
self.exposureTime = 615.037
self.darkTime = 615.0
self.dateAvg = DateTime(54771.610881712964, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = SpherePoint(135.40941055, -2.39999432, degrees)
self.boresightAzAlt = SpherePoint(122.34, 52.02, degrees)
self.boresightAirmass = 1.269
self.rotType = RotType.UNKNOWN
self.obs_longitude = -155.468876*degrees
self.obs_latitude = 19.825252*degrees
self.obs_elevation = 4215
self.weath_airTemperature = 0.90
self.weath_airPressure = 617.65*100 # 100 Pascal/millibar
self.weath_humidity = 39.77
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 4.55388*radians
def tearDown(self):
del self.butler
def assertExposure(self, exp, ccd, checkFilter=True):
print("dataId: ", self.dataId)
print("ccd: ", ccd)
print("width: ", exp.getWidth())
print("height: ", exp.getHeight())
print("detector name: ", exp.getDetector().getName())
print("filter name: ", exp.getFilter().getFilterProperty().getName())
self.assertEqual(exp.getWidth(), self.size[0])
self.assertEqual(exp.getHeight(), self.size[1])
self.assertEqual(exp.getDetector().getName(), "ccd%02d" % ccd)
if checkFilter:
self.assertEqual(exp.getFilter().getFilterProperty().getName(), self.filter)
if display and ccd % 18 == 0:
global frame
frame += 1
ccd = exp.getDetector()
for amp in ccd:
amp = amp
print(ccd.getId(), amp.getId(), amp.getDataSec().toString(),
amp.getBiasSec().toString(), amp.getElectronicParams().getGain())
cameraGeomUtils.showCcd(ccd, ccdImage=exp, frame=frame)
def getTestDataDir(self):
try:
datadir = getPackageDir("testdata_cfht")
except pexExcept.NotFoundError as e:
warnings.warn(e.args[0])
raise unittest.SkipTest("Skipping test as testdata_cfht is not setup")
return datadir
def testRaw(self):
"""Test retrieval of raw image"""
if display:
global frame
frame += 1
cameraGeomUtils.showCamera(self.butler.mapper.camera, frame=frame)
for ccd in range(36):
raw = self.butler.get("raw", self.dataId, ccd=ccd, immediate=True)
self.assertExposure(raw, ccd)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
self.assertAnglesAlmostEqual(visitInfo.getEra(), self.era, maxDiff=0.1*degrees)
self.assertAlmostEqual(visitInfo.getExposureTime(), self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertTrue(math.isnan(visitInfo.getBoresightRotAngle()))
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def getDetrend(self, detrend):
"""Test retrieval of detrend image"""
for ccd in range(36):
flat = self.butler.get(detrend, self.dataId, ccd=ccd, immediate=True)
self.assertExposure(flat, ccd, checkFilter=False)
def testFlat(self):
self.getDetrend("flat")
def testBias(self):
self.getDetrend("bias")
def testFringe(self):
self.getDetrend("fringe")
def testPackageName(self):
name = dafPersist.Butler.getMapperClass(root=self.repoPath).packageName
self.assertEqual(name, "obs_cfht")
def testBoundaryMJD(self):
ts = DateTime(47892.0, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 631152000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 631152025000000000)
self.assertEqual(ts.get(DateTime.MJD, DateTime.UTC), 47892.0)
def testBoundaryMJD(self):
ts = DateTime(47892.0, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), long(631152000000000000))
self.assertEqual(ts.nsecs(DateTime.TAI), long(631152025000000000))
self.assertEqual(ts.get(DateTime.MJD, DateTime.UTC), 47892.0)
def _forcedPhotometry(
self, objects: pandas.DataFrame, latest_objects: pandas.DataFrame,
dt: DateTime,
visit_id: int) -> Tuple[pandas.DataFrame, pandas.DataFrame]:
"""Do forced photometry on latest_objects which are not in objects.
Extends objects catalog with new DiaObjects.
Parameters
----------
objects : `pandas.DataFrame`
Catalog containing DiaObject records
latest_objects : `pandas.DataFrame`
Catalog containing DiaObject records
dt : `DateTime`
Visit time.
visit_id : `int`
Visit ID.
"""
midPointTai = dt.get(system=DateTime.MJD)
if objects.empty:
return pandas.DataFrame(
columns=["diaObjectId", "ccdVisitId", "midPointTai", "flags"
]), objects
# Ids of the detected objects
ids = set(objects['diaObjectId'])
# do forced photometry for all detected DiaObjects
df1 = pandas.DataFrame({
"diaObjectId": objects["diaObjectId"],
"ccdVisitId": visit_id,
"midPointTai": midPointTai,
"flags": 0,
})
# do forced photometry for non-detected DiaObjects (newer than cutoff)
o1 = cast(pandas.DataFrame,
latest_objects[~latest_objects["diaObjectId"].isin(ids)])
# only do it for 30 days after last detection
cutoff = dt.toPython() - timedelta(days=self.config.forced_cutoff_days)
o1 = cast(pandas.DataFrame, o1[o1["lastNonForcedSource"] > cutoff])
if o1.empty:
catalog = df1
else:
df2 = pandas.DataFrame({
"diaObjectId": o1["diaObjectId"],
"ccdVisitId": visit_id,
"midPointTai": midPointTai,
"flags": 0,
})
# extend forced sources
catalog = pandas.concat([df1, df2])
# also extend objects
o2 = pandas.DataFrame({
"diaObjectId": o1["diaObjectId"],
"ra": o1["ra"],
"decl": o1["decl"],
})
objects = objects.append(o2)
return catalog, objects
