def testLeapSecond(self):
trials = ((45205., 21), (41498.99, 10), (41499.01, 11), (57203.99, 35),
(57204.01, 36), (57000., 35), (57210., 36))
for mjd, diff in trials:
ts = DateTime(mjd, DateTime.MJD, DateTime.UTC)
delta = ts.nsecs(DateTime.TAI) - ts.nsecs(DateTime.UTC)
self.assertEqual(delta / 1E9, diff)
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
self.assertTrue(ts.isValid())
def testPointsRA0(self):
"""Test for CcdImages crossing RA=0; this demonstrates a fix for
the bug described in DM-19802.
"""
# Use the same pixel origins as the cfht_minimal data, but put the sky origin at RA=0
crpix = lsst.geom.Point2D(931.517869, 2438.572109)
cd = np.array([[5.19513851e-05, -2.81124812e-07],
[-3.25186974e-07, -5.19112119e-05]])
crval1 = lsst.geom.SpherePoint(0.01, -0.01, lsst.geom.degrees)
crval2 = lsst.geom.SpherePoint(-0.01, 0.01, lsst.geom.degrees)
wcs1 = lsst.afw.geom.makeSkyWcs(crpix, crval1, cd)
wcs2 = lsst.afw.geom.makeSkyWcs(crpix, crval2, cd)
# Put the visit boresights at the WCS origin, for consistency
visitInfo1 = lsst.afw.image.VisitInfo(
exposureId=30577512,
date=DateTime(65321.1),
boresightRaDec=wcs1.getSkyOrigin())
visitInfo2 = lsst.afw.image.VisitInfo(
exposureId=30621144,
date=DateTime(65322.1),
boresightRaDec=wcs1.getSkyOrigin())
struct = lsst.jointcal.testUtils.createTwoFakeCcdImages(
fakeWcses=[wcs1, wcs2], fakeVisitInfos=[visitInfo1, visitInfo2])
self._testPoints(struct.ccdImageList[0], struct.ccdImageList[1],
struct.skyWcs[0], struct.skyWcs[1], struct.bbox)
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
dateBeg = DateTime(56598.26106374757, DateTime.MJD, DateTime.UTC)
dateAvgNSec = dateBeg.nsecs() + int(0.5e9*self.exptime)
self.dateAvg = DateTime(dateAvgNSec, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = MakeRawVisitInfo.centigradeFromKelvin(272.35)
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(621.7)
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
lst = 340.189608333*degrees
self.era = lst - self.obs_longitude
def preprocess(self):
self.activeClipboard = self.inputQueue.getNextDataset()
eventName = self._policy.get("inputEvent")
event = self.activeClipboard.get(eventName)
visitId = event.get("visitId")
exposureId = event.get("exposureId")
fpaExposureId = (long(visitId) << 1) + exposureId
visit = PropertySet()
visit.setInt("visitId", visitId)
visit.setLongLong("exposureId", fpaExposureId)
self.activeClipboard.put("visit" + str(exposureId), visit)
rawFpaExposure = PropertySet()
rawFpaExposure.setLongLong("rawFPAExposureId", fpaExposureId)
rawFpaExposure.set("ra", event.get("ra"))
rawFpaExposure.set("decl", event.get("decl"))
rawFpaExposure.set("filterId",
self.lookupFilterId(event.get("filter")))
rawFpaExposure.set("equinox", event.get("equinox"))
rawFpaExposure.set("dateObs", DateTime(event.get("dateObs")))
rawFpaExposure.set("mjdObs", DateTime(event.get("dateObs")).mjd())
rawFpaExposure.set("expTime", event.get("expTime"))
rawFpaExposure.set("airmass", event.get("airmass"))
self.activeClipboard.put("fpaExposure" + str(exposureId),
rawFpaExposure)
def testNsecsDefault(self):
ts = DateTime(1192755506000000000)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
self.assertTrue(ts.isValid())
def setUp(self):
datadir = self.getTestDataDir()
self.repoPath = os.path.join(datadir, "DATA")
self.calibPath = os.path.join(datadir, "CALIB")
self.butler = dafPersist.Butler(root=self.repoPath,
calibRoot=self.calibPath)
self.size = (2112, 4644)
self.dataId = {'visit': 1038843}
self.filter = "i2"
self.exposureTime = 615.037
self.darkTime = 615.0
dateObs = DateTime(54771.6066250, DateTime.MJD, DateTime.UTC)
self.dateAvg = DateTime(
dateObs.nsecs(DateTime.TAI) + int(0.5e9 * self.exposureTime),
DateTime.TAI)
self.boresightRaDec = IcrsCoord(135.409417 * degrees,
-2.400000 * degrees)
self.boresightAzAlt = Coord(122.34 * degrees, 52.02 * degrees)
self.boresightAirmass = 1.269
self.rotType = RotType.UNKNOWN
self.obs_longitude = -155.468876 * degrees
self.obs_latitude = 19.825252 * degrees
self.obs_elevation = 4204
self.weath_airTemperature = 0.90
self.weath_airPressure = 617.65 * 100 # 100 Pascal/millibar
self.weath_humidity = 39.77
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
lst = 104.16591666666666 * degrees
self.era = lst - self.obs_longitude
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
self.assertTrue(ts.isValid())
def testAstropyComparison(self):
"""Astropy's Time module is based on ERFA, providing a well verified
comparison point.
"""
def check_times(dateTime, time):
with self.subTest("jyear"):
self.assertAlmostEqual(dateTime.get(DateTime.EPOCH), time.tai.jyear)
with self.subTest("mjd"):
self.assertEqual(dateTime.get(DateTime.MJD), time.tai.mjd)
with self.subTest("jd"):
self.assertEqual(dateTime.get(DateTime.JD), time.tai.jd)
# Unix epoch comparison
dateTime = DateTime("19700101T000000Z", DateTime.UTC)
time = astropy.time.Time("1970-01-01T00:00:00", format="isot", scale="utc")
check_times(dateTime, time)
# J2000 epoch comparison
dateTime = DateTime(2000.0, DateTime.EPOCH, DateTime.UTC)
time = astropy.time.Time(2000.0, format="jyear", scale="utc")
check_times(dateTime, time)
# random future MJD epoch comparison
dateTime = DateTime(65432.1, DateTime.MJD, DateTime.TAI)
time = astropy.time.Time(65432.1, format="mjd", scale="tai")
check_times(dateTime, time)
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
self.dateAvg = DateTime(56598.26165414352, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = -0.8
self.weath_airPressure = 62170.
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 2.3659570321481826*radians
def setUp(self):
datadir = self.getTestDataDir()
self.repoPath = os.path.join(datadir, "DATA")
self.calibPath = os.path.join(datadir, "CALIB")
self.butler = dafPersist.Butler(root=self.repoPath,
calibRoot=self.calibPath)
self.size = (2112, 4644)
self.dataId = {'visit': 1038843}
self.filter = "i2"
self.exposureTime = 615.037
self.darkTime = 615.0
self.dateAvg = DateTime(54771.610881712964, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = SpherePoint(135.40941055, -2.39999432, degrees)
self.boresightAzAlt = SpherePoint(122.34, 52.02, degrees)
self.boresightAirmass = 1.269
self.boresightRotAngle = 0 * degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.468876 * degrees
self.obs_latitude = 19.825252 * degrees
self.obs_elevation = 4215
self.weath_airTemperature = 0.90
self.weath_airPressure = 617.65 * 100 # 100 Pascal/millibar
self.weath_humidity = 39.77
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 4.55388 * radians
def testPopMjdDate(self):
for timesys in (None, "UTC", "TAI"):
dataDict = {
"DATE1": 51943.1705801,
"DATE2": 44302.0433218,
"BADMJDDATE": "2001-02-03T04:05:06.123456789",
}
if timesys is not None:
dataDict["TIMESYS"] = timesys
length = len(dataDict)
for key, mjdDate in dataDict.items():
if not key.startswith("DATE"):
continue
lsstSys = dict(
UTC=DateTime.UTC,
TAI=DateTime.TAI,
).get(timesys, DateTime.UTC)
desDate = DateTime(mjdDate, DateTime.MJD, lsstSys)
md = getMetadata(dataDict)
date = self.makeRawVisitInfo.popMjdDate(md, key, timesys=timesys)
self.assertEqual(len(md.names()), length)
self.assertAlmostEqual(date.get(), desDate.get())
badDate = self.makeRawVisitInfo.popMjdDate(md, "BADMJDDATE")
self.assertEqual(badDate, DateTime())
missingDate = self.makeRawVisitInfo.popMjdDate(md, "BADKEY")
self.assertEqual(missingDate, DateTime())
def testMJD(self):
ts = DateTime(45205.125, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 399006000000000000L)
self.assertEqual(ts.nsecs(DateTime.TAI), 399006021000000000L)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.TAI), 45205.125 + 21.0/86400.0)
# Following interface is deprecated
self.assertAlmostEqual(ts.mjd(DateTime.UTC), 45205.125)
self.assertAlmostEqual(ts.mjd(DateTime.TAI), 45205.125 + 21.0/86400.0)
def testIsoNonUTCBasics(self):
"""Test basic ISO string input and output of TAI and TT dates"""
for scale in (DateTime.TAI, DateTime.TT):
for dateSep in ("-", ""): # "-" date separator is optional
for timeSep in (":", ""): # ":" time separator is optional
for decPt in (".", ","): # "." or "," may be used as decimal point
dateStr = "2009{0}04{0}02T07{1}26{1}39{2}314159265".format(dateSep, timeSep, decPt)
ts = DateTime(dateStr, scale)
self.assertEqual(ts.toString(scale), "2009-04-02T07:26:39.314159265")
self.assertTrue(ts.isValid())
def testStr(self):
timeStr1 = "2004-03-01T12:39:45.1"
fullTimeStr1 = "2004-03-01T12:39:45.100000000"
dt1 = DateTime(timeStr1, DateTime.TAI)
self.assertEqual(str(dt1), "DateTime(\"{}\", TAI)".format(fullTimeStr1))
self.assertEqual(repr(dt1), "DateTime(\"{}\", TAI)".format(fullTimeStr1))
timeStr2 = "2004-03-01T12:39:45.000000001"
dt2 = DateTime(timeStr2, DateTime.TAI)
self.assertEqual(str(dt2), "DateTime(\"{}\", TAI)".format(timeStr2))
self.assertEqual(repr(dt2), "DateTime(\"{}\", TAI)".format(timeStr2))
def testIsoNonUTCBasics(self):
"""Test basic ISO string input and output of TAI and TT dates
"""
for scale in (DateTime.TAI, DateTime.TT):
for dateSep in ("-", ""): # "-" date separator is optional
for timeSep in (":", ""): # ":" time separator is optional
for decPt in (".", ","): # "." or "," may be used as decimal point
dateStr = "2009{0}04{0}02T07{1}26{1}39{2}314159265".format(dateSep, timeSep, decPt)
ts = DateTime(dateStr, scale)
self.assertEqual(ts.toString(scale), "2009-04-02T07:26:39.314159265")
self.assertTrue(ts.isValid())
def testIsoUTCBasic(self):
"""Test basic ISO string input and output of UTC dates"""
for dateSep in ("-", ""): # "-" date separator is optional
for timeSep in (":", ""): # ":" time separator is optional
for decPt in (".", ","): # "." or "," may be used as decimal point
dateStr = "2009{0}04{0}02T07{1}26{1}39{2}314159265Z".format(dateSep, timeSep, decPt)
ts = DateTime(dateStr, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.TT), 1238657265498159265)
self.assertEqual(ts.nsecs(DateTime.TAI), 1238657233314159265)
self.assertEqual(ts.nsecs(DateTime.UTC), 1238657199314159265)
self.assertEqual(ts.toString(ts.UTC), "2009-04-02T07:26:39.314159265Z")
def testNsecs(self):
ts = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000)
self.assertEqual(ts.nsecs(), 1192755506000000000)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
ts2 = ts
self.assertEqual(ts, ts2)
ts2 = DateTime(1192755473000000000, DateTime.UTC)
self.assertEqual(ts, ts2)
ts2 = DateTime(1234567890000000000, DateTime.UTC)
self.assertNotEqual(ts, ts2)
def testLeapSecond(self):
trials = ((45205., 21),
(41498.99, 10),
(41499.01, 11),
(57203.99, 35),
(57204.01, 36),
(57000., 35),
(57210., 36))
for mjd, diff in trials:
ts = DateTime(mjd, DateTime.MJD, DateTime.UTC)
delta = ts.nsecs(DateTime.TAI) - ts.nsecs(DateTime.UTC)
self.assertEqual(delta/1E9, diff)
def testIsoUTCBasic(self):
"""Test basic ISO string input and output of UTC dates
"""
for dateSep in ("-", ""): # "-" date separator is optional
for timeSep in (":", ""): # ":" time separator is optional
for decPt in (".", ","): # "." or "," may be used as decimal point
dateStr = "2009{0}04{0}02T07{1}26{1}39{2}314159265Z".format(dateSep, timeSep, decPt)
ts = DateTime(dateStr, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.TT), 1238657265498159265)
self.assertEqual(ts.nsecs(DateTime.TAI), 1238657233314159265)
self.assertEqual(ts.nsecs(DateTime.UTC), 1238657199314159265)
self.assertEqual(ts.toString(ts.UTC), "2009-04-02T07:26:39.314159265Z")
def _visitTimes(start_time: DateTime, interval_sec: int,
count: int) -> Iterator[DateTime]:
"""
Generator for visit times.
"""
nsec = start_time.nsecs(DateTime.TAI)
delta = interval_sec * 1_000_000_000
while count > 0:
dt = DateTime(nsec, DateTime.TAI)
if not _isDayTime(dt):
yield dt
count -= 1
nsec += delta
def testSOFA(self):
"""The SOFA documentation includes an example conversion:
https://www.iausofa.org/2017_0420_C/sofa/sofa_ts_c.pdf
(page 8, section 2.4)
The value in those docs is only ~single precision, so I re-computed
it with pyerfa to get a more correct value.
"""
with self.subTest("jd to jyear"):
self.assertEqual(DateTime(2457073.05631, DateTime.JD, DateTime.TAI).get(DateTime.EPOCH),
2015.1349933196439)
with self.subTest("jyear to jd"):
self.assertEqual(DateTime(2015.1349933196, DateTime.EPOCH, DateTime.TAI).get(DateTime.JD),
2457073.056309984)
def setUp(self):
self.testDir = os.path.dirname(__file__)
def computeLstHA(data):
"""Return LST, Hour Angle, computed from VisitInfoData."""
localEra = data.era + data.observatory.getLongitude()
hourAngle = localEra - data.boresightRaDec[0]
return localEra, hourAngle
fields = [
'exposureId', 'exposureTime', 'darkTime', 'date', 'ut1', 'era',
'boresightRaDec', 'boresightAzAlt', 'boresightAirmass',
'boresightRotAngle', 'rotType', 'observatory', 'weather'
]
VisitInfoData = collections.namedtuple("VisitInfoData", fields)
data1 = VisitInfoData(
exposureId=10313423,
exposureTime=10.01,
darkTime=11.02,
date=DateTime(65321.1, DateTime.MJD, DateTime.TAI),
ut1=12345.1,
era=45.1 * degrees,
boresightRaDec=SpherePoint(23.1 * degrees, 73.2 * degrees),
boresightAzAlt=SpherePoint(134.5 * degrees, 33.3 * degrees),
boresightAirmass=1.73,
boresightRotAngle=73.2 * degrees,
rotType=afwImage.RotType.SKY,
observatory=Observatory(11.1 * degrees, 22.2 * degrees, 0.333),
weather=Weather(1.1, 2.2, 34.5),
)
self.data1 = data1
self.localEra1, self.hourAngle1 = computeLstHA(data1)
data2 = VisitInfoData(
exposureId=1,
exposureTime=15.5,
darkTime=17.8,
date=DateTime(55321.2, DateTime.MJD, DateTime.TAI),
ut1=312345.1,
era=25.1 * degrees,
boresightRaDec=SpherePoint(2.1 * degrees, 33.2 * degrees),
boresightAzAlt=SpherePoint(13.5 * degrees, 83.3 * degrees),
boresightAirmass=2.05,
boresightRotAngle=-53.2 * degrees,
rotType=afwImage.RotType.HORIZON,
observatory=Observatory(22.2 * degrees, 33.3 * degrees, 0.444),
weather=Weather(2.2, 3.3, 44.4),
)
self.data2 = data2
self.localEra2, self.hourAngle2 = computeLstHA(data2)
def testDM7622(self):
"""Test DM-7622: date with unix time = -1 seconds must be usable
Note that the call in question parses the ISO string without paying
attention to the scale (it applies the scale later),
so the same ISO string is wanted in all cases
(except with a trailing Z for UTC, and without for TAI and TT)
"""
negOneSecIso = "1969-12-31T23:59:59.000000000"
for scale in self.timeScales:
dateStr = negOneSecIso + ("Z" if scale == DateTime.UTC else "")
try:
dt = DateTime(dateStr, scale)
except Exception:
self.fail("Date {} unusable; DM-7622 is still with us".format(dateStr, scale))
self.assertEqual(dt.nsecs(scale), int(-1e9))
def popIsoDate(self, md, key, timesys=None):
"""Pop a FITS ISO date as an lsst.daf.base.DateTime
Parameters
----------
md : `lsst.daf.base.PropertyList` or `PropertySet`
Metadata to pull `key` from and remove.
key : `str`
Date key to read and remove from md.
timesys : `str`
Time system as a string (not case sensitive), e.g. "UTC" or None;
if None then look for TIMESYS (but do NOT pop it, since it may be
used for more than one date) and if not found, use UTC.
Returns
-------
`lsst.daf.base.DateTime`
Value of the requested date; `DateTime()` if the key is not found.
"""
isoDateStr = self.popItem(md=md, key=key)
if isoDateStr is not None:
try:
if timesys is None:
timesys = md.getScalar("TIMESYS") if md.exists("TIMESYS") else "UTC"
if isoDateStr.endswith("Z"): # illegal in FITS
isoDateStr = isoDateStr[0:-1]
astropyTime = astropy.time.Time(isoDateStr, scale=timesys.lower(), format="fits")
# DateTime uses nanosecond resolution, regardless of the resolution of the original date
astropyTime.precision = 9
# isot is ISO8601 format with "T" separating date and time and no time zone
return DateTime(astropyTime.tai.isot, DateTime.TAI)
except Exception as e:
self.log.warn("Could not parse {} = {} as an ISO date: {}".format(key, isoDateStr, e))
return BadDate
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
self.dateAvg = DateTime(56598.261654253474, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = -0.8
self.weath_airPressure = 62170.
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 2.3659577248977826*radians
def _testIsEmpty(self, visitInfo):
"""Test that visitInfo is all NaN or 0, as appropriate."""
self.assertEqual(visitInfo.getExposureId(), 0)
self.assertTrue(math.isnan(visitInfo.getExposureTime()))
self.assertTrue(math.isnan(visitInfo.getDarkTime()))
self.assertEqual(visitInfo.getDate(), DateTime())
self.assertTrue(math.isnan(visitInfo.getUt1()))
self.assertTrue(math.isnan(visitInfo.getEra().asDegrees()))
for i in range(2):
self.assertTrue(
math.isnan(visitInfo.getBoresightRaDec()[i].asDegrees()))
self.assertTrue(
math.isnan(visitInfo.getBoresightAzAlt()[i].asDegrees()))
self.assertTrue(math.isnan(visitInfo.getBoresightAirmass()))
self.assertTrue(
math.isnan(visitInfo.getBoresightRotAngle().asDegrees()))
self.assertEqual(visitInfo.getRotType(), afwImage.RotType.UNKNOWN)
self.assertTrue(
math.isnan(visitInfo.getObservatory().getLongitude().asDegrees()))
self.assertTrue(
math.isnan(visitInfo.getObservatory().getLatitude().asDegrees()))
self.assertTrue(math.isnan(visitInfo.getObservatory().getElevation()))
self.assertTrue(math.isnan(visitInfo.getWeather().getAirTemperature()))
self.assertTrue(math.isnan(visitInfo.getWeather().getAirPressure()))
self.assertTrue(math.isnan(visitInfo.getWeather().getHumidity()))
self.assertTrue(math.isnan(visitInfo.getBoresightHourAngle()))
def testDM7622(self):
"""Test DM-7622: date with unix time = -1 seconds must be usable
Note that the call in question parses the ISO string without paying
attention to the scale (it applies the scale later),
so the same ISO string is wanted in all cases
(except with a trailing Z for UTC, and without for TAI and TT)
"""
negOneSecIso = "1969-12-31T23:59:59.000000000"
for scale in self.timeScales:
dateStr = negOneSecIso + ("Z" if scale == DateTime.UTC else "")
try:
dt = DateTime(dateStr, scale)
except Exception:
self.fail("Date {} in scale {} unusable".format(dateStr, scale))
self.assertEqual(dt.nsecs(scale), int(-1e9))
def popMjdDate(self, md, key, timesys=None):
"""Get a FITS MJD date as an lsst.daf.base.DateTime
@param[in,out] md metadata, as an lsst.daf.base.PropertyList or PropertySet
@param[in] dateKey date key to read and remove from md
@param[in] timesys time system as a string, e.g. "UTC" or None;
if None then look for TIMESYS (but do NOT pop it, since it may be used
for more than one date) and if not found, use UTC
@return date as an lsst.daf.base.DateTime; DateTime() if the key is not found
"""
mjdDate = self.popFloat(md, key)
try:
if timesys is None:
timesys = md.get("TIMESYS") if md.exists("TIMESYS") else "UTC"
astropyTime = astropy.time.Time(mjdDate,
format="mjd",
scale=timesys.lower())
# DateTime uses nanosecond resolution, regardless of the resolution of the original date
astropyTime.precision = 9
# isot is ISO8601 format with "T" separating date and time and no time zone
return DateTime(astropyTime.tai.isot, DateTime.TAI)
except Exception as e:
self.log.warn("Could not parse {} = {} as an MJD date: {}".format(
key, mjdDate, e))
return BadDate
def popMjdDate(self, md, key, timesys=None):
"""Get a FITS MJD date as an ``lsst.daf.base.DateTime``.
Parameters
----------
md : `lsst.daf.base.PropertyList` or `PropertySet`
Metadata to pull `key` from and remove.
key : `str`
Date key to read and remove from md.
timesys : `str`
Time system as a string (not case sensitive), e.g. "UTC" or None;
if None then look for TIMESYS (but do NOT pop it, since it may be
used for more than one date) and if not found, use UTC.
Returns
-------
`lsst.daf.base.DateTime`
Value of the requested date; `DateTime()` if the key is not found.
"""
mjdDate = self.popFloat(md, key)
try:
if timesys is None:
timesys = md.get("TIMESYS") if md.exists("TIMESYS") else "UTC"
astropyTime = astropy.time.Time(mjdDate,
format="mjd",
scale=timesys.lower())
# DateTime uses nanosecond resolution, regardless of the resolution of the original date
astropyTime.precision = 9
# isot is ISO8601 format with "T" separating date and time and no time zone
return DateTime(astropyTime.tai.isot, DateTime.TAI)
except Exception as e:
self.log.warn("Could not parse {} = {} as an MJD date: {}".format(
key, mjdDate, e))
return BadDate
def testFracSecs(self):
ts = DateTime("2004-03-01T12:39:45.1Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '2004-03-01T12:39:45.100000000Z')
ts = DateTime("2004-03-01T12:39:45.01Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '2004-03-01T12:39:45.010000000Z')
ts = DateTime("2004-03-01T12:39:45.000000001Z", DateTime.UTC) # nanosecond
self.assertEqual(ts.toString(ts.UTC), '2004-03-01T12:39:45.000000001Z')
ts = DateTime("2004-03-01T12:39:45.0000000001Z", DateTime.UTC) # too small
self.assertEqual(ts.toString(ts.UTC), '2004-03-01T12:39:45.000000000Z')
def testConvert(self):
year = 2012
month = 7
day = 19
hour = 18
minute = 29
second = 33
ts = DateTime(year, month, day, hour, minute, second, DateTime.UTC)
dt = ts.toPython(DateTime.UTC)
self.assertEqual(dt.year, year)
self.assertEqual(dt.month, month)
self.assertEqual(dt.day, day)
self.assertEqual(dt.hour, hour)
self.assertEqual(dt.minute, minute)
self.assertEqual(dt.second, second)
def testWraparound(self):
"""Test that a date later than 2038-01-19, 03:14:07 does not wrap around
This will fail on old versions of unix, and indicates that DateTime is not safe
"""
dateStr = "2040-01-01T00:00:00.000000000"
self.assertEquals(str(DateTime(dateStr, DateTime.TAI)),
"DateTime(\"{}\", TAI)".format(dateStr))
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def testConvert(self):
year = 2012
month = 7
day = 19
hour = 18
minute = 29
second = 33
ts = DateTime(year, month, day, hour, minute, second, DateTime.UTC)
dt = ts.toPython(DateTime.UTC)
self.assertEqual(dt.year, year)
self.assertEqual(dt.month, month)
self.assertEqual(dt.day, day)
self.assertEqual(dt.hour, hour)
self.assertEqual(dt.minute, minute)
self.assertEqual(dt.second, second)
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def testGetDR7(self):
obsSdssDir = lsst.utils.getPackageDir('obs_sdss')
butler = dafPersist.Butler(
root=os.path.join(obsSdssDir, "tests", "data", "dr7", "runs"))
sub = butler.subset("fpC", run=5754, camcol=3, field=280, filter="r")
self.assertEqual(len(sub), 1)
for ref in sub:
im = ref.get("fpC")
w, h = im.getWidth(), im.getHeight()
self.assertEqual(im.__class__, lsst.afw.image.ExposureU)
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
im_md = ref.get("fpC_md")
self.assertEqual(im_md.getScalar("RUN"), 5754)
self.assertEqual(im_md.getScalar("FRAME"), 280)
self.assertEqual(im_md.getScalar("STRIPE"), 82)
msk = ref.get("fpM")
w, h = msk.getWidth(), msk.getHeight()
self.assertEqual(msk.__class__, lsst.afw.image.Mask[lsst.afw.image.MaskPixel])
self.assertEqual(w, 2048)
self.assertEqual(h, 1489)
psf = ref.get("psField")
k = psf.getKernel()
w, h = k.getWidth(), k.getHeight()
self.assertEqual(psf.__class__, lsst.meas.algorithms.PcaPsf)
self.assertEqual(w, 31)
self.assertEqual(h, 31)
wcs = ref.get("asTrans")
self.assertIsInstance(wcs, SkyWcs)
self.assertFalse(wcs.isFlipped)
# comparison is to results from lsst.afw.image.TanWcs class
self.assertSpherePointsAlmostEqual(wcs.pixelToSky(700, 1000),
SpherePoint(343.6507738304687, -0.3509870420713227, degrees))
tsField = ref.get("tsField")
self.assertAlmostEqual(tsField.gain, 4.72, 2)
self.assertAlmostEqual(tsField.airmass, 1.2815132857671601)
self.assertAlmostEqual(tsField.exptime, 53.907456)
predDateStart = DateTime(53664.226070589997, DateTime.MJD, DateTime.TAI)
predDateAvg = DateTime(predDateStart.nsecs(DateTime.TAI) + int(0.5e9*tsField.exptime),
DateTime.TAI)
self.assertAlmostEqual(tsField.dateAvg.get(), predDateAvg.get())
def process(self, clipboard):
metadata = clipboard.get(self.policy.getString("inputKeys.metadata"))
cdb = clipboard.get(self.policy.getString("inputKeys.calibrationDB"))
pathPrefix =
cliboard.get(self.policy.getString("inputKeys.prefixPath"))
when = DateTime(metadata.get("dateObs"))
ccdId = metadata.get("ccdId")
ampId = metadata.get("ampId")
expTime = metadata.get("expTime")
darkToApply = self.policy.getString("whichdark")
darkCalibList = cdb.lookup(when, "dark", ccdId, ampId, all=True)
darkTimeList = []
for d in darkCalibList:
darkTimeList.append(d.expTime)
darkTimeList.sort()
if darktoapply == "min":
darkExpTime = darkTimeList[0]
elif darktoapply == "max":
darkExpTime = darkTimeList[-1]
elif darktoapply == "closest":
minDist = abs(expTime - darkTimeList[0])
minExpTime = darkTimeList[0]
for i in xrange(1, len(darkTimeList)):
dist = abs(expTime - darkTimeList[i])
if dist < minDist:
minDist = dist
minExpTime = darkTimeList[i]
darkExpTime = minExpTime
else:
raise RuntimeError, "Unrecognized method for finding dark to apply: " + str(darktoapply)
biasPath = cdb.lookup(when, "bias", ccdId, ampId)
darkPath = cdb.lookup(when, "dark", ccdId, ampId,
expTime=darkExpTime)
defectPath = cdb.lookup(when, "defect", ccdId, ampId)
flatPath = cdb.lookup(when, "flat", ccdId, ampId,
filter=metadata.get("filter"))
# fringePath = cdb.lookup(when, "fringe", ccdId, ampId,
# filter=metadata.get("filter"))
linearizePath = cdb.lookup(when, "linearize", ccdId, ampId)
calibData = PropertySet()
calibData.set("biasPath", os.path.join(pathPrefix, biasPath))
calibData.set("darkPath", os.path.join(pathPrefix, darkPath))
calibData.set("defectPath", os.path.join(pathPrefix, defectPath))
calibData.set("flatPath", os.path.join(pathPrefix, flatPath))
# calibData.set("fringePath", os.path.join(pathPrefix, fringePath))
calibData.set("linearizePath", os.path.join(pathPrefix, linearizePath))
outputKey = self._policy.get("outputKey")
self.activeClipboard.put(outputKey, calibData)
self.outputQueue.addDataset(self.activeClipboard)
def testNow(self):
successes = 0
for i in xrange(10): # pylint: disable-msg=W0612
secs = time.time()
ts = DateTime.now()
diff = ts.nsecs(DateTime.UTC) / 1.0e9 - secs
if diff > -0.001 and diff < 0.1:
successes += 1
self.assertGreaterEqual(successes, 3)
def testNow(self):
successes = 0
for _ in range(10):
secs = time.time()
ts = DateTime.now()
diff = ts.nsecs(DateTime.UTC) / 1.0e9 - secs
if diff > -0.001 and diff < 0.1:
successes += 1
self.assertGreaterEqual(successes, 3)
def testNow(self):
successes = 0
for _ in range(10):
secs = time.time()
ts = DateTime.now()
diff = ts.nsecs(DateTime.UTC)/1.0e9 - secs
if diff > -0.001 and diff < 0.1:
successes += 1
self.assertGreaterEqual(successes, 3)
def testPopIsoDate(self):
for timesys in (None, "UTC", "TAI"):
dataDict = {
"DATE1": "2001-02-03T04:05:06.123456789",
"DATE2":
"2001-02-03T04:05:06.123456789Z", # Z should be ignored
"DATE3": "1980-03-04T01:02:03.999999999",
"BADISODATE": "51234.354",
}
if timesys is not None:
dataDict["TIMESYS"] = timesys
for key, dateStr in dataDict.items():
if not key.startswith("DATE"):
continue
lsstSys = dict(
UTC=DateTime.UTC,
TAI=DateTime.TAI,
).get(timesys, DateTime.UTC)
# lsstDateStr = dateStr with trailing Z if UTC, else no trailing Z,
# because lsst.daf.base.DateTime is very picky
lsstDateStr = dateStr
if lsstSys == DateTime.UTC:
if not lsstDateStr.endswith("Z"):
lsstDateStr = lsstDateStr + "Z"
elif dateStr.endswith("Z"):
lsstDateStr = lsstDateStr[0:-1]
desDate = DateTime(lsstDateStr, lsstSys)
md = getMetadata(dataDict)
numKeys = len(md.names())
date = self.makeRawVisitInfo.popIsoDate(md,
key,
timesys=timesys)
self.assertEqual(len(md.names()), numKeys - 1)
self.assertEqual(date, desDate)
badDate = self.makeRawVisitInfo.popIsoDate(md, "BADISODATE")
self.assertEqual(badDate, DateTime())
missingDate = self.makeRawVisitInfo.popIsoDate(md, "BADKEY")
self.assertEqual(missingDate, DateTime())
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def tearDown(self):
del self.butler
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=85471048, snap=0, raft='0,3', sensor='0,1', channel='1,0',
immediate=True)
self.assertEqual(raw.getWidth(), 513)
self.assertEqual(raw.getHeight(), 2001)
self.assertEqual(raw.getFilter().getFilterProperty().getName(), "y")
self.assertEqual(raw.getDetector().getName(), "R:0,3 S:0,1")
origin = raw.getWcs().getSkyOrigin()
self.assertAlmostEqual(
origin.getLongitude().asDegrees(), 0.0058520, 6)
self.assertAlmostEqual(
origin.getLatitude().asDegrees(), -2.3052624, 6)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
# Explicit test for NaN here, because phosim output may not have consistent alt/az/ra/dec/time
self.assertTrue(math.isnan(visitInfo.getEra()))
self.assertAlmostEqual(visitInfo.getExposureTime(), self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
class GetRawTestCase(lsst.utils.tests.TestCase):
"""Testing butler raw image retrieval"""
def setUp(self):
self.butler = dafPersist.Butler(root=os.path.join(os.path.dirname(__file__), "data"))
self.exposureTime = 15.0
self.darkTime = 15.0
dateObs = DateTime(49552.28496, DateTime.MJD, DateTime.TAI)
self.dateAvg = DateTime(dateObs.nsecs(DateTime.TAI) + int(0.5e9*self.exposureTime), DateTime.TAI)
self.boresightRaDec = SpherePoint(359.936019771151, -2.3356222648145, degrees)
self.boresightAzAlt = SpherePoint(127.158246182602, 90 - 40.6736117075876, degrees)
self.boresightAirmass = 1.31849492005496
self.boresightRotAngle = -3.43228*degrees
self.rotType = RotType.SKY
self.obs_longitude = -70.749417*degrees
self.obs_latitude = -30.244633*degrees
self.obs_elevation = 2663.0
self.weath_airTemperature = 5.0
self.weath_airPressure = MakeRawVisitInfo.pascalFromMmHg(520.0)
self.weath_humidity = 40.
def tearDown(self):
del self.butler
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=85471048, snap=0, raft='0,3', sensor='0,1', channel='1,0',
immediate=True)
self.assertEqual(raw.getWidth(), 513)
self.assertEqual(raw.getHeight(), 2001)
self.assertEqual(raw.getFilter().getFilterProperty().getName(), "y")
self.assertEqual(raw.getDetector().getName(), "R:0,3 S:0,1")
origin = raw.getWcs().getSkyOrigin()
self.assertAlmostEqual(
origin.getLongitude().asDegrees(), 0.0058520, 6)
self.assertAlmostEqual(
origin.getLatitude().asDegrees(), -2.3052624, 6)
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
# Explicit test for NaN here, because phosim output may not have consistent alt/az/ra/dec/time
self.assertTrue(math.isnan(visitInfo.getEra()))
self.assertAlmostEqual(visitInfo.getExposureTime(), self.exposureTime)
self.assertAlmostEqual(visitInfo.getDarkTime(), self.darkTime)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def createVisitInfo():
return lsst.afw.image.VisitInfo(
10313423, 10.01, 11.02,
DateTime(65321.1, DateTime.MJD,
DateTime.TAI), 12345.1, 45.1 * degrees,
SpherePoint(23.1 * degrees, 73.2 * degrees),
SpherePoint(134.5 * degrees, 33.3 * degrees), 1.73, 73.2 * degrees,
lsst.afw.image.RotType.SKY,
Observatory(11.1 * degrees, 22.2 * degrees, 0.333),
Weather(1.1, 2.2, 34.5), "testCam")
def testPointsRA0(self):
"""Test for CcdImages crossing RA=0; this demonstrates a fix for
the bug described in DM-19802.
"""
wcs1, wcs2 = make_fake_wcs()
# Put the visit boresights at the WCS origin, for consistency
visitInfo1 = lsst.afw.image.VisitInfo(
exposureId=30577512,
date=DateTime(65321.1),
boresightRaDec=wcs1.getSkyOrigin())
visitInfo2 = lsst.afw.image.VisitInfo(
exposureId=30621144,
date=DateTime(65322.1),
boresightRaDec=wcs1.getSkyOrigin())
struct = lsst.jointcal.testUtils.createTwoFakeCcdImages(
fakeWcses=[wcs1, wcs2], fakeVisitInfos=[visitInfo1, visitInfo2])
self._testPoints(struct.ccdImageList[0], struct.ccdImageList[1],
struct.skyWcs[0], struct.skyWcs[1], struct.bbox)
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i+1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList, izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def setUp(self):
datadir = self.getTestDataDir()
self.repoPath = os.path.join(datadir, "DATA")
self.calibPath = os.path.join(datadir, "CALIB")
self.butler = dafPersist.Butler(root=self.repoPath,
calibRoot=self.calibPath)
self.size = (2112, 4644)
self.dataId = {'visit': 1038843}
self.filter = "i2"
self.exposureTime = 615.037
self.darkTime = 615.0
self.dateAvg = DateTime(54771.610881712964, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = SpherePoint(135.40941055, -2.39999432, degrees)
self.boresightAzAlt = SpherePoint(122.34, 52.02, degrees)
self.boresightAirmass = 1.269
self.rotType = RotType.UNKNOWN
self.obs_longitude = -155.468876*degrees
self.obs_latitude = 19.825252*degrees
self.obs_elevation = 4215
self.weath_airTemperature = 0.90
self.weath_airPressure = 617.65*100 # 100 Pascal/millibar
self.weath_humidity = 39.77
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 4.55388*radians
class GetDataTestCase(lsst.utils.tests.TestCase):
"""Demonstrate retrieval of data from a repository."""
@unittest.skipIf(testDataDirectory is None, "%s is not available" % testDataPackage)
def setUp(self):
rawPath = os.path.join(testDataDirectory, "hsc", "raw")
calibPath = os.path.join(testDataDirectory, "hsc", "calib")
self.repoPath = createDataRepository("lsst.obs.hsc.HscMapper", rawPath, calibPath)
self.butler = dafPersist.Butler(root=self.repoPath, calibRoot=calibPath)
# The following properties of the provided data are known a priori.
self.visit = 904024
self.ccdNum = 50
self.filter = 'i'
self.rawSize = (2144, 4241)
self.ccdSize = (2048, 4176)
self.exptime = 30.0
self.darktime = self.exptime # No explicit darktime
self.dateAvg = DateTime(56598.261654253474, DateTime.MJD, DateTime.TAI)
self.boresightRaDec = afwGeom.SpherePoint(320.7499250000, 0.500019444, degrees)
self.boresightAzAlt = afwGeom.SpherePoint(226.68922661, 63.04359233, degrees)
self.boresightAirmass = 1.121626027604189
self.boresightRotAngle = 270.0*degrees
self.rotType = RotType.SKY
self.obs_longitude = -155.476667*degrees
self.obs_latitude = 19.825556*degrees
self.obs_elevation = 4139
self.weath_airTemperature = -0.8
self.weath_airPressure = 62170.
self.weath_humidity = 33.1
# NOTE: if we deal with DM-8053 and get UT1 implemented, ERA will change slightly.
self.era = 2.3659577248977826*radians
def tearDown(self):
del self.butler
rmtree(self.repoPath)
def testRaw(self):
"""Test retrieval of raw image"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
ccd = raw.getDetector()
self.assertEqual(raw.getDimensions(), afwGeom.Extent2I(*self.rawSize))
self.assertEqual(raw.getFilter().getFilterProperty().getName(), self.filter)
self.assertEqual(ccd.getId(), self.ccdNum)
self.assertEqual(ccd.getBBox().getDimensions(), afwGeom.Extent2I(*self.ccdSize))
visitInfo = raw.getInfo().getVisitInfo()
self.assertAlmostEqual(visitInfo.getDate().get(), self.dateAvg.get())
self.assertAnglesAlmostEqual(visitInfo.getEra(), self.era)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightRaDec(), self.boresightRaDec)
self.assertSpherePointsAlmostEqual(visitInfo.getBoresightAzAlt(), self.boresightAzAlt)
self.assertAlmostEqual(visitInfo.getBoresightAirmass(), self.boresightAirmass)
self.assertAnglesAlmostEqual(visitInfo.getBoresightRotAngle(), self.boresightRotAngle)
self.assertEqual(visitInfo.getRotType(), self.rotType)
self.assertEqual(visitInfo.getExposureTime(), self.exptime)
self.assertEqual(visitInfo.getDarkTime(), self.darktime)
observatory = visitInfo.getObservatory()
self.assertAnglesAlmostEqual(observatory.getLongitude(), self.obs_longitude)
self.assertAnglesAlmostEqual(observatory.getLatitude(), self.obs_latitude)
self.assertAlmostEqual(observatory.getElevation(), self.obs_elevation)
weather = visitInfo.getWeather()
self.assertAlmostEqual(weather.getAirTemperature(), self.weath_airTemperature)
self.assertAlmostEqual(weather.getAirPressure(), self.weath_airPressure, places=4)
self.assertAlmostEqual(weather.getHumidity(), self.weath_humidity)
def testPupil(self):
"""Test retrieval of pupil (without checking value)"""
raw = self.butler.get("raw", visit=self.visit, ccd=self.ccdNum)
visitInfo = raw.getInfo().getVisitInfo()
camera = self.butler.get("camera", visit=self.visit, ccd=self.ccdNum)
size = 16.4
npix = 1024
pupilFactory = camera.getPupilFactory(visitInfo, size, npix)
self.assertIsInstance(pupilFactory, lsst.obs.hsc.hscPupil.HscPupilFactory)
pupil = pupilFactory.getPupil(afwGeom.Point2D(0.0, 0.0))
self.assertEqual(pupil.size, size)
self.assertEqual(pupil.scale, size/npix)
self.assertEqual(pupil.illuminated.shape, (npix, npix))
def testRawMetadata(self):
"""Test retrieval of raw image metadata"""
md = self.butler.get("raw_md", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(md.getScalar("DET-ID"), self.ccdNum)
def testBias(self):
"""Test retrieval of bias frame"""
bias = self.butler.get("bias", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(bias.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(bias.getDetector().getId(), self.ccdNum)
def testDark(self):
"""Test retrieval of dark frame"""
dark = self.butler.get("dark", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(dark.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(dark.getDetector().getId(), self.ccdNum)
self.assertEqual(dark.getInfo().getVisitInfo().getExposureTime(), 1.0)
def testFlat(self):
"""Test retrieval of flat"""
flat = self.butler.get("flat", visit=self.visit, ccd=self.ccdNum)
self.assertEqual(flat.getDimensions(), afwGeom.Extent2I(*self.ccdSize))
self.assertEqual(flat.getDetector().getId(), self.ccdNum)
def testLinearizer(self):
self.assertTrue(self.butler.get('linearizer', ccdnum=1))
def testNsecsTAI(self):
ts = DateTime(1192755506000000000L, DateTime.TAI)
self.assertEqual(ts.nsecs(DateTime.UTC), 1192755473000000000L)
self.assertEqual(ts.nsecs(DateTime.TAI), 1192755506000000000L)
self.assertEqual(ts.nsecs(), 1192755506000000000L)
self.assertAlmostEqual(ts.get(DateTime.MJD, DateTime.UTC), 54392.040196759262)
def testIsoBasic(self):
ts = DateTime("20090402T072639.314159265Z")
self.assertEqual(ts.nsecs(DateTime.TAI), 1238657233314159265L)
self.assertEqual(ts.nsecs(DateTime.UTC), 1238657199314159265L)
self.assertEqual(ts.toString(), "2009-04-02T07:26:39.314159265Z")
def testIsoEpoch(self):
ts = DateTime("19700101T000000Z", DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 0)
self.assertEqual(ts.toString(ts.UTC), "1970-01-01T00:00:00.000000000Z")
def testPolicies(self):
ps = orcaProv.Provenance(self.user, self.runId, self.dbLoc,
self.globalDbLoc)
paths = ("tests/policy/dc2pipe.paf",
"tests/policy/imageSubtractionDetection.paf")
for p in paths:
ps.recordPolicy(p)
db = DbStorage()
db.setRetrieveLocation(LogicalLocation(self.dbLoc))
for p in paths:
md5 = hashlib.md5()
f = open(p, 'r')
for line in f:
md5.update(line)
f.close()
hash = md5.hexdigest()
mod = DateTime(os.stat(p)[8] * 1000000000L, DateTime.UTC)
pol = pexPolicy.Policy(p)
names = pol.paramNames()
db.startTransaction()
db.setTableListForQuery(
("prv_PolicyFile", "prv_PolicyKey", "prv_cnf_PolicyKey"))
db.outColumn("hashValue")
db.outColumn("modifiedDate")
db.outColumn("keyName")
db.outColumn("keyType")
db.outColumn("value")
db.setQueryWhere(
"""pathname = '%s'
AND prv_PolicyFile.policyFileId = prv_PolicyKey.policyFileId
AND prv_PolicyKey.policyKeyId = prv_cnf_PolicyKey.policyKeyId
""" % (p))
db.query()
while db.next():
self.assert_(not db.columnIsNull(0))
self.assert_(not db.columnIsNull(1))
self.assert_(not db.columnIsNull(2))
self.assert_(not db.columnIsNull(3))
self.assert_(not db.columnIsNull(4))
self.assertEqual(db.getColumnByPosString(0), hash)
self.assertEqual(db.getColumnByPosInt64(1), mod.nsecs())
key = db.getColumnByPosString(2)
self.assert_(pol.exists(key))
self.assert_(key in names)
names.remove(key)
self.assertEqual(db.getColumnByPosString(3),
pol.getTypeName(key))
correct = pol.str(key)
correct = re.sub(r'\0', r'', correct)
self.assertEqual(db.getColumnByPosString(4), correct)
db.finishQuery()
db.endTransaction()
self.assertEqual(len(names), 0)
def testIsoNoNSecs(self):
ts = DateTime("2009-04-02T07:26:39Z", DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.TAI), 1238657233000000000)
self.assertEqual(ts.nsecs(DateTime.UTC), 1238657199000000000)
self.assertEqual(ts.toString(ts.UTC), "2009-04-02T07:26:39.000000000Z")
self.assertTrue(ts.isValid())
def testIsoExpanded(self):
ts = DateTime("2009-04-02T07:26:39.314159265Z", DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.TAI), 1238657233314159265)
self.assertEqual(ts.nsecs(DateTime.UTC), 1238657199314159265)
self.assertEqual(ts.toString(ts.UTC), "2009-04-02T07:26:39.314159265Z")
self.assertTrue(ts.isValid())
def testIsoThrow(self):
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04-01T23:36:05", DateTime.UTC) # Z time zone required for UTC
for scale in (DateTime.TAI, DateTime.TT):
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04-01T23:36:05Z", scale) # Z time zone forbidden for TAI or TT
for scale in self.timeScales:
with self.assertRaises(pexExcept.DomainError):
DateTime("20090401", scale) # time required
with self.assertRaises(pexExcept.DomainError):
DateTime("20090401T", DateTime.UTC) # time required
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04-01T", DateTime.UTC) # time required
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04-01T23:36:05-0700", DateTime.UTC) # time zone offset not supported
with self.assertRaises(pexExcept.DomainError):
DateTime("2009/04/01T23:36:05Z", DateTime.UTC) # "/" not valid
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04-01T23:36", DateTime.UTC) # partial time
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04", DateTime.UTC) # partial date without time
with self.assertRaises(pexExcept.DomainError):
DateTime("2009-04T23:36.05", DateTime.UTC) # partial date with time
with self.assertRaises(pexExcept.DomainError):
DateTime("09-04-01T23:36:05", DateTime.UTC) # 2 digit year
# earliest allowed UTC date is the earliest date in the leap second table
try:
minLeapSecUTC = "1961-01-01T00:00:00Z"
dt = DateTime(minLeapSecUTC, DateTime.UTC)
dt.toString(DateTime.UTC)
except Exception:
self.fail("minLeapSecUTC={} failed, but should be OK".format(minLeapSecUTC))
with self.assertRaises(pexExcept.DomainError):
DateTime("1960-01-01T23:59:59Z", DateTime.UTC) # just before leap second table starts
# earliest allowed date for TAI and TT is year = 1902
for timeSys in (DateTime.TAI, DateTime.TT):
try:
earliestDate = "1902-01-01T00:00:00"
dt = DateTime(earliestDate, timeSys)
dt.toString(DateTime.TAI)
dt.toString(DateTime.TT)
except Exception:
self.fail("{} system={} failed, but should be OK".format(earliestDate, timeSys))
# dates before the leap second table can be created using TAI or TT, but not viewed in UTC
earlyDt = DateTime("1960-01-01T00:00:00", DateTime.TAI)
with self.assertRaises(pexExcept.DomainError):
earlyDt.toString(DateTime.UTC)
with self.assertRaises(pexExcept.DomainError):
DateTime("1901-12-12T23:59:59Z", DateTime.TAI) # too early
with self.assertRaises(pexExcept.DomainError):
DateTime("1700-01-01T00:00:00Z", DateTime.TAI) # way too early
with self.assertRaises(pexExcept.DomainError):
DateTime("2262-01-01T00:00:00Z", DateTime.TAI) # too late
with self.assertRaises(pexExcept.DomainError):
DateTime("3200-01-01T00:00:00Z", DateTime.TAI) # way too late
def testInvalid(self):
ts = DateTime()
self.assertFalse(ts.isValid())
for scale in self.timeScales:
self.assertEqual(ts.nsecs(scale), DateTime.invalid_nsecs)
for system in self.dateSystems:
with self.assertRaises(RuntimeError):
ts.get(system, scale)
with self.assertRaises(RuntimeError):
ts.gmtime(scale)
with self.assertRaises(RuntimeError):
ts.timespec(scale)
with self.assertRaises(RuntimeError):
ts.timeval(scale)
with self.assertRaises(RuntimeError):
ts.toString(scale)
self.assertEqual(repr(ts), "DateTime()")
def testCrossBoundaryNsecs(self):
ts = DateTime(631151998000000000, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 631151998000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 631152022000000000)
def testBoundaryMJD(self):
ts = DateTime(47892.0, DateTime.MJD, DateTime.UTC)
self.assertEqual(ts.nsecs(DateTime.UTC), 631152000000000000)
self.assertEqual(ts.nsecs(DateTime.TAI), 631152025000000000)
self.assertEqual(ts.get(DateTime.MJD, DateTime.UTC), 47892.0)
def testNegative(self):
ts = DateTime("1969-03-01T00:00:32Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-03-01T00:00:32.000000000Z')
ts = DateTime("1969-01-01T00:00:00Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-01-01T00:00:00.000000000Z')
ts = DateTime("1969-01-01T00:00:40Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-01-01T00:00:40.000000000Z')
ts = DateTime("1969-01-01T00:00:38Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-01-01T00:00:38.000000000Z')
ts = DateTime("1969-03-01T12:39:45Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-03-01T12:39:45.000000000Z')
ts = DateTime("1969-03-01T12:39:45.000000001Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-03-01T12:39:45.000000001Z')
self.assertTrue(ts.isValid())
# Note slight inaccuracy in UTC-TAI-UTC round-trip
ts = DateTime("1969-03-01T12:39:45.12345Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-03-01T12:39:45.123449996Z')
ts = DateTime("1969-03-01T12:39:45.123456Z", DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-03-01T12:39:45.123455996Z')
ts = DateTime(-1, DateTime.TAI)
self.assertEqual(ts.toString(ts.UTC), '1969-12-31T23:59:51.999918239Z')
ts = DateTime(0, DateTime.TAI)
self.assertEqual(ts.toString(ts.UTC), '1969-12-31T23:59:51.999918240Z')
ts = DateTime(1, DateTime.TAI)
self.assertEqual(ts.toString(ts.UTC), '1969-12-31T23:59:51.999918241Z')
ts = DateTime(-1, DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1969-12-31T23:59:59.999999999Z')
ts = DateTime(0, DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1970-01-01T00:00:00.000000000Z')
ts = DateTime(1, DateTime.UTC)
self.assertEqual(ts.toString(ts.UTC), '1970-01-01T00:00:00.000000001Z')
