def testArrayValues(self):
"""Check that only the final value is used from an array
"""
metadata = PropertyList()
# work around DM-13232 by setting ABOOL one value at a time
for value in [True, True, True, False]:
metadata.add("ABOOL", value)
dataList = [
("AFLOAT", [1.2e25, -5.6]),
("ANINT", [-5, 752, 1052]),
("ASTRING1", ["value for string", "more"]),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = F",
"AFLOAT = -5.6",
"ANINT = 1052",
"ASTRING1= 'more'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def testReadOldTanFits(self):
"""Test reading a FITS file containing data for an lsst::afw::image::TanWcs
That file was made using the same metadata follows
(like self.metadata without the distortion)
"""
tanMetadata = PropertyList()
# the following was fit using CreateWcsWithSip from meas_astrom
# and is valid over this bbox: (minimum=(0, 0), maximum=(3030, 3030))
# This same metadata was used to create testdata/oldTanSipwWs.fits
for name, value in (
("RADESYS", "ICRS"),
("CTYPE1", "RA---TAN"),
("CTYPE2", "DEC--TAN"),
("CRPIX1", 1531.1824767147),
("CRPIX2", 1531.1824767147),
("CRVAL1", 43.035511801383),
("CRVAL2", 44.305697682784),
("CUNIT1", "deg"),
("CUNIT2", "deg"),
("CD1_1", 0.00027493991598151),
("CD1_2", -3.2758487104158e-06),
("CD2_1", 3.2301310675830e-06),
("CD2_2", 0.00027493937506632),
):
tanMetadata.set(name, value)
dataDir = os.path.join(os.path.split(__file__)[0], "data")
filePath = os.path.join(dataDir, "oldTanWcs.fits")
wcsFromFits = SkyWcs.readFits(filePath)
wcsFromMetadata = makeSkyWcs(tanMetadata)
bbox = Box2D(Point2D(-1000, -1000), Extent2D(3000, 3000))
self.assertWcsAlmostEqualOverBBox(wcsFromFits, wcsFromMetadata, bbox)
def testBasics(self):
"""Check basic formatting and skipping bad values
"""
metadata = PropertyList()
dataList = [
("ABOOL", True),
("AFLOAT", 1.2e25),
("ANINT", -5),
("LONGNAME1", 1), # name is longer than 8 characters; skip it
("LONGSTR", "skip this item because the formatted value "
"is too long: longer than 80 characters "),
("ASTRING1", "value for string"),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = 1",
"AFLOAT = 1.2E+25",
"ANINT = -5",
"ASTRING1= 'value for string'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def testArrayValues(self):
"""Check that only the final value is used from an array
"""
metadata = PropertyList()
# work around DM-13232 by setting ABOOL one value at a time
for value in [True, True, True, False]:
metadata.add("ABOOL", value)
dataList = [
("AFLOAT", [1.2e25, -5.6]),
("ANINT", [-5, 752, 1052]),
("ASTRING1", ["value for string", "more"]),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = F",
"AFLOAT = -5.6",
"ANINT = 1052",
"ASTRING1= 'more'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def testCD_PC(self):
"""Test that we can read a FITS file with both CD and PC keys (like early Suprimecam files)"""
md = PropertyList()
for k, v in (
("EQUINOX", 2000.0),
("RADESYS", "ICRS"),
("CRPIX1", 5353.0),
("CRPIX2", -35.0),
("CD1_1", 0.0),
("CD1_2", -5.611E-05),
("CD2_1", -5.611E-05),
("CD2_2", -0.0),
("CRVAL1", 4.5789875),
("CRVAL2", 16.30004444),
("CUNIT1", "deg"),
("CUNIT2", "deg"),
("CTYPE1", "RA---TAN"),
("CTYPE2", "DEC--TAN"),
("CDELT1", -5.611E-05),
("CDELT2", 5.611E-05),
):
md.set(k, v)
wcs = makeSkyWcs(md, strip=False)
pixPos = Point2D(1000, 2000)
pred_skyPos = SpherePoint(4.459815023498577 * degrees,
16.544199850984768 * degrees)
skyPos = wcs.pixelToSky(pixPos)
self.assertSpherePointsAlmostEqual(skyPos, pred_skyPos)
for badPC in (False, True):
for k, v in (
("PC001001", 0.0),
("PC001002", -1.0 if badPC else 1.0),
("PC002001", 1.0 if badPC else -1.0),
("PC002002", 0.0),
):
md.set(k, v)
# Check Greisen and Calabretta A&A 395 1061 (2002), Eq. 3
if not badPC:
for i in (
1,
2,
):
for j in (
1,
2,
):
self.assertEqual(
md.get("CD%d_%d" % (i, j)),
md.get("CDELT%d" % i) * md.get("PC00%d00%d" %
(i, j)))
wcs2 = makeSkyWcs(md, strip=False)
skyPos2 = wcs2.pixelToSky(pixPos)
self.assertSpherePointsAlmostEqual(skyPos2, pred_skyPos)
def testDecoratedImage(self):
image = self.maskedImage.getImage()
decoImage = afwImage.DecoratedImageF(image)
metadata = PropertyList()
metadata.set("FOO", "BAR")
decoImage.setMetadata(metadata)
exposure = exposureFromImage(decoImage)
self.assertImagesEqual(exposure.getMaskedImage().getImage(), image)
md = exposure.getMetadata()
self.assertEqual(md.get("FOO"), "BAR")
def testStripVisitInfoKeywords(self):
for argList in (self.data1, self.data2):
visitInfo = afwImage.VisitInfo(*argList)
metadata = PropertyList()
afwImage.setVisitInfoMetadata(metadata, visitInfo)
# add an extra keyword that will not be stripped
metadata.set("EXTRA", 5)
self.assertEqual(metadata.nameCount(), 21)
afwImage.stripVisitInfoKeywords(metadata)
self.assertEqual(metadata.nameCount(), 1)
def testDecoratedImageBadWcs(self):
"""Test that exposureFromImage() attaches a None wcs to the exposure
when the WCS cannot be constructed
"""
image = self.maskedImage.getImage()
decoImage = afwImage.DecoratedImageF(image)
metadata = PropertyList()
metadata.set("CTYPE1", "RA---TPV")
metadata.set("CTYPE2", "DEC--TPV")
decoImage.setMetadata(metadata)
exposure = exposureFromImage(decoImage)
self.assertIs(exposure.getWcs(), None)
def make_dm_wcs(galsim_wcs):
"""
convert galsim wcs to stack wcs
Parameters
----------
galsim_wcs: galsim WCS
Should be TAN or TAN-SIP
Returns
-------
DM Stack sky wcs
"""
if galsim_wcs.wcs_type == 'TAN':
crpix = galsim_wcs.crpix
# DM uses 0 offset, galsim uses FITS 1 offset
stack_crpix = Point2D(crpix[0] - 1, crpix[1] - 1)
cd_matrix = galsim_wcs.cd
crval = geom.SpherePoint(
galsim_wcs.center.ra / coord.radians,
galsim_wcs.center.dec / coord.radians,
geom.radians,
)
stack_wcs = makeSkyWcs(
crpix=stack_crpix,
crval=crval,
cdMatrix=cd_matrix,
)
elif galsim_wcs.wcs_type == 'TAN-SIP':
# No currently supported
# this works with the 1-offset assumption from galsim
#
# this is not used if the lower bounds are 1, but the extra keywords
# GS_{X,Y}MIN are set which we will remove below
fake_bounds = galsim.BoundsI(1, 10, 1, 10)
hdr = {}
galsim_wcs.writeToFitsHeader(hdr, fake_bounds)
del hdr["GS_XMIN"]
del hdr["GS_YMIN"]
metadata = PropertyList()
for key, value in hdr.items():
metadata.set(key, value)
stack_wcs = makeSkyWcs(metadata)
return stack_wcs
def testDecoratedImageBadWcs(self):
"""Test that exposureFromImage() attaches a None wcs to the exposure
when makeWcs() raises an invalidParameter error
"""
image = self.maskedImage.getImage()
decoImage = afwImage.DecoratedImageF(image)
metadata = PropertyList()
metadata.set("CTYPE1", "RA---TPV")
metadata.set("CTYPE2", "DEC--TPV")
decoImage.setMetadata(metadata)
exposure = exposureFromImage(decoImage)
self.assertIs(exposure.getWcs(), None)
def addRefCatMetadata(catalog):
"""Add metadata to a new (not yet populated) reference catalog.
Parameters
----------
catalog : `lsst.afw.table.SimpleCatalog`
Catalog to which metadata should be attached. Will be modified
in-place.
"""
md = catalog.getMetadata()
if md is None:
md = PropertyList()
md.set("REFCAT_FORMAT_VERSION", LATEST_FORMAT_VERSION)
catalog.setMetadata(md)
def addRefCatMetadata(catalog):
"""Add metadata to a new (not yet populated) reference catalog.
Parameters
----------
catalog : `lsst.afw.table.SimpleCatalog`
Catalog to which metadata should be attached. Will be modified
in-place.
"""
md = catalog.getMetadata()
if md is None:
md = PropertyList()
md.set("REFCAT_FORMAT_VERSION", LATEST_FORMAT_VERSION)
catalog.setMetadata(md)
def testGetSipMatrixFromMetadata(self):
"""Test getSipMatrixFromMetadata and makeSipMatrixMetadata
"""
for badName in ("X", "AA"):
self.assertFalse(hasSipMatrix(self.metadata, badName))
with self.assertRaises(TypeError):
getSipMatrixFromMetadata(self.metadata, badName)
for name in ("A", "B", "AP", "BP"):
self.assertTrue(hasSipMatrix(self.metadata, name))
sipMatrix = getSipMatrixFromMetadata(self.metadata, name)
width = self.metadata.getScalar("%s_ORDER" % (name, )) + 1
self.assertEqual(sipMatrix.shape, (width, width))
for i in range(width):
for j in range(width):
# SIP matrix terms use 0-based indexing
cardName = "%s_%d_%d" % (name, i, j)
if self.metadata.exists(cardName):
self.assertEqual(sipMatrix[i, j],
self.metadata.getScalar(cardName))
else:
self.assertEqual(sipMatrix[i, j], 0.0)
metadata = makeSipMatrixMetadata(sipMatrix, name)
for name in metadata.names(False):
value = metadata.getScalar(name)
if (name.endswith("ORDER")):
self.assertEqual(width, value + 1)
else:
self.assertEqual(value, self.metadata.getScalar(name))
self.assertNotEqual(value, 0.0) # 0s are omitted
# try metadata with only the ORDER keyword; the matrix should be all zeros
# except for the invalid case of order < 0
for order in (-3, -1, 0, 3):
metadata2 = PropertyList()
metadata2.set("W_ORDER", order)
if order < 0:
# invalid order
self.assertFalse(hasSipMatrix(metadata2, "W"))
with self.assertRaises(TypeError):
getSipMatrixFromMetadata(metadata2, "W")
else:
self.assertTrue(hasSipMatrix(metadata2, "W"))
zeroMatrix = getSipMatrixFromMetadata(metadata2, "W")
self.assertEqual(zeroMatrix.shape, (order + 1, order + 1))
for i in range(order + 1):
for j in range(order + 1):
self.assertEqual(zeroMatrix[i, j], 0.0)
def testGetSipMatrixFromMetadata(self):
"""Test getSipMatrixFromMetadata and makeSipMatrixMetadata
"""
for badName in ("X", "AA"):
self.assertFalse(hasSipMatrix(self.metadata, badName))
with self.assertRaises(TypeError):
getSipMatrixFromMetadata(self.metadata, badName)
for name in ("A", "B", "AP", "BP"):
self.assertTrue(hasSipMatrix(self.metadata, name))
sipMatrix = getSipMatrixFromMetadata(self.metadata, name)
width = self.metadata.getScalar("%s_ORDER" % (name,)) + 1
self.assertEqual(sipMatrix.shape, (width, width))
for i in range(width):
for j in range(width):
# SIP matrix terms use 0-based indexing
cardName = "%s_%d_%d" % (name, i, j)
if self.metadata.exists(cardName):
self.assertEqual(sipMatrix[i, j], self.metadata.getScalar(cardName))
else:
self.assertEqual(sipMatrix[i, j], 0.0)
metadata = makeSipMatrixMetadata(sipMatrix, name)
for name in metadata.names(False):
value = metadata.getScalar(name)
if (name.endswith("ORDER")):
self.assertEqual(width, value + 1)
else:
self.assertEqual(value, self.metadata.getScalar(name))
self.assertNotEqual(value, 0.0) # 0s are omitted
# try metadata with only the ORDER keyword; the matrix should be all zeros
# except for the invalid case of order < 0
for order in (-3, -1, 0, 3):
metadata2 = PropertyList()
metadata2.set("W_ORDER", order)
if order < 0:
# invalid order
self.assertFalse(hasSipMatrix(metadata2, "W"))
with self.assertRaises(TypeError):
getSipMatrixFromMetadata(metadata2, "W")
else:
self.assertTrue(hasSipMatrix(metadata2, "W"))
zeroMatrix = getSipMatrixFromMetadata(metadata2, "W")
self.assertEqual(zeroMatrix.shape, (order + 1, order + 1))
for i in range(order + 1):
for j in range(order + 1):
self.assertEqual(zeroMatrix[i, j], 0.0)
def getTanWcsMetata():
metadata = PropertyList()
for name, value in (
("RADESYS", "ICRS"),
("CTYPE1", "RA---TAN"),
("CTYPE2", "DEC--TAN"),
("CRPIX1", 1531.1824767147),
("CRPIX2", 1531.1824767147),
("CRVAL1", 43.035511801383),
("CRVAL2", 44.305697682784),
("CUNIT1", "deg"),
("CUNIT2", "deg"),
("CD1_1", 0.00027493991598151),
("CD1_2", -3.2758487104158e-06),
("CD2_1", 3.2301310675830e-06),
("CD2_2", 0.00027493937506632),
):
metadata.set(name, value)
return metadata
def setUp(self):
metadata = PropertyList()
for name, value in (
("RADESYS", "ICRS"),
("EQUINOX", 2000.),
("CRVAL1", 215.604025685476),
("CRVAL2", 53.1595451514076),
("CRPIX1", 1109.99981456774),
("CRPIX2", 560.018167811613),
("CTYPE1", "RA---TAN"),
("CTYPE2", "DEC--TAN"),
("CUNIT1", "deg"),
("CUNIT2", "deg"),
("CD1_1", 5.10808596133527E-05),
("CD1_2", 1.85579539217196E-07),
("CD2_2", -5.10281493481982E-05),
("CD2_1", -1.85579539217196E-07),
):
metadata.set(name, value)
self.metadata = metadata
def testBasics(self):
"""Check basic formatting and skipping bad values
"""
metadata = PropertyList()
dataList = [
("ABOOL", True),
("AFLOAT", 1.2e25),
("AFLOAT2", 1.0e30),
("ANINT", -5),
("AFLOATZ", 0.0), # ensure a float stays a float
("INTFLOAT", -5.0),
("LONGFLT", 0.0089626337538440005),
("ANUNDEF", None),
("LONGNAME1", 1), # name is longer than 8 characters; skip it
("LONGSTR", "skip this item because the formatted value "
"is too long: longer than 80 characters "),
("ASTRING1", "value for string"),
("ANAN", float("NaN"))
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = T",
"AFLOAT = 1.2E+25",
"AFLOAT2 = 1E+30",
"ANINT = -5",
"AFLOATZ = 0.0",
"INTFLOAT= -5.0",
"LONGFLT = 0.0089626337538440005",
"ANUNDEF =",
"ASTRING1= 'value for string'",
"ANAN =",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def make_stack_wcs(wcs):
"""
convert galsim tan wcs to stack wcs
"""
if wcs.wcs_type == 'TAN':
crpix = wcs.crpix
stack_crpix = geom.Point2D(crpix[0], crpix[1])
cd_matrix = wcs.cd
crval = geom.SpherePoint(
wcs.center.ra / coord.radians,
wcs.center.dec / coord.radians,
geom.radians,
)
stack_wcs = makeSkyWcs(
crpix=stack_crpix,
crval=crval,
cdMatrix=cd_matrix,
)
elif wcs.wcs_type == 'TAN-SIP':
import galsim
# this is not used if the lower bounds are 1, but the extra keywords
# GS_{X,Y}MIN are set which we will remove below
fake_bounds = galsim.BoundsI(1, 10, 1, 10)
hdr = {}
wcs.writeToFitsHeader(hdr, fake_bounds)
del hdr["GS_XMIN"]
del hdr["GS_YMIN"]
metadata = PropertyList()
for key, value in hdr.items():
metadata.set(key, value)
stack_wcs = makeSkyWcs(metadata)
return stack_wcs
def testBasics(self):
"""Check basic formatting and skipping bad values
"""
metadata = PropertyList()
dataList = [
("ABOOL", True),
("AFLOAT", 1.2e25),
("AFLOAT2", 1.0e30),
("ANINT", -5),
("AFLOATZ", 0.0), # ensure a float stays a float
("INTFLOAT", -5.0),
("LONGFLT", 0.0089626337538440005),
("ANUNDEF", None),
("LONGNAME1", 1), # name is longer than 8 characters; skip it
("LONGSTR", "skip this item because the formatted value "
"is too long: longer than 80 characters "),
("ASTRING1", "value for string"),
]
for name, value in dataList:
metadata.set(name, value)
header = makeLimitedFitsHeader(metadata)
expectedLines = [ # without padding to 80 chars
"ABOOL = T",
"AFLOAT = 1.2E+25",
"AFLOAT2 = 1E+30",
"ANINT = -5",
"AFLOATZ = 0.0",
"INTFLOAT= -5.0",
"LONGFLT = 0.0089626337538440005",
"ANUNDEF =",
"ASTRING1= 'value for string'",
]
expectedHeader = "".join("%-80s" % val for val in expectedLines)
self.assertHeadersEqual(header, expectedHeader)
self.checkExcludeNames(metadata, expectedLines)
def makeStdCat(stdSchema, stdStruct, goodBands):
"""
Make the standard star catalog for persistence
Parameters
----------
stdSchema: `lsst.afw.table.Schema`
Standard star catalog schema
stdStruct: `numpy.ndarray`
Standard star structure in FGCM format
goodBands: `list`
List of good band names used in stdStruct
Returns
-------
stdCat: `lsst.afw.table.BaseCatalog`
Standard star catalog for persistence
"""
stdCat = afwTable.SimpleCatalog(stdSchema)
stdCat.resize(stdStruct.size)
stdCat['id'][:] = stdStruct['FGCM_ID']
stdCat['coord_ra'][:] = stdStruct['RA'] * geom.degrees
stdCat['coord_dec'][:] = stdStruct['DEC'] * geom.degrees
stdCat['ngood'][:, :] = stdStruct['NGOOD'][:, :]
stdCat['ntotal'][:, :] = stdStruct['NTOTAL'][:, :]
stdCat['mag_std_noabs'][:, :] = stdStruct['MAG_STD'][:, :]
stdCat['magErr_std'][:, :] = stdStruct['MAGERR_STD'][:, :]
stdCat['npsfcand'][:, :] = stdStruct['NPSFCAND'][:, :]
md = PropertyList()
md.set("BANDS", list(goodBands))
stdCat.setMetadata(md)
return stdCat
def testIgnoreInvalid(self):
"""Test that invalid items in the either argument are ignored
"""
md1 = PropertyList()
# Set COMMENT and HISTORY to invalid values -- anything other than string
# (regardless if it is a scalar or an array);
# for md1 use arrays and md2 use scalars, just to try both
md1.set("COMMENT", [5, 6])
md1.set("HISTORY", [3.5, 6.1])
md1Copy = md1.deepCopy()
md2 = PropertyList()
# Set COMMENT and HISTORY to invalid values; see comment above md1.set("COMMENT", ...)
md2.set("COMMENT", 7)
md2.set("HISTORY", 1.06)
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
resultNames = result.getOrderedNames()
self.assertEqual(resultNames, [])
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def testIgnoreInvalid(self):
"""Test that invalid items in the either argument are ignored
"""
md1 = PropertyList()
# Set COMMENT and HISTORY to invalid values -- anything other than string
# (regardless if it is a scalar or an array);
# for md1 use arrays and md2 use scalars, just to try both
md1.set("COMMENT", [5, 6])
md1.set("HISTORY", [3.5, 6.1])
md1Copy = md1.deepCopy()
md2 = PropertyList()
# Set COMMENT and HISTORY to invalid values; see comment above md1.set("COMMENT", ...)
md2.set("COMMENT", 7)
md2.set("HISTORY", 1.06)
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
resultNames = result.getOrderedNames()
self.assertEqual(resultNames, [])
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def testUndefinedVector(self):
header = PropertyList()
header.set("FOO", [None, None])
metadata = self.writeAndRead(header)
self.assertEqual(metadata.getArray("FOO"), [None, None])
def testReplaceDuplicates(self):
"""Test that names in `second` override those in `first`, regardless of type
"""
# names that start with "item" appear in both sets of metadata
md1 = PropertyList()
md1.set("int1", 5)
md1.set("itema", [1, 2])
md1.set("float1", 3.1)
md1.set("itemb", 1.23)
md1.set("string1", "md1 string1 value")
md1.set("itemc", "md1 string value")
md1Copy = md1.deepCopy()
md2 = PropertyList()
md2.set("itemc", 2)
md2.set("int2", 2)
md2.set("itemb", ["some data", "more data"])
md2.set("float2", 2.34)
md2.set("itema", 5.27)
md2.set("string2", "md2 string value")
md2Names = md2.getOrderedNames()
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
expectedNames = ["int1", "float1", "string1"] + list(md2Names)
self.assertEqual(result.getOrderedNames(), expectedNames)
md2NameSet = set(md2Names)
for name in result.getOrderedNames():
if name in md2NameSet:
self.assertEqual(result.getScalar(name), md2.getArray(name)[-1])
else:
self.assertEqual(result.getScalar(name), md1.getArray(name)[-1])
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def testNoConflicts(self):
"""Test combination with valid values and no overlap,
except COMMENT and HISTORY, which are combined
"""
md1 = PropertyList()
md1.set("int1", [1, 2])
md1.set("float1", 1.23)
md1.set("string1", "md1 string1 value")
md1.set("COMMENT", "md1 comment")
md1.set("HISTORY", "md1 history")
md1Copy = md1.deepCopy()
md2 = PropertyList()
md2.set("int2", 2)
md2.set("float2", [2.34, -3.45])
md2.set("string2", "md2 string2 value")
md2.set("COMMENT", "md2 comment")
md2.set("HISTORY", "md2 history")
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
self.assertEqual(result.getOrderedNames(),
["int1", "float1", "string1", "COMMENT", "HISTORY",
"int2", "float2", "string2"])
self.assertEqual(result.getArray("COMMENT"), ["md1 comment", "md2 comment"])
self.assertEqual(result.getArray("HISTORY"), ["md1 history", "md2 history"])
for name in md1.getOrderedNames():
if name in ("COMMENT", "HISTORY"):
continue
self.assertEqual(result.getScalar(name), md1.getArray(name)[-1])
for name in md2.getOrderedNames():
if name in ("COMMENT", "HISTORY"):
continue
self.assertEqual(result.getScalar(name), md2.getArray(name)[-1])
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def setUp(self):
# Actual WCS from processing Suprime-Cam
self.width = 2048
self.height = 4177
metadata = PropertyList()
for name, value in (
('NAXIS', 2),
('EQUINOX', 2000.0000000000),
('RADESYS', "ICRS"),
('CRPIX1', -3232.7544925483),
('CRPIX2', 4184.4881091129),
('CD1_1', -5.6123808607273e-05),
('CD1_2', 2.8951544956703e-07),
('CD2_1', 2.7343044348306e-07),
('CD2_2', 5.6100888336445e-05),
('CRVAL1', 5.6066137655191),
('CRVAL2', -0.60804032498548),
('CUNIT1', "deg"),
('CUNIT2', "deg"),
('A_ORDER', 5),
('A_0_2', 1.9749832126246e-08),
('A_0_3', 9.3734869173527e-12),
('A_0_4', 1.8812994578840e-17),
('A_0_5', -2.3524013652433e-19),
('A_1_1', -9.8443908806559e-10),
('A_1_2', -4.9278297504858e-10),
('A_1_3', -2.8491604610001e-16),
('A_1_4', 2.3185723720750e-18),
('A_2_0', 4.9546089730708e-08),
('A_2_1', -8.8592221672777e-12),
('A_2_2', 3.3560100338765e-16),
('A_2_3', 3.0469486185035e-21),
('A_3_0', -4.9332471706700e-10),
('A_3_1', -5.3126029725748e-16),
('A_3_2', 4.7795824885726e-18),
('A_4_0', 1.3128844828963e-16),
('A_4_1', 4.4014452170715e-19),
('A_5_0', 2.1781986904162e-18),
('B_ORDER', 5),
('B_0_2', -1.0607653075899e-08),
('B_0_3', -4.8693887937365e-10),
('B_0_4', -1.0363305097301e-15),
('B_0_5', 1.9621640066919e-18),
('B_1_1', 3.0340657679481e-08),
('B_1_2', -5.0763819284853e-12),
('B_1_3', 2.8987281654754e-16),
('B_1_4', 1.8253389678593e-19),
('B_2_0', -2.4772849184248e-08),
('B_2_1', -4.9775588352207e-10),
('B_2_2', -3.6806326254887e-16),
('B_2_3', 4.4136985315418e-18),
('B_3_0', -1.7807191001742e-11),
('B_3_1', -2.4136396882531e-16),
('B_3_2', 2.9165413645768e-19),
('B_4_0', 4.1029951148438e-16),
('B_4_1', 2.3711874424169e-18),
('B_5_0', 4.9333635889310e-19),
('AP_ORDER', 5),
('AP_0_1', -5.9740855298291e-06),
('AP_0_2', -2.0433429597268e-08),
('AP_0_3', -8.6810071023434e-12),
('AP_0_4', -2.4974690826778e-17),
('AP_0_5', 1.9819631102516e-19),
('AP_1_0', -4.5896648256716e-05),
('AP_1_1', -1.5248993348644e-09),
('AP_1_2', 5.0283116166943e-10),
('AP_1_3', 4.3796281513144e-16),
('AP_1_4', -2.1447889127908e-18),
('AP_2_0', -4.7550300344365e-08),
('AP_2_1', 1.0924172283232e-11),
('AP_2_2', -4.9862026098260e-16),
('AP_2_3', -5.4470851768869e-20),
('AP_3_0', 5.0130654116966e-10),
('AP_3_1', 6.8649554020012e-16),
('AP_3_2', -4.2759588436342e-18),
('AP_4_0', -3.6306802581471e-16),
('AP_4_1', -5.3885285875084e-19),
('AP_5_0', -1.8802693525108e-18),
('BP_ORDER', 5),
('BP_0_1', -2.6627855995942e-05),
('BP_0_2', 1.1143451873584e-08),
('BP_0_3', 4.9323396530135e-10),
('BP_0_4', 1.1785185735421e-15),
('BP_0_5', -1.6169957016415e-18),
('BP_1_0', -5.7914490267576e-06),
('BP_1_1', -3.0565765766244e-08),
('BP_1_2', 5.7727475030971e-12),
('BP_1_3', -4.0586821113726e-16),
('BP_1_4', -2.0662723654322e-19),
('BP_2_0', 2.3705520015164e-08),
('BP_2_1', 5.0530823594352e-10),
('BP_2_2', 3.8904979943489e-16),
('BP_2_3', -3.8346209540986e-18),
('BP_3_0', 1.9505421473262e-11),
('BP_3_1', 1.7583146713289e-16),
('BP_3_2', -3.4876779564534e-19),
('BP_4_0', -3.3690937119054e-16),
('BP_4_1', -2.0853007589561e-18),
('BP_5_0', -5.5344298912288e-19),
('CTYPE1', "RA---TAN-SIP"),
('CTYPE2', "DEC--TAN-SIP"),
):
metadata.set(name, value)
self.metadata = metadata
def setUp(self):
# Actual WCS from processing Suprime-Cam
self.width = 2048
self.height = 4177
metadata = PropertyList()
for name, value in (
('NAXIS', 2),
('EQUINOX', 2000.0000000000),
('RADESYS', "ICRS"),
('CRPIX1', -3232.7544925483),
('CRPIX2', 4184.4881091129),
('CD1_1', -5.6123808607273e-05),
('CD1_2', 2.8951544956703e-07),
('CD2_1', 2.7343044348306e-07),
('CD2_2', 5.6100888336445e-05),
('CRVAL1', 5.6066137655191),
('CRVAL2', -0.60804032498548),
('CUNIT1', "deg"),
('CUNIT2', "deg"),
('A_ORDER', 5),
('A_0_2', 1.9749832126246e-08),
('A_0_3', 9.3734869173527e-12),
('A_0_4', 1.8812994578840e-17),
('A_0_5', -2.3524013652433e-19),
('A_1_1', -9.8443908806559e-10),
('A_1_2', -4.9278297504858e-10),
('A_1_3', -2.8491604610001e-16),
('A_1_4', 2.3185723720750e-18),
('A_2_0', 4.9546089730708e-08),
('A_2_1', -8.8592221672777e-12),
('A_2_2', 3.3560100338765e-16),
('A_2_3', 3.0469486185035e-21),
('A_3_0', -4.9332471706700e-10),
('A_3_1', -5.3126029725748e-16),
('A_3_2', 4.7795824885726e-18),
('A_4_0', 1.3128844828963e-16),
('A_4_1', 4.4014452170715e-19),
('A_5_0', 2.1781986904162e-18),
('B_ORDER', 5),
('B_0_2', -1.0607653075899e-08),
('B_0_3', -4.8693887937365e-10),
('B_0_4', -1.0363305097301e-15),
('B_0_5', 1.9621640066919e-18),
('B_1_1', 3.0340657679481e-08),
('B_1_2', -5.0763819284853e-12),
('B_1_3', 2.8987281654754e-16),
('B_1_4', 1.8253389678593e-19),
('B_2_0', -2.4772849184248e-08),
('B_2_1', -4.9775588352207e-10),
('B_2_2', -3.6806326254887e-16),
('B_2_3', 4.4136985315418e-18),
('B_3_0', -1.7807191001742e-11),
('B_3_1', -2.4136396882531e-16),
('B_3_2', 2.9165413645768e-19),
('B_4_0', 4.1029951148438e-16),
('B_4_1', 2.3711874424169e-18),
('B_5_0', 4.9333635889310e-19),
('AP_ORDER', 5),
('AP_0_1', -5.9740855298291e-06),
('AP_0_2', -2.0433429597268e-08),
('AP_0_3', -8.6810071023434e-12),
('AP_0_4', -2.4974690826778e-17),
('AP_0_5', 1.9819631102516e-19),
('AP_1_0', -4.5896648256716e-05),
('AP_1_1', -1.5248993348644e-09),
('AP_1_2', 5.0283116166943e-10),
('AP_1_3', 4.3796281513144e-16),
('AP_1_4', -2.1447889127908e-18),
('AP_2_0', -4.7550300344365e-08),
('AP_2_1', 1.0924172283232e-11),
('AP_2_2', -4.9862026098260e-16),
('AP_2_3', -5.4470851768869e-20),
('AP_3_0', 5.0130654116966e-10),
('AP_3_1', 6.8649554020012e-16),
('AP_3_2', -4.2759588436342e-18),
('AP_4_0', -3.6306802581471e-16),
('AP_4_1', -5.3885285875084e-19),
('AP_5_0', -1.8802693525108e-18),
('BP_ORDER', 5),
('BP_0_1', -2.6627855995942e-05),
('BP_0_2', 1.1143451873584e-08),
('BP_0_3', 4.9323396530135e-10),
('BP_0_4', 1.1785185735421e-15),
('BP_0_5', -1.6169957016415e-18),
('BP_1_0', -5.7914490267576e-06),
('BP_1_1', -3.0565765766244e-08),
('BP_1_2', 5.7727475030971e-12),
('BP_1_3', -4.0586821113726e-16),
('BP_1_4', -2.0662723654322e-19),
('BP_2_0', 2.3705520015164e-08),
('BP_2_1', 5.0530823594352e-10),
('BP_2_2', 3.8904979943489e-16),
('BP_2_3', -3.8346209540986e-18),
('BP_3_0', 1.9505421473262e-11),
('BP_3_1', 1.7583146713289e-16),
('BP_3_2', -3.4876779564534e-19),
('BP_4_0', -3.3690937119054e-16),
('BP_4_1', -2.0853007589561e-18),
('BP_5_0', -5.5344298912288e-19),
('CTYPE1', "RA---TAN-SIP"),
('CTYPE2', "DEC--TAN-SIP"),
):
metadata.set(name, value)
self.metadata = metadata
def testNoConflicts(self):
"""Test combination with valid values and no overlap,
except COMMENT and HISTORY, which are combined
"""
md1 = PropertyList()
md1.set("int1", [1, 2])
md1.set("float1", 1.23)
md1.set("string1", "md1 string1 value")
md1.set("COMMENT", "md1 comment")
md1.set("HISTORY", "md1 history")
md1Copy = md1.deepCopy()
md2 = PropertyList()
md2.set("int2", 2)
md2.set("float2", [2.34, -3.45])
md2.set("string2", "md2 string2 value")
md2.set("COMMENT", "md2 comment")
md2.set("HISTORY", "md2 history")
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
self.assertEqual(result.getOrderedNames(), [
"int1", "float1", "string1", "COMMENT", "HISTORY", "int2",
"float2", "string2"
])
self.assertEqual(result.get("COMMENT"), ["md1 comment", "md2 comment"])
self.assertEqual(result.get("HISTORY"), ["md1 history", "md2 history"])
for name in md1.getOrderedNames():
if name in ("COMMENT", "HISTORY"):
continue
self.assertEqual(result.get(name), getLast(md1.get(name)))
for name in md2.getOrderedNames():
if name in ("COMMENT", "HISTORY"):
continue
self.assertEqual(result.get(name), getLast(md2.get(name)))
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def testReplaceDuplicates(self):
"""Test that names in `second` override those in `first`, regardless of type
"""
# names that start with "item" appear in both sets of metadata
md1 = PropertyList()
md1.set("int1", 5)
md1.set("itema", [1, 2])
md1.set("float1", 3.1)
md1.set("itemb", 1.23)
md1.set("string1", "md1 string1 value")
md1.set("itemc", "md1 string value")
md1Copy = md1.deepCopy()
md2 = PropertyList()
md2.set("itemc", 2)
md2.set("int2", 2)
md2.set("itemb", ["some data", "more data"])
md2.set("float2", 2.34)
md2.set("itema", 5.27)
md2.set("string2", "md2 string value")
md2Names = md2.getOrderedNames()
md2Copy = md2.deepCopy()
result = combineMetadata(md1, md2)
expectedNames = ["int1", "float1", "string1"] + list(md2Names)
self.assertEqual(result.getOrderedNames(), expectedNames)
md2NameSet = set(md2Names)
for name in result.getOrderedNames():
if name in md2NameSet:
self.assertEqual(result.get(name), getLast(md2.get(name)))
else:
self.assertEqual(result.get(name), getLast(md1.get(name)))
# input should be unchanged
self.assertMetadataEqual(md1, md1Copy)
self.assertMetadataEqual(md2, md2Copy)
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None):
"""
Use FGCM code to match observations into unique stars.
Parameters
----------
visitCat: `afw.table.BaseCatalog`
Catalog with visit data for fgcm
obsCat: `afw.table.BaseCatalog`
Full catalog of star observations for fgcm
lutHandle: `lsst.daf.butler.DeferredDatasetHandle`, optional
Data reference to fgcm look-up table (used if matching reference stars).
Returns
-------
fgcmStarIdCat: `afw.table.BaseCatalog`
Catalog of unique star identifiers and index keys
fgcmStarIndicesCat: `afwTable.BaseCatalog`
Catalog of unique star indices
fgcmRefCat: `afw.table.BaseCatalog`
Catalog of matched reference stars.
Will be None if `config.doReferenceMatches` is False.
"""
# get filter names into a numpy array...
# This is the type that is expected by the fgcm code
visitFilterNames = np.zeros(len(visitCat), dtype='a30')
for i in range(len(visitCat)):
visitFilterNames[i] = visitCat[i]['physicalFilter']
# match to put filterNames with observations
visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit'])
obsFilterNames = visitFilterNames[visitIndex]
if self.config.doReferenceMatches:
# Get the reference filter names, using the LUT
lutCat = lutHandle.get()
stdFilterDict = {
filterName: stdFilter
for (filterName, stdFilter
) in zip(lutCat[0]['physicalFilters'].split(','),
lutCat[0]['stdPhysicalFilters'].split(','))
}
stdLambdaDict = {
stdFilter: stdLambda
for (stdFilter, stdLambda
) in zip(lutCat[0]['stdPhysicalFilters'].split(','),
lutCat[0]['lambdaStdFilter'])
}
del lutCat
referenceFilterNames = self._getReferenceFilterNames(
visitCat, stdFilterDict, stdLambdaDict)
self.log.info("Using the following reference filters: %s" %
(', '.join(referenceFilterNames)))
else:
# This should be an empty list
referenceFilterNames = []
# make the fgcm starConfig dict
starConfig = {
'logger': self.log,
'useHtm': True,
'filterToBand': self.config.physicalFilterMap,
'requiredBands': self.config.requiredBands,
'minPerBand': self.config.minPerBand,
'matchRadius': self.config.matchRadius,
'isolationRadius': self.config.isolationRadius,
'matchNSide': self.config.matchNside,
'coarseNSide': self.config.coarseNside,
'densNSide': self.config.densityCutNside,
'densMaxPerPixel': self.config.densityCutMaxPerPixel,
'randomSeed': self.config.randomSeed,
'primaryBands': self.config.primaryBands,
'referenceFilterNames': referenceFilterNames
}
# initialize the FgcmMakeStars object
fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
# make the primary stars
# note that the ra/dec native Angle format is radians
# We determine the conversion from the native units (typically
# radians) to degrees for the first observation. This allows us
# to treate ra/dec as numpy arrays rather than Angles, which would
# be approximately 600x slower.
conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
obsCat['dec'] * conv,
filterNameArray=obsFilterNames,
bandSelected=False)
# and match all the stars
fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
obsCat['dec'] * conv, obsFilterNames)
if self.config.doReferenceMatches:
fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
# now persist
objSchema = self._makeFgcmObjSchema()
# make catalog and records
fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
for i in range(fgcmMakeStars.objIndexCat.size):
fgcmStarIdCat.addNew()
# fill the catalog
fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat[
'obsarrindex']
fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
obsSchema = self._makeFgcmObsSchema()
fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
for i in range(fgcmMakeStars.obsIndexCat.size):
fgcmStarIndicesCat.addNew()
fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat[
'obsindex']
if self.config.doReferenceMatches:
refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
fgcmRefCat = afwTable.BaseCatalog(refSchema)
fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
for i in range(fgcmMakeStars.referenceCat.size):
fgcmRefCat.addNew()
fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat[
'refMagErr']
md = PropertyList()
md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
md.set("FILTERNAMES", referenceFilterNames)
fgcmRefCat.setMetadata(md)
else:
fgcmRefCat = None
return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
def setUp(self):
metadata = PropertyList()
# the following was fit using CreateWcsWithSip from meas_astrom
# and is valid over this bbox: (minimum=(0, 0), maximum=(3030, 3030))
# This same metadata was used to create testdata/oldTanSipwWs.fits
for name, value in (
("RADESYS", "ICRS"),
("CTYPE1", "RA---TAN-SIP"),
("CTYPE2", "DEC--TAN-SIP"),
("CRPIX1", 1531.1824767147),
("CRPIX2", 1531.1824767147),
("CRVAL1", 43.035511801383),
("CRVAL2", 44.305697682784),
("CUNIT1", "deg"),
("CUNIT2", "deg"),
("CD1_1", 0.00027493991598151),
("CD1_2", -3.2758487104158e-06),
("CD2_1", 3.2301310675830e-06),
("CD2_2", 0.00027493937506632),
("A_ORDER", 5),
("A_0_2", -1.7769487466972e-09),
("A_0_3", 5.3745894718340e-13),
("A_0_4", -7.2921116596880e-17),
("A_0_5", 8.6947236956136e-21),
("A_1_1", 5.4246387438098e-08),
("A_1_2", -1.5689083084641e-12),
("A_1_3", 1.2424130500997e-16),
("A_1_4", 3.9982572658006e-20),
("A_2_0", 4.9268299826160e-08),
("A_2_1", 1.6365657558495e-12),
("A_2_2", 1.1976983061953e-16),
("A_2_3", -1.7262037266467e-19),
("A_3_0", -5.9235031179999e-13),
("A_3_1", -3.4444326387310e-16),
("A_3_2", 1.4377441160800e-19),
("A_4_0", 1.8736407845095e-16),
("A_4_1", 2.9213314172884e-20),
("A_5_0", -5.3601346091084e-20),
("B_ORDER", 5),
("B_0_2", 4.9268299822979e-08),
("B_0_3", -5.9235032026906e-13),
("B_0_4", 1.8736407776035e-16),
("B_0_5", -5.3601341373220e-20),
("B_1_1", 5.4246387435453e-08),
("B_1_2", 1.6365657531115e-12),
("B_1_3", -3.4444326228808e-16),
("B_1_4", 2.9213312399941e-20),
("B_2_0", -1.7769487494962e-09),
("B_2_1", -1.5689082999319e-12),
("B_2_2", 1.1976983393279e-16),
("B_2_3", 1.4377441169892e-19),
("B_3_0", 5.3745894237186e-13),
("B_3_1", 1.2424130479929e-16),
("B_3_2", -1.7262036838229e-19),
("B_4_0", -7.2921117326608e-17),
("B_4_1", 3.9982566975450e-20),
("B_5_0", 8.6947240592408e-21),
("AP_ORDER", 6),
("AP_0_0", -5.4343024221207e-11),
("AP_0_1", 5.5722265946666e-12),
("AP_0_2", 1.7769484042400e-09),
("AP_0_3", -5.3773609554820e-13),
("AP_0_4", 7.3035278852156e-17),
("AP_0_5", -8.7151153799062e-21),
("AP_0_6", 3.2535945427624e-27),
("AP_1_0", -3.8944805432871e-12),
("AP_1_1", -5.4246388067582e-08),
("AP_1_2", 1.5741716194971e-12),
("AP_1_3", -1.2447067748187e-16),
("AP_1_4", -3.9960260822306e-20),
("AP_1_5", 1.1297941471380e-26),
("AP_2_0", -4.9268299293185e-08),
("AP_2_1", -1.6256111849359e-12),
("AP_2_2", -1.1973373130440e-16),
("AP_2_3", 1.7266948205700e-19),
("AP_2_4", -3.7059606160753e-26),
("AP_3_0", 5.9710911995811e-13),
("AP_3_1", 3.4464427650041e-16),
("AP_3_2", -1.4381853884204e-19),
("AP_3_3", -7.6527426974322e-27),
("AP_4_0", -1.8748435698960e-16),
("AP_4_1", -2.9267280226373e-20),
("AP_4_2", 4.8004317051259e-26),
("AP_5_0", 5.3657330221120e-20),
("AP_5_1", -1.6904065766661e-27),
("AP_6_0", -1.9484495120493e-26),
("BP_ORDER", 6),
("BP_0_0", -5.4291220607725e-11),
("BP_0_1", -3.8944871307931e-12),
("BP_0_2", -4.9268299290361e-08),
("BP_0_3", 5.9710912831833e-13),
("BP_0_4", -1.8748435594265e-16),
("BP_0_5", 5.3657325543368e-20),
("BP_0_6", -1.9484577299247e-26),
("BP_1_0", 5.5722051513577e-12),
("BP_1_1", -5.4246388065000e-08),
("BP_1_2", -1.6256111821465e-12),
("BP_1_3", 3.4464427499767e-16),
("BP_1_4", -2.9267278448109e-20),
("BP_1_5", -1.6904244067295e-27),
("BP_2_0", 1.7769484069376e-09),
("BP_2_1", 1.5741716110182e-12),
("BP_2_2", -1.1973373446176e-16),
("BP_2_3", -1.4381853893526e-19),
("BP_2_4", 4.8004294492911e-26),
("BP_3_0", -5.3773609074713e-13),
("BP_3_1", -1.2447067726801e-16),
("BP_3_2", 1.7266947774875e-19),
("BP_3_3", -7.6527556667042e-27),
("BP_4_0", 7.3035279660505e-17),
("BP_4_1", -3.9960255158200e-20),
("BP_4_2", -3.7059659675039e-26),
("BP_5_0", -8.7151157361284e-21),
("BP_5_1", 1.1297944388060e-26),
("BP_6_0", 3.2535788867488e-27),
):
metadata.set(name, value)
self.metadata = metadata
self.bbox = Box2D(Point2D(-1000, -1000), Extent2D(3000, 3000))
