def __init__(self, butler=None, schema=None, refObjLoader=None, **kwargs):
"""!
@param[in] butler: the butler can be used to retrieve schema or passed to the refObjLoader constructor
in case it is needed.
@param[in] schema: the schema of the source detection catalog used as input.
@param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
catalog. May be None if the butler argument is provided or all steps requiring a reference
catalog are disabled.
"""
BatchPoolTask.__init__(self, **kwargs)
if schema is None:
assert butler is not None, "Butler not provided"
schema = butler.get(self.config.coaddName + "Coadd_det_schema",
immediate=True).schema
self.butler = butler
self.makeSubtask("detectCoaddSources")
self.makeSubtask("mergeCoaddDetections", schema=schema)
self.makeSubtask("measureCoaddSources",
schema=afwTable.Schema(
self.mergeCoaddDetections.schema),
peakSchema=afwTable.Schema(
self.mergeCoaddDetections.merged.getPeakSchema()),
refObjLoader=refObjLoader,
butler=butler)
self.makeSubtask("mergeCoaddMeasurements",
schema=afwTable.Schema(
self.measureCoaddSources.schema))
self.makeSubtask("forcedPhotCoadd",
refSchema=afwTable.Schema(
self.mergeCoaddMeasurements.schema))
def testCatalogExtras(self):
butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2)
schema = afwTable.Schema()
aa = schema.addField("a", type=np.int32, doc="a")
bb = schema.addField("b", type=np.float64, doc="b")
catalog = lsst.afw.table.BaseCatalog(schema)
row = catalog.addNew()
row.set(aa, 12345)
row.set(bb, 1.2345)
size = len(catalog)
dataId = dict(visit=123, ccd=45)
butler.put(catalog, "someCatalog", dataId)
filename = butler.get("someCatalog_filename", dataId)[0]
try:
self.assertTrue(os.path.exists(filename))
self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
self.assertEqual(butler.get("someCatalog_len", dataId), size)
header = butler.get("someCatalog_md", dataId)
self.assertEqual(header.getScalar("NAXIS2"), size)
finally:
try:
os.remove(filename)
except OSError as exc:
print("Warning: could not remove file %r: %s" %
(filename, exc))
def _wrap_result(data,
column_names,
table=None,
index_col=None,
coerce_float=True,
column_dtypes=None,
parse_dates=None):
"""Wrap result set of query in a afw table """
result_size = len(data)
# Turn into columns first
from pandas import lib
data = list(lib.to_object_array_tuples(data).T)
arrays = [lib.maybe_convert_objects(arr, try_float=True) for arr in data]
_harmonize_columns(arrays, column_names, table, column_dtypes, parse_dates)
schema = afw_table.Schema()
# build schema
for i, column_name in enumerate(column_names):
column_type = arrays[i].dtype
schema.addField(column_name, type=column_type.type)
catalog = afw_table.BaseCatalog(schema)
# Preallocate rows based on first column length
catalog.preallocate(result_size)
for i in range(result_size):
record = catalog.addNew()
for column_i in range(len(column_names)):
record.set(column_names[column_i], arrays[column_i][i])
return catalog
def toTableDataTable(self, metadata):
"""Produce linearity catalog from table data
Parameters
----------
metadata : `lsst.daf.base.PropertyList`
Linearizer metadata
Returns
-------
catalog : `lsst.afw.table.BaseCatalog`
Catalog to write
"""
schema = afwTable.Schema()
dimensions = self.tableData.shape
lut = schema.addField("LOOKUP_VALUES",
type='ArrayF',
size=dimensions[1],
doc="linearity lookup data")
catalog = afwTable.BaseCatalog(schema)
catalog.resize(dimensions[0])
for ii in range(dimensions[0]):
catalog[ii][lut] = np.array(self.tableData[ii], dtype=np.float32)
metadata["LINEARITY_LOOKUP"] = True
catalog.setMetadata(metadata)
return catalog
def _makeFgcmRefSchema(self, nReferenceBands):
"""
Make a schema for the referenceCat from fgcmMakeStars
Parameters
----------
nReferenceBands: `int`
Number of reference bands
Returns
-------
schema: `afwTable.Schema`
"""
refSchema = afwTable.Schema()
refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
refSchema.addField('refMag',
type='ArrayF',
doc='Reference magnitude array (AB)',
size=nReferenceBands)
refSchema.addField('refMagErr',
type='ArrayF',
doc='Reference magnitude error array',
size=nReferenceBands)
return refSchema
def _get_afw_table():
schema = afw_table.Schema()
aa = schema.addField("a", type=np.int64, doc="a")
bb = schema.addField("b", type=np.float64, doc="b")
cat = afw_table.BaseCatalog(schema)
row = cat.addNew()
row.set(aa, 12345)
row.set(bb, 1.2345)
row = cat.addNew()
row.set(aa, 4321)
row.set(bb, 4.123)
return cat
def _makeFgcmVisitSchema(self, nCcd):
"""
Make a schema for an fgcmVisitCatalog
Parameters
----------
nCcd: `int`
Number of CCDs in the camera
Returns
-------
schema: `afwTable.Schema`
"""
schema = afwTable.Schema()
schema.addField('visit', type=np.int64, doc="Visit number")
schema.addField('physicalFilter',
type=str,
size=30,
doc="Physical filter")
schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
schema.addField('telha',
type=np.float64,
doc="Pointing Hour Angle (deg)")
schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)")
schema.addField('mjd', type=np.float64, doc="MJD of visit")
schema.addField('exptime', type=np.float32, doc="Exposure time")
schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
schema.addField('psfSigma',
type=np.float32,
doc="PSF sigma (reference CCD)")
schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
schema.addField('skyBackground',
type=np.float32,
doc="Sky background (ADU) (reference CCD)")
# the following field is not used yet
schema.addField('deepFlag', type=np.int32, doc="Deep observation")
schema.addField('scaling',
type='ArrayD',
doc="Scaling applied due to flat adjustment",
size=nCcd)
schema.addField('used',
type=np.int32,
doc="This visit has been ingested.")
schema.addField('sources_read',
type='Flag',
doc="This visit had sources read.")
return schema
def _makeFgcmObsSchema(self):
"""
Make a schema for the obsIndexCat from fgcmMakeStars
Returns
-------
schema: `afwTable.Schema`
"""
obsSchema = afwTable.Schema()
obsSchema.addField('obsIndex',
type=np.int32,
doc='Index in observation table')
return obsSchema
def __init__(self, *args, **kwargs):
"""
"""
super(BuildCovarianceTask, self).__init__(*args, **kwargs)
self.ncolors = 0
self.bfd = dbfd.BFDConfig(use_conc=self.config.use_conc,
use_mag=self.config.use_mag,
ncolors=self.ncolors)
self.n_even = self.bfd.BFDConfig.MSIZE
self.n_odd = self.bfd.BFDConfig.XYSIZE
self.size_even = self.n_even * (self.n_even + 1) // 2
self.size_odd = self.n_odd * (self.n_odd + 1) // 2
self.schema = afwTable.Schema()
self.labelKey = self.schema.addField("label",
type=str,
doc="name of bin",
size=10)
self.minKey = self.schema.addField("min",
type=float,
doc="minimum value of the variance")
self.maxKey = self.schema.addField("max",
type=float,
doc="maximum value of the variance")
self.isoCovEvenKey = self.schema.addField(
"isoCovEven",
doc="isotropized moment covariance matrix",
type="ArrayF",
size=self.size_even)
self.isoCovOddKey = self.schema.addField(
"isoCovOdd",
doc="isotropized moment covariance matrix",
type="ArrayF",
size=self.size_odd)
self.covEvenKey = self.schema.addField("covEven",
doc="moment covariance matrix",
type="ArrayF",
size=self.size_even)
self.covOddKey = self.schema.addField("covOdd",
doc="moment covariance matrix",
type="ArrayF",
size=self.size_odd)
self.catalog = afwTable.BaseCatalog(self.schema)
def _makeForcedSourceCatalog(objects):
"""Make a catalog containing a bunch of DiaFourceSources associated with
the input diaObjects.
"""
# make some sources
schema = afwTable.Schema()
schema.addField("diaObjectId", "L")
schema.addField("ccdVisitId", "L")
schema.addField("flags", "L")
catalog = afwTable.BaseCatalog(schema)
oids = []
for obj in objects:
record = catalog.addNew()
record.set("diaObjectId", obj["id"])
record.set("ccdVisitId", 1)
record.set("flags", 0)
oids.append(obj["id"])
return catalog, oids
def _makeFgcmVisitSchema(self, nCcd):
"""
Make a schema for an fgcmVisitCatalog
Parameters
----------
nCcd: `int`
Number of CCDs in the camera
Returns
-------
schema: `afwTable.Schema`
"""
schema = afwTable.Schema()
schema.addField('visit', type=np.int32, doc="Visit number")
# Note that the FGCM code currently handles filternames up to 2 characters long
schema.addField('filtername', type=str, size=2, doc="Filter name")
schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
schema.addField('telha',
type=np.float64,
doc="Pointing Hour Angle (deg)")
schema.addField('mjd', type=np.float64, doc="MJD of visit")
schema.addField('exptime', type=np.float32, doc="Exposure time")
schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
schema.addField('psfSigma',
type=np.float32,
doc="PSF sigma (reference CCD)")
schema.addField('deltaAper', type=np.float32, doc="Delta-aperture")
schema.addField('skyBackground',
type=np.float32,
doc="Sky background (ADU) (reference CCD)")
# the following field is not used yet
schema.addField('deepFlag', type=np.int32, doc="Deep observation")
schema.addField('scaling',
type='ArrayD',
doc="Scaling applied due to flat adjustment",
size=nCcd)
return schema
def _makeFgcmObjSchema(self):
"""
Make a schema for the objIndexCat from fgcmMakeStars
Returns
-------
schema: `afwTable.Schema`
"""
objSchema = afwTable.Schema()
objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
# Will investigate making these angles...
objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)')
objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)')
objSchema.addField('obsArrIndex',
type=np.int32,
doc='Index in obsIndexTable for first observation')
objSchema.addField('nObs',
type=np.int32,
doc='Total number of observations')
return objSchema
def makeAtmSchema():
"""
Make the atmosphere schema
Returns
-------
atmSchema: `lsst.afw.table.Schema`
"""
atmSchema = afwTable.Schema()
atmSchema.addField('visit', type=np.int32, doc='Visit number')
atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)')
atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)')
atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth')
atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope')
atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)')
atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)')
atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor')
atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction')
return atmSchema
def run(self, dataRef, selectDataList=[]):
if self.config.fileOutName == "":
if self.config.dirOutName == "" :
dirOutName = dataRef.getButler().mapper.root+"/"+self.config.coaddName+"Coadd-results"
self.log.info("WARNING: the output file will be written in {0:s}.".format(dirOutName))
else:
dirOutName = self.config.dirOutName
fileOutName = "{0}/{1}/{2}/{3}/multiCat-{2}-{3}.fits".format(dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"])
else:
fileOutName = self.config.fileOutName
if os.path.isfile(fileOutName) and not self.config.clobber:
self.log.info("File for %s exists. Exiting..." % (dataRef.dataId))
return
self.log.info("Processing %s" % (dataRef.dataId))
filters = self.config.filters.split("^")
dustCoefs = [float(c) for c in self.config.dustCoefs.split("^") ]
ref = dataRef.get("deepCoadd_ref")
catalogs = dict(self.readCatalog(dataRef, f) for f in filters)
coadds = dict(self.readCoadd(dataRef, f) for f in filters)
ghostFilters = []
if self.config.ghostFilters != "":
ghostFilters = self.config.ghostFilters.split("^")
self.log.info("Ghost filters: %s" % ghostFilters)
# print ref.schema.getOrderedNames()
# print dir(ref.schema)
# print catalogs[filters[0]].schema.getOrderedNames()
# return
fluxMag0 = {}
for f in filters:
fluxMag0[f] = coadds[f].getCalib().getFluxMag0()[0]
self.log.info("Mag ZP for filter {0:s}: {1:f}".format(f, 2.5*np.log10(fluxMag0[f])))
wcs = coadds[filters[0]].getWcs()
pixel_scale = wcs.pixelScale().asDegrees()*3600.0
aperId = [int(a) for a in self.config.aperId.split(",")]
"""display which aperture diameter size will be used
"""
aperSize = []
for j, a in enumerate(aperId):
aperSize.append(catalogs[filters[0]].getMetadata().get("flux_aperture_radii")[a] * 2.0 * pixel_scale)
self.log.info("Diameter of flux apertures: {0:f}\"".format(aperSize[j]))
"""create new table table
"""
mergedSchema = afwTable.Schema()
"""define table fields
"""
fields=[]
fields.append(mergedSchema.addField("id", type="L", doc="Unique id"))
fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]"))
fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]"))
fields.append(mergedSchema.addField("tract", type="I", doc="tract number"))
fields.append(mergedSchema.addField("patch", type="String", size=3, doc="patch number"))
fields.append(mergedSchema.addField("refFilter", type="String", size=10, doc="Name of the filter used as reference"))
fields.append(mergedSchema.addField("countInputs", type="I", doc="Number of input single exposures for the reference filter"))
fields.append(mergedSchema.addField("detRadius", type="F", doc="Determinant radius for the object in the reference filter = sigma if gaussian [arcsec]"))
fields.append(mergedSchema.addField("PSFDetRadius", type="F", doc="Determinant radius for the PSF in the reference filter at the object position = sigma if gaussian [arcsec]"))
fields.append(mergedSchema.addField("cmodel_fracDev", type="F", doc="fraction of flux in de Vaucouleur component"))
fields.append(mergedSchema.addField("blendedness", type="F", doc="Ranges from 0 (unblended) to 1 (blended)"))
fields.append(mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]"))
fields.append(mergedSchema.addField("extendedness", type="F", doc="probability of being extended from PSF/cmodel flux difference"))
fields.append(mergedSchema.addField("hasBadCentroid", type="I", doc="1 if has bad centroid (but not used in islean"))
fields.append(mergedSchema.addField("isSky", type="I", doc="1 if sky object"))
fields.append(mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch"))
fields.append(mergedSchema.addField("isParent", type="I", doc="1 if parent of a deblended object"))
fields.append(mergedSchema.addField("isClean_refFilter", type="I", doc="1 if none of other flags is set for reference filter"))
for f in filters+ghostFilters:
fields.append(mergedSchema.addField("hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if interpolated, saturated, suspect, has CR at center or near bright object for filter {0:s}".format(f)))
for f in filters+ghostFilters:
fields.append(mergedSchema.addField("isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if offImage or in region masked EDGE or NO_DATA for filter {0:s}".format(f)))
"""photometry estimates
"""
photo = ["flux.aperture", "flux.kron", "flux.psf", "cmodel.flux"]
for p in photo:
for f in filters+ghostFilters:
if p == "flux.aperture":
for a in aperSize:
keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_")
fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s} within {2:f}\" diameter aperture".format(p,f,a)))
fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f)))
else:
keyName = (p+"_"+f).replace(".", "_").replace("-", "_")
fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s}".format(p,f)))
fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f)))
fields.append(mergedSchema.addField(p+"_flag".replace(".", "_"), type="I", doc="Highest flag value among filters for {0:s}".format(p)))
"""dust corrections
"""
for f in filters+ghostFilters:
fields.append(mergedSchema.addField(("EB_V_corr_"+f).replace(".", "_").replace("-", "_"), type="F", doc="Milky way dust flux correction for filter {0:s}".format(f)))
"""create table object
"""
merged = afwTable.BaseCatalog(mergedSchema)
N = len(ref)
# count = 0
for i in range(N):
# for i in range(10000,10200):
# for i in range(1,100):
"""create new record
"""
record = merged.addNew()
coord = ref[i].get('coord')
"""record common info from reference filter
"""
record.set(mergedSchema['id'].asKey(), ref[i].get('id'))
record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees())
record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees())
record.set(mergedSchema['tract'].asKey(), dataRef.dataId["tract"])
record.set(mergedSchema['patch'].asKey(), dataRef.dataId["patch"])
record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs'))
record.set(mergedSchema['detRadius'].asKey(), ref[i].get("shape.sdss").getDeterminantRadius()*pixel_scale)
record.set(mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius()*pixel_scale)
record.set(mergedSchema['cmodel_fracDev'].asKey(), ref[i].get('cmodel.fracDev'))
record.set(mergedSchema['blendedness'].asKey(), ref[i].get('blendedness.abs.flux'))
record.set(mergedSchema['extendedness'].asKey(), ref[i].get('classification.extendedness'))
record.set(mergedSchema['hasBadCentroid'].asKey(), int(ref[i].get('centroid.sdss.flags')))
record.set(mergedSchema['isSky'].asKey(), int(ref[i].get('merge.footprint.sky')))
record.set(mergedSchema['isDuplicated'].asKey(), int(not ref[i].get('detect.is-primary')))
record.set(mergedSchema['isParent'].asKey(), int(ref[i].get('deblend.nchild') != 0))
"""record the name of the filter used as reference
"""
for f in filters:
name = afwImage.Filter(afwImage.Filter(f).getId()).getName()
if ref[i].get("merge.measurement."+name):
refName = f.replace(".", "_").replace("-", "_")
record.set(mergedSchema["refFilter"].asKey(), refName)
break
"""photometry measurement flags
"""
for p in photo:
flag = 0
for f in filters:
if catalogs[f][i].get(p+".flags") > flag:
flag = catalogs[f][i].get(p+".flags")
record.set(mergedSchema[p+"_flag".replace(".", "_")].asKey(), flag)
"""record bad photometry flag for each filter
"""
for f in filters:
record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int((catalogs[f][i].get('flags.pixel.offimage')) | (catalogs[f][i].get('flags.pixel.edge'))))
record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int(
(catalogs[f][i].get('flags.pixel.interpolated.center')) \
| (catalogs[f][i].get('flags.pixel.saturated.center')) \
| (catalogs[f][i].get('flags.pixel.suspect.center')) \
| (catalogs[f][i].get('flags.pixel.cr.center')) \
| (catalogs[f][i].get('flags.pixel.bad')) \
| (catalogs[f][i].get('flags.pixel.bright.object.center'))))
for f in ghostFilters:
record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1)
record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1)
"""record isClean flag for reference filter
"""
hasBadPhotometry_refFilter = \
(ref[i].get('flags.pixel.interpolated.center')) \
| (ref[i].get('flags.pixel.saturated.center')) \
| (ref[i].get('flags.pixel.suspect.center')) \
| (ref[i].get('flags.pixel.cr.center')) \
| (ref[i].get('flags.pixel.bad')) \
| (ref[i].get('flags.pixel.bright.object.center'))
isNoData_refFilter = (ref[i].get('flags.pixel.offimage')) | (ref[i].get('flags.pixel.edge'))
isSky_refFilter = ref[i].get('merge.footprint.sky')
isDuplicated_refFilter = not ref[i].get('detect.is-primary')
isParent_refFilter = ref[i].get('deblend.nchild') != 0
record.set(mergedSchema["isClean_refFilter"].asKey(), int(
(not hasBadPhotometry_refFilter) \
& (not isNoData_refFilter) \
& (not isSky_refFilter) \
& (not isDuplicated_refFilter) \
& (not isParent_refFilter)))
#isExtended = (ref[i].get('classification.extendedness'))
#isExtended = (catalogs[f][i].get('flux.kron') > 0.8*catalogs[f][i].get('flux.psf')) | \
# (ref.get("shape.sdss").getDeterminantRadius() > 1.1*ref.get("shape.sdss.psf").getDeterminantRadius())
#if not (isExtended == isExtended):
# isExtended = 0
#record.set(mergedSchema['isExtended'].asKey(), int(isExtended))
"""dust correction
"""
EB_V = self.getDustCorrection(self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey()))
record.set(mergedSchema['EB_V'].asKey(), EB_V)
"""flux in micro Jansky
"""
for f, dc in zip(filters, dustCoefs):
record.set(mergedSchema[("EB_V_corr_"+f).replace(".", "_").replace("-", "_")].asKey(), pow(10.0, +0.4* dc * EB_V))
for p in photo:
if p == "flux.aperture":
for a in aperId:
flux = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p)[a] / fluxMag0[f]
flux_err = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p+".err")[a] / fluxMag0[f]
keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_")
record.set(mergedSchema[keyName].asKey(), flux)
record.set(mergedSchema[keyName+"_err"].asKey(), flux_err)
else:
flux = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p)/fluxMag0[f]
flux_err = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p+".err")/fluxMag0[f]
keyName = (p+"_"+f).replace(".", "_").replace("-", "_")
record.set(mergedSchema[keyName].asKey(), flux)
record.set(mergedSchema[keyName+"_err"].asKey(), flux_err)
# count += 1
"""write catalog
"""
self.log.info("Writing {0:s}".format(fileOutName))
self.mkdir_p(os.path.dirname(fileOutName))
merged.writeFits(fileOutName)
return
def _makeLutSchema(self, physicalFilterString, stdPhysicalFilterString,
atmosphereTableName):
"""
Make the LUT schema
Parameters
----------
physicalFilterString: `str`
Combined string of all the physicalFilters
stdPhysicalFilterString: `str`
Combined string of all the standard physicalFilters
atmosphereTableName: `str`
Name of the atmosphere table used to generate LUT
Returns
-------
lutSchema: `afwTable.schema`
"""
lutSchema = afwTable.Schema()
lutSchema.addField('tablename',
type=str,
doc='Atmosphere table name',
size=len(atmosphereTableName))
lutSchema.addField('elevation',
type=float,
doc="Telescope elevation used for LUT")
lutSchema.addField('physicalFilters',
type=str,
doc='physicalFilters in LUT',
size=len(physicalFilterString))
lutSchema.addField('stdPhysicalFilters',
type=str,
doc='Standard physicalFilters in LUT',
size=len(stdPhysicalFilterString))
lutSchema.addField('pmb',
type='ArrayD',
doc='Barometric Pressure',
size=self.fgcmLutMaker.pmb.size)
lutSchema.addField('pmbFactor',
type='ArrayD',
doc='PMB scaling factor',
size=self.fgcmLutMaker.pmb.size)
lutSchema.addField('pmbElevation',
type=np.float64,
doc='PMB Scaling at elevation')
lutSchema.addField('pwv',
type='ArrayD',
doc='Preciptable Water Vapor',
size=self.fgcmLutMaker.pwv.size)
lutSchema.addField('o3',
type='ArrayD',
doc='Ozone',
size=self.fgcmLutMaker.o3.size)
lutSchema.addField('tau',
type='ArrayD',
doc='Aerosol optical depth',
size=self.fgcmLutMaker.tau.size)
lutSchema.addField('lambdaNorm', type=np.float64, doc='AOD wavelength')
lutSchema.addField('alpha',
type='ArrayD',
doc='Aerosol alpha',
size=self.fgcmLutMaker.alpha.size)
lutSchema.addField('zenith',
type='ArrayD',
doc='Zenith angle',
size=self.fgcmLutMaker.zenith.size)
lutSchema.addField('nCcd', type=np.int32, doc='Number of CCDs')
# and the standard values
lutSchema.addField('pmbStd', type=np.float64, doc='PMB Standard')
lutSchema.addField('pwvStd', type=np.float64, doc='PWV Standard')
lutSchema.addField('o3Std', type=np.float64, doc='O3 Standard')
lutSchema.addField('tauStd', type=np.float64, doc='Tau Standard')
lutSchema.addField('alphaStd', type=np.float64, doc='Alpha Standard')
lutSchema.addField('zenithStd',
type=np.float64,
doc='Zenith angle Standard')
lutSchema.addField('lambdaRange',
type='ArrayD',
doc='Wavelength range',
size=2)
lutSchema.addField('lambdaStep',
type=np.float64,
doc='Wavelength step')
lutSchema.addField('lambdaStd',
type='ArrayD',
doc='Standard Wavelength',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('lambdaStdFilter',
type='ArrayD',
doc='Standard Wavelength (raw)',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i0Std',
type='ArrayD',
doc='I0 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i1Std',
type='ArrayD',
doc='I1 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i10Std',
type='ArrayD',
doc='I10 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i2Std',
type='ArrayD',
doc='I2 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('lambdaB',
type='ArrayD',
doc='Wavelength for passband (no atm)',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('atmLambda',
type='ArrayD',
doc='Atmosphere wavelengths (Angstrom)',
size=self.fgcmLutMaker.atmLambda.size)
lutSchema.addField('atmStdTrans',
type='ArrayD',
doc='Standard Atmosphere Throughput',
size=self.fgcmLutMaker.atmStdTrans.size)
# and the look-up-tables
lutSchema.addField('luttype',
type=str,
size=20,
doc='Look-up table type')
lutSchema.addField('lut',
type='ArrayF',
doc='Look-up table for luttype',
size=self.fgcmLutMaker.lut['I0'].size)
return lutSchema
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = butlerQC.get(inputRefs)
self.log.info("Running with %d sourceTable_visit handles",
(len(handleDict['source_catalogs'])))
# Run the build stars tasks
tract = butlerQC.quantum.dataId['tract']
handleDict['sourceSchema'] = self.sourceSchema
sourceTableHandles = handleDict['source_catalogs']
sourceTableHandleDict = {
sourceTableHandle.dataId['visit']: sourceTableHandle
for sourceTableHandle in sourceTableHandles
}
visitSummaryHandles = handleDict['visitSummary']
visitSummaryHandleDict = {
visitSummaryHandle.dataId['visit']: visitSummaryHandle
for visitSummaryHandle in visitSummaryHandles
}
handleDict['sourceTableHandleDict'] = sourceTableHandleDict
handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict
# And the outputs
if self.config.fgcmOutputProducts.doZeropointOutput:
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
handleDict['fgcmPhotoCalibs'] = photoCalibRefDict
if self.config.fgcmOutputProducts.doAtmosphereOutput:
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
handleDict['fgcmTransmissionAtmospheres'] = atmRefDict
if self.config.fgcmBuildStars.doReferenceMatches:
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
buildStarsRefObjLoader = loader
else:
buildStarsRefObjLoader = None
if self.config.fgcmOutputProducts.doReferenceCalibration:
refConfig = self.config.fgcmOutputProducts.refObjLoader
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
self.fgcmOutputProducts.refObjLoader = loader
struct = self.run(handleDict,
tract,
buildStarsRefObjLoader=buildStarsRefObjLoader)
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
# Turn raw repeatability into simple catalog for persistence
schema = afwTable.Schema()
schema.addField('rawRepeatability',
type=np.float64,
doc="Per-band raw repeatability in FGCM calibration.")
repeatabilityCat = afwTable.BaseCatalog(schema)
repeatabilityCat.resize(len(struct.repeatability))
repeatabilityCat['rawRepeatability'][:] = struct.repeatability
butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
return
true_y.append(gtrue.y)
for filter_name, filter_ in filters.items():
tmp = cgal.drawImage(filter_,
image=images[filter_name],
add_to_image=True,
offset=(offsetx, offsety))
gtrue.fluxes[filter_name] = tmp.added_flux
true_gals.append(gtrue)
for filter_name, filter_ in filters.items():
tot_images[filter_name][bounds] = images[filter_name]
# Write out truth catalogs
true_schema = afwTable.Schema()
xKey = true_schema.addField("x", type=float, doc="x position")
yKey = true_schema.addField("y", type=float, doc="y position")
radKey = true_schema.addField("size", type=float, doc="size")
zKey = true_schema.addField("redshift", type=float, doc="redshift")
fluxKeys = {}
for filter_ in filter_names:
fluxKeys[filter_] = true_schema.addField("flux_%s" % filter_,
type=float,
doc="flux in %s" % filter_)
w_vals = np.arange(100, 1500, 5)
wKey = true_schema.addField("wave",
type="ArrayF",
doc="wavelengths",
size=len(w_vals))
sedKey = true_schema.addField("sed",
def toAmpTable(self, metadata):
"""Produce linearity catalog
Parameters
----------
metadata : `lsst.daf.base.PropertyList`
Linearizer metadata
Returns
-------
catalog : `lsst.afw.table.BaseCatalog`
Catalog to write
"""
metadata["LINEARITY_SCHEMA"] = "Linearity table"
metadata["LINEARITY_VERSION"] = 1
# Now pack it into a fits table.
length = max([
len(self.linearityCoeffs[x]) for x in self.linearityCoeffs.keys()
])
schema = afwTable.Schema()
names = schema.addField("AMPLIFIER_NAME",
type="String",
size=16,
doc="linearity amplifier name")
types = schema.addField("TYPE",
type="String",
size=16,
doc="linearity type names")
coeffs = schema.addField("COEFFS",
type="ArrayD",
size=length,
doc="linearity coefficients")
boxX = schema.addField("BBOX_X0",
type="I",
doc="linearity bbox minimum x")
boxY = schema.addField("BBOX_Y0",
type="I",
doc="linearity bbox minimum y")
boxDx = schema.addField("BBOX_DX",
type="I",
doc="linearity bbox x dimension")
boxDy = schema.addField("BBOX_DY",
type="I",
doc="linearity bbox y dimension")
if (self.fitParams):
lengthFitParams = max(
[len(self.fitParams[x]) for x in self.fitParams.keys()])
fitParams = schema.addField(
"FIT_PARAMS",
type="ArrayD",
size=lengthFitParams,
doc="parameters of linearity polynomial fit")
fitParamsErr = schema.addField(
"FIT_PARAMS_ERR",
type="ArrayD",
size=lengthFitParams,
doc="errors of parameters of linearity polynomial fit")
reducedChiSquared = schema.addField(
"RED_CHI_SQ",
type="D",
doc="unweighted reduced chi sq. from linearity pol. fit")
catalog = afwTable.BaseCatalog(schema)
catalog.resize(len(self.linearityCoeffs.keys()))
for ii, ampName in enumerate(self.linearityType):
catalog[ii][names] = ampName
catalog[ii][types] = self.linearityType[ampName]
catalog[ii][coeffs] = np.array(self.linearityCoeffs[ampName],
dtype=float)
if (self.fitParams):
catalog[ii][fitParams] = np.array(self.fitParams[ampName],
dtype=float)
catalog[ii][fitParamsErr] = np.array(
self.fitParamsErr[ampName], dtype=float)
catalog[ii][
reducedChiSquared] = self.linearityFitReducedChiSquared[
ampName]
bbox = self.linearityBBox[ampName]
catalog[ii][boxX], catalog[ii][boxY] = bbox.getMin()
catalog[ii][boxDx], catalog[ii][boxDy] = bbox.getDimensions()
catalog.setMetadata(metadata)
return catalog
def _fgcmMatchStars(self, butler, visitCat):
"""
Use FGCM code to match observations into unique stars.
Parameters
----------
butler: lsst.daf.persistence.Butler
visitCat: afw.table.BaseCatalog
Catalog with visit data for FGCM
Returns
-------
None
"""
obsCat = butler.get('fgcmStarObservations')
# get filter names into a numpy array...
visitFilterNames = np.zeros(len(visitCat), dtype='a2')
for i in xrange(len(visitCat)):
visitFilterNames[i] = visitCat[i]['filtername']
# match to put filterNames with observations
visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit'])
obsFilterNames = visitFilterNames[visitIndex]
# make the fgcm starConfig dict
starConfig = {
'logger': self.log,
'filterToBand': self.config.filterToBand,
'requiredBands': self.config.requiredBands,
'minPerBand': self.config.minPerBand,
'matchRadius': self.config.matchRadius,
'isolationRadius': self.config.isolationRadius,
'matchNSide': self.config.matchNside,
'coarseNSide': self.config.coarseNside,
'densNSide': self.config.densityCutNside,
'densMaxPerPixel': self.config.densityCutMaxPerPixel,
'referenceBands': self.config.referenceBands
}
# initialize the FgcmMakeStars object
fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
# make the reference stars
# note that the ra/dec native Angle format is radians
fgcmMakeStars.makeReferenceStars(np.rad2deg(obsCat['ra']),
np.rad2deg(obsCat['dec']),
filterNameArray=obsFilterNames,
bandSelected=False)
# and match all the stars
fgcmMakeStars.makeMatchedStars(np.rad2deg(obsCat['ra']),
np.rad2deg(obsCat['dec']),
obsFilterNames)
# now persist
# afwTable for objects
objSchema = afwTable.Schema()
objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID')
# FIXME: should be angle?
objSchema.addField('ra', type=np.float64, doc='Mean object RA')
objSchema.addField('dec', type=np.float64, doc='Mean object Dec')
objSchema.addField('obsarrindex',
type=np.int32,
doc='Index in obsIndexTable for first observation')
objSchema.addField('nobs',
type=np.int32,
doc='Total number of observations')
# make catalog and records
fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
fgcmStarIdCat.table.preallocate(fgcmMakeStars.objIndexCat.size)
for i in xrange(fgcmMakeStars.objIndexCat.size):
fgcmStarIdCat.addNew()
# fill the catalog
fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
fgcmStarIdCat['obsarrindex'][:] = fgcmMakeStars.objIndexCat[
'obsarrindex']
fgcmStarIdCat['nobs'][:] = fgcmMakeStars.objIndexCat['nobs']
butler.put(fgcmStarIdCat, 'fgcmStarIds')
# afwTable for observation indices
obsSchema = afwTable.Schema()
obsSchema.addField('obsindex',
type=np.int32,
doc='Index in observation table')
fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
fgcmStarIndicesCat.table.preallocate(fgcmMakeStars.obsIndexCat.size)
for i in xrange(fgcmMakeStars.obsIndexCat.size):
fgcmStarIndicesCat.addNew()
fgcmStarIndicesCat['obsindex'][:] = fgcmMakeStars.obsIndexCat[
'obsindex']
butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
# and we're done with the stars
return None
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(
inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(
inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(
inputRefs.fgcmZeropoints)
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(
inputRefs.fgcmAtmosphereParameters)
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset',
type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
def __init__(self, *args, **kwargs):
BatchPoolTask.__init__(self, *args, **kwargs)
self.makeSubtask("sky")
# Disposable schema suppresses warning from SourceDetectionTask.__init__
self.makeSubtask("detection", schema=afwTable.Schema())
def run(self, dataRef, selectDataList=[]):
self.log.info("Processing %s" % (dataRef.dataId))
filters = self.config.filters.split("^")
catalogs = dict(self.readCatalog(dataRef, f) for f in filters)
coadds = dict(self.readCoadd(dataRef, f) for f in filters)
wcs = coadds[filters[0]].getWcs()
pixel_scale = wcs.pixelScale().asDegrees() * 3600.0
ref = catalogs[self.config.refFilterName]
# create new table table
mergedSchema = afwTable.Schema()
fields = []
# define table fields
fields.append(mergedSchema.addField("id", type="L", doc="Unique id"))
fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]"))
fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]"))
fields.append(
mergedSchema.addField(
"countInputs",
type="I",
doc="Number of input single exposures for the reference filter"
))
fields.append(
mergedSchema.addField(
"PSFDetRadius",
type="F",
doc=
"Determinant radius for the PSF at the object position = sigma if gaussian [arcsec]"
))
fields.append(
mergedSchema.addField("EB_V",
type="F",
doc="Milky Way dust E(B-V) [mag]"))
fields.append(
mergedSchema.addField("isDuplicated",
type="I",
doc="1 if outside the inner tract or patch"))
fields.append(
mergedSchema.addField(
"isEdge",
type="I",
doc="1 if offImage or in region masked EDGE or NO_DATA"))
fields.append(
mergedSchema.addField(
"hasBadPhotometry",
type="I",
doc=
"1 if interpolated, saturated, suspect, has CR at center or near bright object"
))
fields.append(
mergedSchema.addField("isClean",
type="I",
doc="1 if none of other flags is set"))
if self.config.hasDepthInfo:
fields.append(
mergedSchema.addField(
"isFullDepthColor",
type="I",
doc="1 if point located in full depth and color area"))
# create table object
merged = afwTable.BaseCatalog(mergedSchema)
N = len(ref)
for i in range(N):
# for i in range(100,110):
# create new record
record = merged.addNew()
coord = ref[i].get('coord')
# record if any of the filter is flagged as bad photometry
for f in filters:
hasBadPhotometry = (catalogs[f][i].get('flags.pixel.interpolated.center')) \
| (catalogs[f][i].get('flags.pixel.saturated.center')) \
| (catalogs[f][i].get('flags.pixel.suspect.center')) \
| (catalogs[f][i].get('flags.pixel.cr.center')) \
| (catalogs[f][i].get('flags.pixel.bad')) \
| (catalogs[f][i].get('flags.pixel.bright.object.center'))
if hasBadPhotometry:
break
isDuplicated = not ref[i].get('detect.is-primary')
isEdge = (ref[i].get('flags.pixel.offimage')) | (
ref[i].get('flags.pixel.edge'))
isClean = (not hasBadPhotometry) & (not isDuplicated) & (
not isEdge)
# record common info from reference filter
record.set(mergedSchema['id'].asKey(), ref[i].get('id'))
record.set(mergedSchema['ra'].asKey(),
coord.toFk5().getRa().asDegrees())
record.set(mergedSchema['dec'].asKey(),
coord.toFk5().getDec().asDegrees())
record.set(mergedSchema['countInputs'].asKey(),
ref[i].get('countInputs'))
record.set(
mergedSchema['PSFDetRadius'].asKey(),
ref[i].get("shape.sdss.psf").getDeterminantRadius() *
pixel_scale)
record.set(mergedSchema['isDuplicated'].asKey(), int(isDuplicated))
record.set(mergedSchema['isEdge'].asKey(), int(isEdge))
record.set(mergedSchema['hasBadPhotometry'].asKey(),
int(hasBadPhotometry))
record.set(mergedSchema['isClean'].asKey(), int(isClean))
if self.config.hasDepthInfo:
record.set(mergedSchema['isFullDepthColor'].asKey(),
int(ref[i].get('isFullDepthColor')))
# dust correction
EB_V = self.getDustCorrection(
self.dustMap, record.get(mergedSchema['ra'].asKey()),
record.get(mergedSchema['dec'].asKey()))
record.set(mergedSchema['EB_V'].asKey(), EB_V)
# write catalog
if self.config.fileOutName == "":
# get output dir
# TO DO: create new PAF
# see /Users/coupon/local/source/hscPipe/install/DarwinX86/solvetansip/6.5.1p_hsc/python/hsc/meas/tansip/utils.py
# and /Users/coupon/local/source/hscPipe/install/DarwinX86/pex_policy/HSC-4.0.0/tests/Policy_1.py
if self.config.dirOutName == "":
dirOutName = dataRef.getButler(
).mapper.root + "/" + self.config.coaddName + "Coadd-results"
self.log.info(
"WARNING: the output file will be written in {0:s}.".
format(dirOutName))
else:
dirOutName = self.config.dirOutName
fileOutName = "{0}/{1}/{2}/{3}/multiRanCat-{2}-{3}.fits".format(
dirOutName, "merged", dataRef.dataId["tract"],
dataRef.dataId["patch"])
else:
fileOutName = self.config.fileOutName
self.log.info("Writing {0:s}".format(fileOutName))
self.mkdir_p(os.path.dirname(fileOutName))
merged.writeFits(fileOutName)
# write catalog
#self.log.info("Writing {0:s}".format(self.config.fileOutName))
#if self.config.fileOutName == "":
# fileOutName = "{0}/{1}/{2}/{3}/MultiRanCat-{2}-{3}.fits".format(self.config.dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"])
# self.mkdir_p(os.path.dirname(fileOutName))
#else:
# fileOutName = self.config.fileOutName
#merged.writeFits(fileOutName)
return
def setUp(self):
self.schema = afwTable.Schema()
self.schema.addField('test1', type='ArrayF', size=430000025)
self.schema.addField('test2', type='ArrayF', size=430000025)
self.cat = afwTable.BaseCatalog(self.schema)
def setUp(self):
"""Create a catalog to run through the ingestion process."""
# First, connect to the database.
self.host = os.environ.get("TEST_MYSQL_HOST",
"lsst-db.ncsa.illinois.edu")
self.db = os.environ.get("TEST_MYSQL_DB", "test")
self.port = int(os.environ.get("TEST_MYSQL_PORT", "3306"))
self.conn = None
try:
self.conn = IngestCatalogTask.connect(
host=self.host, port=self.port, db=self.db)
except:
pass
# Create a schema containing one of every kind of afw field
schema = afw_table.Schema()
keys = (
# Scalar fields
schema.addField("scalar.u", type="U"),
schema.addField("scalar.i", type="I"),
schema.addField("scalar.l", type="L"),
schema.addField("scalar.f", type="F"),
schema.addField("scalar.d", type="D"),
schema.addField("scalar.flag", type="Flag"),
schema.addField("scalar.angle", type="Angle"),
# Fixed-length array types
schema.addField("fix.string", type="String", size=4),
schema.addField("fix.array.u", type="ArrayU", size=2),
schema.addField("fix.array.i", type="ArrayI", size=2),
schema.addField("fix.array.f", type="ArrayF", size=2),
schema.addField("fix.array.d", type="ArrayD", size=2),
# Variable-length array types
schema.addField("var.array.u", type="ArrayU", size=0),
schema.addField("var.array.i", type="ArrayI", size=0),
schema.addField("var.array.f", type="ArrayF", size=0),
schema.addField("var.array.d", type="ArrayD", size=0),
)
# Setup schema aliases
aliases = dict(
S="scalar",
F="fix",
af="fix.array",
V="var",
av="var.array",
vla="av",
)
for source, target in aliases.iteritems():
schema.getAliasMap().set(source, target)
# Create two rows of fake data...
self.rows = (
(
0, -2147483648, -9223372036854775808, 1.0, math.pi,
False, Angle(1.0),
"ab ",
np.array([0, 65535], dtype=np.uint16),
np.array([-2147483648, 2147483647], dtype=np.int32),
np.array([1.0, 2.0], dtype=np.float32),
np.array([math.pi, math.e], dtype=np.float64),
np.array(range(0), dtype=np.uint16),
np.array(range(1), dtype=np.int32),
np.array(range(3), dtype=np.float32),
np.array(range(4), dtype=np.float64),
),
(
65535, 2147483647, 9223372036854775807, 2.0, math.e,
True, Angle(2.0), "",
np.array(range(2), dtype=np.uint16),
np.array(range(2), dtype=np.int32),
np.array(range(2), dtype=np.float32),
np.array(range(2), dtype=np.float64),
np.array([], dtype=np.uint16),
np.array([], dtype=np.int32),
np.array([], dtype=np.float32),
np.array([], dtype=np.float64),
)
)
# and a corresponding catalog
self.catalog = afw_table.BaseCatalog(schema)
for row in self.rows:
record = self.catalog.addNew()
for i, k in enumerate(keys):
record.set(k, row[i])
# Finally, choose table/view names that are unique with very high
# probability.
suffix = uuid.uuid4().hex
self.table_name = "catalog_" + suffix
self.view_name = "view_" + suffix
def _fgcmMakeVisitCatalog(self, butler, dataRefs):
"""
Make a visit catalog with all the key data from each visit
Parameters
----------
butler: lsst.daf.persistence.Butler
dataRefs: list of lsst.daf.persistence.ButlerDataRef
Data references for the input visits
If this is an empty list, all visits with src catalogs in
the repository are used.
Only one individual dataRef from a visit need be specified
and the code will find the other source catalogs from
each visit
Returns
-------
visitCat: afw.table.BaseCatalog
"""
startTime = time.time()
camera = butler.get('camera')
nCcd = len(camera)
if len(dataRefs) == 0:
# We did not specify any datarefs, so find all of them
if not self.config.checkAllCcds:
# Faster mode, scan through referenceCCD
allVisits = butler.queryMetadata(
'src',
format=[self.config.visitDataRefName, 'filter'],
dataId={
self.config.ccdDataRefName: self.config.referenceCCD
})
srcVisits = []
srcCcds = []
for dataset in allVisits:
if (butler.datasetExists('src',
dataId={
self.config.visitDataRefName:
dataset[0],
self.config.ccdDataRefName:
self.config.referenceCCD
})):
srcVisits.append(dataset[0])
srcCcds.append(self.config.referenceCCD)
else:
# Slower mode, check all CCDs
allVisits = butler.queryMetadata(
'src', format=[self.config.visitDataRefName, 'filter'])
srcVisits = []
srcCcds = []
for dataset in allVisits:
if dataset[0] in srcVisits:
continue
for ccd in xrange(nCcd):
if (butler.datasetExists(
'src',
dataId={
self.config.visitDataRefName: dataset[0],
self.config.ccdDataRefName: ccd
})):
srcVisits.append(dataset[0])
srcCcds.append(ccd)
# Once we find that a butler dataset exists, break out
break
else:
# get the visits from the datarefs, only for referenceCCD
srcVisits = [
d.dataId[self.config.visitDataRefName] for d in dataRefs
if d.dataId[self.config.ccdDataRefName] ==
self.config.referenceCCD
]
srcCcds = [self.config.referenceCCD] * len(srcVisits)
# Sort the visits for searching/indexing
srcVisits.sort()
self.log.info("Found %d visits in %.2f s" %
(len(srcVisits), time.time() - startTime))
schema = afwTable.Schema()
schema.addField('visit', type=np.int32, doc="Visit number")
schema.addField('filtername', type=str, size=2, doc="Filter name")
schema.addField('telra', type=np.float64, doc="Pointing RA (deg)")
schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)")
schema.addField('telha',
type=np.float64,
doc="Pointing Hour Angle (deg)")
schema.addField('mjd', type=np.float64, doc="MJD of visit")
schema.addField('exptime', type=np.float32, doc="Exposure time")
schema.addField('pmb', type=np.float32, doc="Pressure (millibar)")
schema.addField('psfsigma',
type=np.float32,
doc="PSF sigma (reference CCD)")
schema.addField('deltaaper', type=np.float32, doc="Delta-aperture")
schema.addField('skybackground',
type=np.float32,
doc="Sky background (ADU) (reference CCD)")
# the following field is not used yet
schema.addField('deepflag', type=np.int32, doc="Deep observation")
schema.addField('scaling',
type='ArrayD',
doc="Scaling applied due to flat adjustment",
size=len(camera))
visitCat = afwTable.BaseCatalog(schema)
visitCat.table.preallocate(len(srcVisits))
startTime = time.time()
# reading in a small bbox is faster for non-gzipped images
bbox = afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.PointI(1, 1))
# now loop over visits and get the information
for i, srcVisit in enumerate(srcVisits):
# We don't use the bypasses since we need the psf info which does
# not have a bypass
dataId = {
self.config.visitDataRefName: srcVisit,
self.config.ccdDataRefName: srcCcds[i]
}
exp = butler.get('calexp_sub',
dataId=dataId,
bbox=bbox,
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
visitInfo = exp.getInfo().getVisitInfo()
f = exp.getFilter()
psf = exp.getPsf()
rec = visitCat.addNew()
rec['visit'] = srcVisit
rec['filtername'] = f.getName()
radec = visitInfo.getBoresightRaDec()
rec['telra'] = radec.getRa().asDegrees()
rec['teldec'] = radec.getDec().asDegrees()
rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
rec['exptime'] = visitInfo.getExposureTime()
# convert from Pa to millibar
# Note that I don't know if this unit will need to be per-camera config
rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
rec['deepflag'] = 0
rec['scaling'][:] = 1.0
rec['deltaaper'] = 0.0
rec['psfsigma'] = psf.computeShape().getDeterminantRadius()
if butler.datasetExists('calexpBackground', dataId=dataId):
# Get background for reference CCD
# This approximation is good enough for now
bgStats = (
bg[0].getStatsImage().getImage().array
for bg in butler.get('calexpBackground', dataId=dataId))
rec['skybackground'] = sum(
np.median(bg[np.isfinite(bg)]) for bg in bgStats)
else:
rec['skybackground'] = -1.0
# Compute flat scaling if desired...
if self.config.renormalizeFlats:
self.log.info("Reading flats for renormalizeFlats")
scalingValues = self._computeFlatScaling(butler, visitCat)
visitCat['scaling'] *= scalingValues
self.log.info("Found all VisitInfo in %.2f s" %
(time.time() - startTime))
# and now persist it
butler.put(visitCat, 'fgcmVisitCatalog')
return visitCat
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# Disposable schema suppresses warning from SourceDetectionTask.__init__
self.makeSubtask("detection", schema=afwTable.Schema())
self.makeSubtask("interpolate")
self.makeSubtask("subtractBackground")
def __init__(self,
butler=None,
schema=None,
refObjLoader=None,
reuse=tuple(),
**kwargs):
"""!
@param[in] butler: the butler can be used to retrieve schema or passed to the refObjLoader constructor
in case it is needed.
@param[in] schema: the schema of the source detection catalog used as input.
@param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
catalog. May be None if the butler argument is provided or all steps requiring a reference
catalog are disabled.
"""
BatchPoolTask.__init__(self, **kwargs)
if schema is None:
assert butler is not None, "Butler not provided"
schema = butler.get(self.config.coaddName + "Coadd_det_schema",
immediate=True).schema
self.butler = butler
self.reuse = tuple(reuse)
self.makeSubtask("detectCoaddSources")
self.makeSubtask("mergeCoaddDetections", schema=schema)
if self.config.measureCoaddSources.inputCatalog.startswith(
"deblended"):
# Ensure that the output from deblendCoaddSources matches the input to measureCoaddSources
self.measurementInput = self.config.measureCoaddSources.inputCatalog
self.deblenderOutput = []
if self.config.deblendCoaddSources.simultaneous:
self.deblenderOutput.append("deblendedModel")
else:
self.deblenderOutput.append("deblendedFlux")
if self.measurementInput not in self.deblenderOutput:
err = "Measurement input '{0}' is not in the list of deblender output catalogs '{1}'"
raise ValueError(
err.format(self.measurementInput, self.deblenderOutput))
self.makeSubtask(
"deblendCoaddSources",
schema=afwTable.Schema(self.mergeCoaddDetections.schema),
peakSchema=afwTable.Schema(
self.mergeCoaddDetections.merged.getPeakSchema()),
butler=butler)
measureInputSchema = afwTable.Schema(
self.deblendCoaddSources.schema)
else:
measureInputSchema = afwTable.Schema(
self.mergeCoaddDetections.schema)
self.makeSubtask("measureCoaddSources",
schema=measureInputSchema,
peakSchema=afwTable.Schema(
self.mergeCoaddDetections.merged.getPeakSchema()),
refObjLoader=refObjLoader,
butler=butler)
self.makeSubtask("mergeCoaddMeasurements",
schema=afwTable.Schema(
self.measureCoaddSources.schema))
self.makeSubtask("forcedPhotCoadd",
refSchema=afwTable.Schema(
self.mergeCoaddMeasurements.schema))
if self.config.hasFakes:
self.coaddType = "fakes_" + self.config.coaddName
else:
self.coaddType = self.config.coaddName
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize):
"""
Make the zeropoint schema
Parameters
----------
superStarChebyshevSize: `int`
Length of the superstar chebyshev array
zptChebyshevSize: `int`
Length of the zeropoint chebyshev array
Returns
-------
zptSchema: `lsst.afw.table.schema`
"""
zptSchema = afwTable.Schema()
zptSchema.addField('visit', type=np.int32, doc='Visit number')
zptSchema.addField('ccd', type=np.int32, doc='CCD number')
zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: '
'1: Photometric, used in fit; '
'2: Photometric, not used in fit; '
'4: Non-photometric, on partly photometric night; '
'8: Non-photometric, on non-photometric night; '
'16: No zeropoint could be determined; '
'32: Too few stars for reliable gray computation'))
zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)')
zptSchema.addField('fgcmZptErr', type=np.float64,
doc='Error on zeropoint, estimated from repeatability + number of obs')
zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2,
doc='maximum x/maximum y to scale to apply chebyshev parameters')
zptSchema.addField('fgcmfZptCheb', type='ArrayD',
size=zptChebyshevSize,
doc='Chebyshev parameters (flattened) for zeropoint')
zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD',
size=superStarChebyshevSize,
doc='Chebyshev parameters (flattened) for superStarFlat')
zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband')
zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral')
zptSchema.addField('fgcmR0', type=np.float64,
doc='Retrieved i0 integral, estimated from stars (only for flag 1)')
zptSchema.addField('fgcmR10', type=np.float64,
doc='Retrieved i10 integral, estimated from stars (only for flag 1)')
zptSchema.addField('fgcmGry', type=np.float64,
doc='Estimated gray extinction relative to atmospheric solution; '
'only for fgcmFlag <= 4 (see fgcmFlag) ')
zptSchema.addField('fgcmDeltaChrom', type=np.float64,
doc='Mean chromatic correction for stars in this ccd; '
'only for fgcmFlag <= 4 (see fgcmFlag)')
zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd')
zptSchema.addField('fgcmTilings', type=np.float64,
doc='Number of photometric tilings used for solution for ccd')
zptSchema.addField('fgcmFpGry', type=np.float64,
doc='Average gray extinction over the full focal plane '
'(same for all ccds in a visit)')
zptSchema.addField('fgcmFpGryBlue', type=np.float64,
doc='Average gray extinction over the full focal plane '
'for 25% bluest stars')
zptSchema.addField('fgcmFpGryBlueErr', type=np.float64,
doc='Error on Average gray extinction over the full focal plane '
'for 25% bluest stars')
zptSchema.addField('fgcmFpGryRed', type=np.float64,
doc='Average gray extinction over the full focal plane '
'for 25% reddest stars')
zptSchema.addField('fgcmFpGryRedErr', type=np.float64,
doc='Error on Average gray extinction over the full focal plane '
'for 25% reddest stars')
zptSchema.addField('fgcmFpVar', type=np.float64,
doc='Variance of gray extinction over the full focal plane '
'(same for all ccds in a visit)')
zptSchema.addField('fgcmDust', type=np.float64,
doc='Gray dust extinction from the primary/corrector'
'at the time of the exposure')
zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction')
zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm')
zptSchema.addField('fgcmDeltaMagBkg', type=np.float64,
doc=('Local background correction from brightest percentile '
'(value set by deltaMagBkgOffsetPercentile) calibration '
'stars.'))
zptSchema.addField('exptime', type=np.float32, doc='Exposure time')
zptSchema.addField('filtername', type=str, size=10, doc='Filter name')
return zptSchema
def _fgcmMakeLut(self, butler):
"""
Make a FGCM Look-up Table
Parameters
----------
butler: lsst.daf.persistence.Butler
(used for mapper information)
"""
# need the camera for the detectors
camera = butler.get('camera')
# number of ccds from the length of the camera iterator
nCcd = len(camera)
self.log.info("Found %d ccds for look-up table" % (nCcd))
# Load in optics, etc.
self._loadThroughputs(butler)
# create the stub of the lutConfig
lutConfig = {}
lutConfig['logger'] = self.log
lutConfig['filterNames'] = self.config.filterNames
lutConfig['stdFilterNames'] = self.config.stdFilterNames
lutConfig['nCCD'] = nCcd
# atmosphereTable already validated if available
if self.config.atmosphereTableName is not None:
lutConfig['atmosphereTableName'] = self.config.atmosphereTableName
else:
# use the regular paramters (also validated if needed)
lutConfig['elevation'] = self.config.parameters.elevation
lutConfig['pmbRange'] = self.config.parameters.pmbRange
lutConfig['pmbSteps'] = self.config.parameters.pmbSteps
lutConfig['pwvRange'] = self.config.parameters.pwvRange
lutConfig['pwvSteps'] = self.config.parameters.pwvSteps
lutConfig['o3Range'] = self.config.parameters.o3Range
lutConfig['o3Steps'] = self.config.parameters.o3Steps
lutConfig['tauRange'] = self.config.parameters.tauRange
lutConfig['tauSteps'] = self.config.parameters.tauSteps
lutConfig['alphaRange'] = self.config.parameters.alphaRange
lutConfig['alphaSteps'] = self.config.parameters.alphaSteps
lutConfig['zenithRange'] = self.config.parameters.zenithRange
lutConfig['zenithSteps'] = self.config.parameters.zenithSteps
lutConfig['pmbStd'] = self.config.parameters.pmbStd
lutConfig['pwvStd'] = self.config.parameters.pwvStd
lutConfig['o3Std'] = self.config.parameters.o3Std
lutConfig['tauStd'] = self.config.parameters.tauStd
lutConfig['alphaStd'] = self.config.parameters.alphaStd
lutConfig['airmassStd'] = self.config.parameters.airmassStd
lutConfig['lambdaRange'] = self.config.parameters.lambdaRange
lutConfig['lambdaStep'] = self.config.parameters.lambdaStep
lutConfig['lambdaNorm'] = self.config.parameters.lambdaNorm
# make the lut object
self.log.info("Making the LUT maker object")
self.fgcmLutMaker = fgcm.FgcmLUTMaker(lutConfig)
# generate the throughput dictionary. Fun!
# do this internally here at first. Later, break it into its own thing
# these will be in Angstroms
# note that lambdaStep is currently in nm, because dumb. convert to A
throughputLambda = np.arange(
self.fgcmLutMaker.lambdaRange[0],
self.fgcmLutMaker.lambdaRange[1] +
self.fgcmLutMaker.lambdaStep * 10,
self.fgcmLutMaker.lambdaStep * 10.)
self.log.info("Built throughput lambda, %.1f-%.1f, step %.2f" %
(throughputLambda[0], throughputLambda[-1],
throughputLambda[1] - throughputLambda[0]))
throughputDict = {}
for i, filterName in enumerate(self.config.filterNames):
tDict = {}
tDict['LAMBDA'] = throughputLambda
for ccdIndex, detector in enumerate(camera):
tDict[ccdIndex] = self._getThroughputDetector(
detector, filterName, throughputLambda)
throughputDict[filterName] = tDict
# set the throughputs
self.fgcmLutMaker.setThroughputs(throughputDict)
# make the LUT
self.log.info("Making LUT")
self.fgcmLutMaker.makeLUT()
# and save the LUT
lutSchema = afwTable.Schema()
# new version, which gets around the afwTable row length limitation
# each LUT will be saved in a different row
# there is overhead of the arrays that we only need one copy, but this
# is going to be insignificant overall
# build the index values
comma = ','
filterNameString = comma.join(self.config.filterNames)
stdFilterNameString = comma.join(self.config.stdFilterNames)
atmosphereTableName = 'NoTableWasUsed'
if self.config.atmosphereTableName is not None:
atmosphereTableName = self.config.atmosphereTableName
lutSchema.addField('tablename',
type=str,
doc='Atmosphere table name',
size=len(atmosphereTableName))
lutSchema.addField('elevation',
type=float,
doc="Telescope elevation used for LUT")
lutSchema.addField('filternames',
type=str,
doc='filterNames in LUT',
size=len(filterNameString))
lutSchema.addField('stdfilternames',
type=str,
doc='Standard filterNames in LUT',
size=len(stdFilterNameString))
lutSchema.addField('pmb',
type='ArrayD',
doc='Barometric Pressure',
size=self.fgcmLutMaker.pmb.size)
lutSchema.addField('pmbfactor',
type='ArrayD',
doc='PMB scaling factor',
size=self.fgcmLutMaker.pmb.size)
lutSchema.addField('pmbelevation',
type=np.float64,
doc='PMB Scaling at elevation')
lutSchema.addField('pwv',
type='ArrayD',
doc='Preciptable Water Vapor',
size=self.fgcmLutMaker.pwv.size)
lutSchema.addField('o3',
type='ArrayD',
doc='Ozone',
size=self.fgcmLutMaker.o3.size)
lutSchema.addField('tau',
type='ArrayD',
doc='Aerosol optical depth',
size=self.fgcmLutMaker.tau.size)
lutSchema.addField('lambdanorm', type=np.float64, doc='AOD wavelength')
lutSchema.addField('alpha',
type='ArrayD',
doc='Aerosol alpha',
size=self.fgcmLutMaker.alpha.size)
lutSchema.addField('zenith',
type='ArrayD',
doc='Zenith angle',
size=self.fgcmLutMaker.zenith.size)
lutSchema.addField('nccd', type=np.int32, doc='Number of CCDs')
# and the standard values
lutSchema.addField('pmbstd', type=np.float64, doc='PMB Standard')
lutSchema.addField('pwvstd', type=np.float64, doc='PWV Standard')
lutSchema.addField('o3std', type=np.float64, doc='O3 Standard')
lutSchema.addField('taustd', type=np.float64, doc='Tau Standard')
lutSchema.addField('alphastd', type=np.float64, doc='Alpha Standard')
lutSchema.addField('zenithstd',
type=np.float64,
doc='Zenith angle Standard')
lutSchema.addField('lambdarange',
type='ArrayD',
doc='Wavelength range',
size=2)
lutSchema.addField('lambdastep',
type=np.float64,
doc='Wavelength step')
lutSchema.addField('lambdastd',
type='ArrayD',
doc='Standard Wavelength',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('lambdastdfilter',
type='ArrayD',
doc='Standard Wavelength (raw)',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i0std',
type='ArrayD',
doc='I0 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i1std',
type='ArrayD',
doc='I1 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('i10std',
type='ArrayD',
doc='I10 Standard',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('lambdab',
type='ArrayD',
doc='Wavelength for passband (no atm)',
size=len(self.fgcmLutMaker.filterNames))
lutSchema.addField('atmlambda',
type='ArrayD',
doc='Atmosphere wavelengths (A)',
size=self.fgcmLutMaker.atmLambda.size)
lutSchema.addField('atmstdtrans',
type='ArrayD',
doc='Standard Atmosphere Throughput',
size=self.fgcmLutMaker.atmStdTrans.size)
# and the look-up-tables
lutSchema.addField('luttype',
type=str,
size=20,
doc='Look-up table type')
lutSchema.addField('lut',
type='ArrayF',
doc='Look-up table for luttype',
size=self.fgcmLutMaker.lut['I0'].size)
lutCat = afwTable.BaseCatalog(lutSchema)
lutCat.table.preallocate(14)
# first fill the first index
rec = lutCat.addNew()
rec['tablename'] = atmosphereTableName
rec['elevation'] = self.fgcmLutMaker.atmosphereTable.elevation
rec['filternames'] = filterNameString
rec['stdfilternames'] = stdFilterNameString
rec['pmb'][:] = self.fgcmLutMaker.pmb
rec['pmbfactor'][:] = self.fgcmLutMaker.pmbFactor
rec['pmbelevation'] = self.fgcmLutMaker.pmbElevation
rec['pwv'][:] = self.fgcmLutMaker.pwv
rec['o3'][:] = self.fgcmLutMaker.o3
rec['tau'][:] = self.fgcmLutMaker.tau
rec['lambdanorm'] = self.fgcmLutMaker.lambdaNorm
rec['alpha'][:] = self.fgcmLutMaker.alpha
rec['zenith'][:] = self.fgcmLutMaker.zenith
rec['nccd'] = self.fgcmLutMaker.nCCD
rec['pmbstd'] = self.fgcmLutMaker.pmbStd
rec['pwvstd'] = self.fgcmLutMaker.pwvStd
rec['o3std'] = self.fgcmLutMaker.o3Std
rec['taustd'] = self.fgcmLutMaker.tauStd
rec['alphastd'] = self.fgcmLutMaker.alphaStd
rec['zenithstd'] = self.fgcmLutMaker.zenithStd
rec['lambdarange'][:] = self.fgcmLutMaker.lambdaRange
rec['lambdastep'] = self.fgcmLutMaker.lambdaStep
rec['lambdastd'][:] = self.fgcmLutMaker.lambdaStd
rec['lambdastdfilter'][:] = self.fgcmLutMaker.lambdaStdFilter
rec['i0std'][:] = self.fgcmLutMaker.I0Std
rec['i1std'][:] = self.fgcmLutMaker.I1Std
rec['i10std'][:] = self.fgcmLutMaker.I10Std
rec['lambdab'][:] = self.fgcmLutMaker.lambdaB
rec['atmlambda'][:] = self.fgcmLutMaker.atmLambda
rec['atmstdtrans'][:] = self.fgcmLutMaker.atmStdTrans
rec['luttype'] = 'I0'
rec['lut'][:] = self.fgcmLutMaker.lut['I0'].flatten()
# and add the rest
rec = lutCat.addNew()
rec['luttype'] = 'I1'
rec['lut'][:] = self.fgcmLutMaker.lut['I1'].flatten()
derivTypes = [
'D_PMB', 'D_PWV', 'D_O3', 'D_LNTAU', 'D_ALPHA', 'D_SECZENITH',
'D_PMB_I1', 'D_PWV_I1', 'D_O3_I1', 'D_LNTAU_I1', 'D_ALPHA_I1',
'D_SECZENITH_I1'
]
for derivType in derivTypes:
rec = lutCat.addNew()
rec['luttype'] = derivType
rec['lut'][:] = self.fgcmLutMaker.lutDeriv[derivType].flatten()
butler.put(lutCat, 'fgcmLookUpTable')
