def toTableDataTable(self, metadata):
"""Produce linearity catalog from table data
Parameters
----------
metadata : `lsst.daf.base.PropertyList`
Linearizer metadata
Returns
-------
catalog : `lsst.afw.table.BaseCatalog`
Catalog to write
"""
schema = afwTable.Schema()
dimensions = self.tableData.shape
lut = schema.addField("LOOKUP_VALUES",
type='ArrayF',
size=dimensions[1],
doc="linearity lookup data")
catalog = afwTable.BaseCatalog(schema)
catalog.resize(dimensions[0])
for ii in range(dimensions[0]):
catalog[ii][lut] = np.array(self.tableData[ii], dtype=np.float32)
metadata["LINEARITY_LOOKUP"] = True
catalog.setMetadata(metadata)
return catalog
def makeAtmCat(atmSchema, atmStruct):
"""
Make the atmosphere catalog for persistence
Parameters
----------
atmSchema: `lsst.afw.table.Schema`
Atmosphere catalog schema
atmStruct: `numpy.ndarray`
Atmosphere structure from fgcm
Returns
-------
atmCat: `lsst.afw.table.BaseCatalog`
Atmosphere catalog for persistence
"""
atmCat = afwTable.BaseCatalog(atmSchema)
atmCat.resize(atmStruct.size)
atmCat['visit'][:] = atmStruct['VISIT']
atmCat['pmb'][:] = atmStruct['PMB']
atmCat['pwv'][:] = atmStruct['PWV']
atmCat['tau'][:] = atmStruct['TAU']
atmCat['alpha'][:] = atmStruct['ALPHA']
atmCat['o3'][:] = atmStruct['O3']
atmCat['secZenith'][:] = atmStruct['SECZENITH']
atmCat['cTrans'][:] = atmStruct['CTRANS']
atmCat['lamStd'][:] = atmStruct['LAMSTD']
return atmCat
def _wrap_result(data,
column_names,
table=None,
index_col=None,
coerce_float=True,
column_dtypes=None,
parse_dates=None):
"""Wrap result set of query in a afw table """
result_size = len(data)
# Turn into columns first
from pandas import lib
data = list(lib.to_object_array_tuples(data).T)
arrays = [lib.maybe_convert_objects(arr, try_float=True) for arr in data]
_harmonize_columns(arrays, column_names, table, column_dtypes, parse_dates)
schema = afw_table.Schema()
# build schema
for i, column_name in enumerate(column_names):
column_type = arrays[i].dtype
schema.addField(column_name, type=column_type.type)
catalog = afw_table.BaseCatalog(schema)
# Preallocate rows based on first column length
catalog.preallocate(result_size)
for i in range(result_size):
record = catalog.addNew()
for column_i in range(len(column_names)):
record.set(column_names[column_i], arrays[column_i][i])
return catalog
def _fgcmMakeVisitCatalog(self, butler, dataRefs):
"""
Make a visit catalog with all the key data from each visit
Parameters
----------
butler: `lsst.daf.persistence.Butler`
dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef`
Data references for the input visits.
If this is an empty list, all visits with src catalogs in
the repository are used.
Only one individual dataRef from a visit need be specified
and the code will find the other source catalogs from
each visit.
Returns
-------
visitCat: `afw.table.BaseCatalog`
"""
startTime = time.time()
camera = butler.get('camera')
nCcd = len(camera)
# TODO: related to DM-13730, this dance of looking for source visits
# will be unnecessary with Gen3 Butler. This should be part of
# DM-13730.
if len(dataRefs) == 0:
srcVisits, srcCcds = self._findSourceVisits(butler, nCcd)
else:
# get the visits from the datarefs, only for referenceCCD
srcVisits = [
d.dataId[self.config.visitDataRefName] for d in dataRefs
if d.dataId[self.config.ccdDataRefName] ==
self.config.referenceCCD
]
srcCcds = [self.config.referenceCCD] * len(srcVisits)
# Sort the visits for searching/indexing
srcVisits.sort()
self.log.info("Found %d visits in %.2f s" %
(len(srcVisits), time.time() - startTime))
schema = self._makeFgcmVisitSchema(nCcd)
visitCat = afwTable.BaseCatalog(schema)
visitCat.table.preallocate(len(srcVisits))
startTime = time.time()
self._fillVisitCatalog(butler, visitCat, srcVisits, srcCcds)
self.log.info("Found all VisitInfo in %.2f s" %
(time.time() - startTime))
return visitCat
def _performTransform(self, transformClass, inCat, doExtend=True):
"""Operate on inCat with a transform of class transformClass"""
mapper = afwTable.SchemaMapper(inCat.schema)
config = SillyCentroidConfig()
transform = transformClass(config, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
if doExtend:
outCat.extend(inCat, mapper=mapper)
transform(inCat, outCat, makeWcs(), afwImage.Calib())
return outCat
def testApplyCppTransform(self):
"""Test that we can apply a simple C++ transform"""
inCat = self._generateCatalog()
sillyControl = testLib.SillyCentroidControl()
mapper = afwTable.SchemaMapper(inCat.schema)
sillyTransform = testLib.SillyTransform(sillyControl, self.pluginName, mapper)
outCat = afwTable.BaseCatalog(mapper.getOutputSchema())
outCat.extend(inCat, mapper=mapper)
self.assertEqual(len(inCat), len(outCat))
sillyTransform(inCat, outCat, makeWcs(), afwImage.Calib())
self._checkSillyOutputs(inCat, outCat)
def makeZptCat(zptSchema, zpStruct):
"""
Make the zeropoint catalog for persistence
Parameters
----------
zptSchema: `lsst.afw.table.Schema`
Zeropoint catalog schema
zpStruct: `numpy.ndarray`
Zeropoint structure from fgcm
Returns
-------
zptCat: `afwTable.BaseCatalog`
Zeropoint catalog for persistence
"""
zptCat = afwTable.BaseCatalog(zptSchema)
zptCat.reserve(zpStruct.size)
for filterName in zpStruct['FILTERNAME']:
rec = zptCat.addNew()
rec['filtername'] = filterName.decode('utf-8')
zptCat['visit'][:] = zpStruct['VISIT']
zptCat['ccd'][:] = zpStruct['CCD']
zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG']
zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT']
zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR']
zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX']
zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB']
zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB']
zptCat['fgcmI0'][:] = zpStruct['FGCM_I0']
zptCat['fgcmI10'][:] = zpStruct['FGCM_I10']
zptCat['fgcmR0'][:] = zpStruct['FGCM_R0']
zptCat['fgcmR10'][:] = zpStruct['FGCM_R10']
zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY']
zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM']
zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR']
zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS']
zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY']
zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0]
zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0]
zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2]
zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2]
zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR']
zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST']
zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT']
zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR']
zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG']
zptCat['exptime'][:] = zpStruct['EXPTIME']
return zptCat
def _get_afw_table():
schema = afw_table.Schema()
aa = schema.addField("a", type=np.int64, doc="a")
bb = schema.addField("b", type=np.float64, doc="b")
cat = afw_table.BaseCatalog(schema)
row = cat.addNew()
row.set(aa, 12345)
row.set(bb, 1.2345)
row = cat.addNew()
row.set(aa, 4321)
row.set(bb, 4.123)
return cat
def fgcmMakeVisitCatalog(self,
camera,
groupedDataRefs,
visitCatDataRef=None,
inVisitCat=None):
"""
Make a visit catalog with all the keys from each visit
Parameters
----------
camera: `lsst.afw.cameraGeom.Camera`
Camera from the butler
groupedDataRefs: `dict`
Dictionary with visit keys, and `list`s of
`lsst.daf.persistence.ButlerDataRef`
visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
Dataref to write visitCat for checkpoints
inVisitCat: `afw.table.BaseCatalog`
Input (possibly incomplete) visit catalog
Returns
-------
visitCat: `afw.table.BaseCatalog`
"""
self.log.info("Assembling visitCatalog from %d %ss" %
(len(groupedDataRefs), self.config.visitDataRefName))
nCcd = len(camera)
if inVisitCat is None:
schema = self._makeFgcmVisitSchema(nCcd)
visitCat = afwTable.BaseCatalog(schema)
visitCat.reserve(len(groupedDataRefs))
for i, visit in enumerate(sorted(groupedDataRefs)):
rec = visitCat.addNew()
rec['visit'] = visit
rec['used'] = 0
rec['sources_read'] = 0
else:
visitCat = inVisitCat
# No matter what, fill the catalog. This will check if it was
# already read.
self._fillVisitCatalog(visitCat,
groupedDataRefs,
visitCatDataRef=visitCatDataRef)
return visitCat
def buildCatalog(self, momentPrior):
outCat = afwTable.BaseCatalog(self.schema)
for temp in momentPrior.templates:
rec = outCat.addNew()
rec.set(self.mKey, temp.m)
rec.set(self.dmKey, temp.dm.flatten())
rec.set(self.dxyKey, temp.dxy.flatten())
rec.set(self.ndaKey, temp.nda)
rec.set(self.idKey, temp.id)
if self.config.zFile:
rec.set(self.zKey, self.z_list[temp.id])
# rec.set(self.zIdKey, self.z_id_list[temp.id])
return outCat
def concatenate(catalogList):
"""Concatenate multiple catalogs (FITS tables from lsst.afw.table)"""
catalogList = [afwTable.BaseCatalog.readFits(c) if isinstance(c, basestring) else c for c in catalogList]
schema = catalogList[0].schema
for i, c in enumerate(catalogList[1:]):
if c.schema != schema:
raise RuntimeError("Schema for catalog %d not consistent" % (i+1))
out = afwTable.BaseCatalog(schema)
num = reduce(lambda n, c: n + len(c), catalogList, 0)
out.preallocate(num)
for catalog in catalogList:
for record in catalog:
out.append(out.table.copyRecord(record))
return out
def __init__(self, *args, **kwargs):
"""
"""
super(BuildCovarianceTask, self).__init__(*args, **kwargs)
self.ncolors = 0
self.bfd = dbfd.BFDConfig(use_conc=self.config.use_conc,
use_mag=self.config.use_mag,
ncolors=self.ncolors)
self.n_even = self.bfd.BFDConfig.MSIZE
self.n_odd = self.bfd.BFDConfig.XYSIZE
self.size_even = self.n_even * (self.n_even + 1) // 2
self.size_odd = self.n_odd * (self.n_odd + 1) // 2
self.schema = afwTable.Schema()
self.labelKey = self.schema.addField("label",
type=str,
doc="name of bin",
size=10)
self.minKey = self.schema.addField("min",
type=float,
doc="minimum value of the variance")
self.maxKey = self.schema.addField("max",
type=float,
doc="maximum value of the variance")
self.isoCovEvenKey = self.schema.addField(
"isoCovEven",
doc="isotropized moment covariance matrix",
type="ArrayF",
size=self.size_even)
self.isoCovOddKey = self.schema.addField(
"isoCovOdd",
doc="isotropized moment covariance matrix",
type="ArrayF",
size=self.size_odd)
self.covEvenKey = self.schema.addField("covEven",
doc="moment covariance matrix",
type="ArrayF",
size=self.size_even)
self.covOddKey = self.schema.addField("covOdd",
doc="moment covariance matrix",
type="ArrayF",
size=self.size_odd)
self.catalog = afwTable.BaseCatalog(self.schema)
def _makeForcedSourceCatalog(objects):
"""Make a catalog containing a bunch of DiaFourceSources associated with
the input diaObjects.
"""
# make some sources
schema = afwTable.Schema()
schema.addField("diaObjectId", "L")
schema.addField("ccdVisitId", "L")
schema.addField("flags", "L")
catalog = afwTable.BaseCatalog(schema)
oids = []
for obj in objects:
record = catalog.addNew()
record.set("diaObjectId", obj["id"])
record.set("ccdVisitId", 1)
record.set("flags", 0)
oids.append(obj["id"])
return catalog, oids
def run(self, inputCat, wcs, calib):
"""!Transform raw source measurements to calibrated quantities.
@param[in] inputCat SourceCatalog of sources to transform.
@param[in] wcs The world coordinate system under which transformations will take place.
@param[in] calib The calibration under which transformations will take place.
@return A BaseCatalog containing the transformed measurements.
"""
outputCat = afwTable.BaseCatalog(self.mapper.getOutputSchema())
outputCat.extend(inputCat, mapper=self.mapper)
# Transforms may use a ColumnView on the input and output catalogs,
# which requires that the data be contiguous in memory.
inputCat = makeContiguous(inputCat)
outputCat = makeContiguous(outputCat)
for transform in self.transforms:
transform(inputCat, outputCat, wcs, calib)
return outputCat
def _makeSourceCatalog(objects):
"""Make a catalog containing a bunch of DiaSources associated with the
input diaObjects.
"""
# make some sources
schema = make_minimal_dia_source_schema()
catalog = afwTable.BaseCatalog(schema)
oids = []
for sid, obj in enumerate(objects):
record = catalog.addNew()
record.set("id", sid)
record.set("ccdVisitId", 1)
record.set("diaObjectId", obj["id"])
record.set("parent", 0)
record.set("coord_ra", obj["coord_ra"])
record.set("coord_dec", obj["coord_dec"])
record.set("flags", 0)
record.set("pixelId", obj["pixelId"])
oids.append(obj["id"])
return catalog, oids
def fgcmMakeVisitCatalog(self, camera, groupedHandles, bkgHandleDict=None):
"""
Make a visit catalog with all the keys from each visit
Parameters
----------
camera: `lsst.afw.cameraGeom.Camera`
Camera from the butler
groupedHandles: `dict` [`list` [`lsst.daf.butler.DeferredDatasetHandle`]]
Dataset handles, grouped by visit.
bkgHandleDict: `dict`, optional
Dictionary of `lsst.daf.butler.DeferredDatasetHandle` for background info.
Returns
-------
visitCat: `afw.table.BaseCatalog`
"""
self.log.info("Assembling visitCatalog from %d visits",
len(groupedHandles))
nCcd = len(camera)
schema = self._makeFgcmVisitSchema(nCcd)
visitCat = afwTable.BaseCatalog(schema)
visitCat.reserve(len(groupedHandles))
visitCat.resize(len(groupedHandles))
visitCat['visit'] = list(groupedHandles.keys())
visitCat['used'] = 0
visitCat['sources_read'] = False
# No matter what, fill the catalog. This will check if it was
# already read.
self._fillVisitCatalog(visitCat,
groupedHandles,
bkgHandleDict=bkgHandleDict)
return visitCat
def run(self, dataRef, selectDataList=[]):
if self.config.fileOutName == "":
if self.config.dirOutName == "" :
dirOutName = dataRef.getButler().mapper.root+"/"+self.config.coaddName+"Coadd-results"
self.log.info("WARNING: the output file will be written in {0:s}.".format(dirOutName))
else:
dirOutName = self.config.dirOutName
fileOutName = "{0}/{1}/{2}/{3}/multiCat-{2}-{3}.fits".format(dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"])
else:
fileOutName = self.config.fileOutName
if os.path.isfile(fileOutName) and not self.config.clobber:
self.log.info("File for %s exists. Exiting..." % (dataRef.dataId))
return
self.log.info("Processing %s" % (dataRef.dataId))
filters = self.config.filters.split("^")
dustCoefs = [float(c) for c in self.config.dustCoefs.split("^") ]
ref = dataRef.get("deepCoadd_ref")
catalogs = dict(self.readCatalog(dataRef, f) for f in filters)
coadds = dict(self.readCoadd(dataRef, f) for f in filters)
ghostFilters = []
if self.config.ghostFilters != "":
ghostFilters = self.config.ghostFilters.split("^")
self.log.info("Ghost filters: %s" % ghostFilters)
# print ref.schema.getOrderedNames()
# print dir(ref.schema)
# print catalogs[filters[0]].schema.getOrderedNames()
# return
fluxMag0 = {}
for f in filters:
fluxMag0[f] = coadds[f].getCalib().getFluxMag0()[0]
self.log.info("Mag ZP for filter {0:s}: {1:f}".format(f, 2.5*np.log10(fluxMag0[f])))
wcs = coadds[filters[0]].getWcs()
pixel_scale = wcs.pixelScale().asDegrees()*3600.0
aperId = [int(a) for a in self.config.aperId.split(",")]
"""display which aperture diameter size will be used
"""
aperSize = []
for j, a in enumerate(aperId):
aperSize.append(catalogs[filters[0]].getMetadata().get("flux_aperture_radii")[a] * 2.0 * pixel_scale)
self.log.info("Diameter of flux apertures: {0:f}\"".format(aperSize[j]))
"""create new table table
"""
mergedSchema = afwTable.Schema()
"""define table fields
"""
fields=[]
fields.append(mergedSchema.addField("id", type="L", doc="Unique id"))
fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]"))
fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]"))
fields.append(mergedSchema.addField("tract", type="I", doc="tract number"))
fields.append(mergedSchema.addField("patch", type="String", size=3, doc="patch number"))
fields.append(mergedSchema.addField("refFilter", type="String", size=10, doc="Name of the filter used as reference"))
fields.append(mergedSchema.addField("countInputs", type="I", doc="Number of input single exposures for the reference filter"))
fields.append(mergedSchema.addField("detRadius", type="F", doc="Determinant radius for the object in the reference filter = sigma if gaussian [arcsec]"))
fields.append(mergedSchema.addField("PSFDetRadius", type="F", doc="Determinant radius for the PSF in the reference filter at the object position = sigma if gaussian [arcsec]"))
fields.append(mergedSchema.addField("cmodel_fracDev", type="F", doc="fraction of flux in de Vaucouleur component"))
fields.append(mergedSchema.addField("blendedness", type="F", doc="Ranges from 0 (unblended) to 1 (blended)"))
fields.append(mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]"))
fields.append(mergedSchema.addField("extendedness", type="F", doc="probability of being extended from PSF/cmodel flux difference"))
fields.append(mergedSchema.addField("hasBadCentroid", type="I", doc="1 if has bad centroid (but not used in islean"))
fields.append(mergedSchema.addField("isSky", type="I", doc="1 if sky object"))
fields.append(mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch"))
fields.append(mergedSchema.addField("isParent", type="I", doc="1 if parent of a deblended object"))
fields.append(mergedSchema.addField("isClean_refFilter", type="I", doc="1 if none of other flags is set for reference filter"))
for f in filters+ghostFilters:
fields.append(mergedSchema.addField("hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if interpolated, saturated, suspect, has CR at center or near bright object for filter {0:s}".format(f)))
for f in filters+ghostFilters:
fields.append(mergedSchema.addField("isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if offImage or in region masked EDGE or NO_DATA for filter {0:s}".format(f)))
"""photometry estimates
"""
photo = ["flux.aperture", "flux.kron", "flux.psf", "cmodel.flux"]
for p in photo:
for f in filters+ghostFilters:
if p == "flux.aperture":
for a in aperSize:
keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_")
fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s} within {2:f}\" diameter aperture".format(p,f,a)))
fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f)))
else:
keyName = (p+"_"+f).replace(".", "_").replace("-", "_")
fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s}".format(p,f)))
fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f)))
fields.append(mergedSchema.addField(p+"_flag".replace(".", "_"), type="I", doc="Highest flag value among filters for {0:s}".format(p)))
"""dust corrections
"""
for f in filters+ghostFilters:
fields.append(mergedSchema.addField(("EB_V_corr_"+f).replace(".", "_").replace("-", "_"), type="F", doc="Milky way dust flux correction for filter {0:s}".format(f)))
"""create table object
"""
merged = afwTable.BaseCatalog(mergedSchema)
N = len(ref)
# count = 0
for i in range(N):
# for i in range(10000,10200):
# for i in range(1,100):
"""create new record
"""
record = merged.addNew()
coord = ref[i].get('coord')
"""record common info from reference filter
"""
record.set(mergedSchema['id'].asKey(), ref[i].get('id'))
record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees())
record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees())
record.set(mergedSchema['tract'].asKey(), dataRef.dataId["tract"])
record.set(mergedSchema['patch'].asKey(), dataRef.dataId["patch"])
record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs'))
record.set(mergedSchema['detRadius'].asKey(), ref[i].get("shape.sdss").getDeterminantRadius()*pixel_scale)
record.set(mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius()*pixel_scale)
record.set(mergedSchema['cmodel_fracDev'].asKey(), ref[i].get('cmodel.fracDev'))
record.set(mergedSchema['blendedness'].asKey(), ref[i].get('blendedness.abs.flux'))
record.set(mergedSchema['extendedness'].asKey(), ref[i].get('classification.extendedness'))
record.set(mergedSchema['hasBadCentroid'].asKey(), int(ref[i].get('centroid.sdss.flags')))
record.set(mergedSchema['isSky'].asKey(), int(ref[i].get('merge.footprint.sky')))
record.set(mergedSchema['isDuplicated'].asKey(), int(not ref[i].get('detect.is-primary')))
record.set(mergedSchema['isParent'].asKey(), int(ref[i].get('deblend.nchild') != 0))
"""record the name of the filter used as reference
"""
for f in filters:
name = afwImage.Filter(afwImage.Filter(f).getId()).getName()
if ref[i].get("merge.measurement."+name):
refName = f.replace(".", "_").replace("-", "_")
record.set(mergedSchema["refFilter"].asKey(), refName)
break
"""photometry measurement flags
"""
for p in photo:
flag = 0
for f in filters:
if catalogs[f][i].get(p+".flags") > flag:
flag = catalogs[f][i].get(p+".flags")
record.set(mergedSchema[p+"_flag".replace(".", "_")].asKey(), flag)
"""record bad photometry flag for each filter
"""
for f in filters:
record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int((catalogs[f][i].get('flags.pixel.offimage')) | (catalogs[f][i].get('flags.pixel.edge'))))
record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int(
(catalogs[f][i].get('flags.pixel.interpolated.center')) \
| (catalogs[f][i].get('flags.pixel.saturated.center')) \
| (catalogs[f][i].get('flags.pixel.suspect.center')) \
| (catalogs[f][i].get('flags.pixel.cr.center')) \
| (catalogs[f][i].get('flags.pixel.bad')) \
| (catalogs[f][i].get('flags.pixel.bright.object.center'))))
for f in ghostFilters:
record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1)
record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1)
"""record isClean flag for reference filter
"""
hasBadPhotometry_refFilter = \
(ref[i].get('flags.pixel.interpolated.center')) \
| (ref[i].get('flags.pixel.saturated.center')) \
| (ref[i].get('flags.pixel.suspect.center')) \
| (ref[i].get('flags.pixel.cr.center')) \
| (ref[i].get('flags.pixel.bad')) \
| (ref[i].get('flags.pixel.bright.object.center'))
isNoData_refFilter = (ref[i].get('flags.pixel.offimage')) | (ref[i].get('flags.pixel.edge'))
isSky_refFilter = ref[i].get('merge.footprint.sky')
isDuplicated_refFilter = not ref[i].get('detect.is-primary')
isParent_refFilter = ref[i].get('deblend.nchild') != 0
record.set(mergedSchema["isClean_refFilter"].asKey(), int(
(not hasBadPhotometry_refFilter) \
& (not isNoData_refFilter) \
& (not isSky_refFilter) \
& (not isDuplicated_refFilter) \
& (not isParent_refFilter)))
#isExtended = (ref[i].get('classification.extendedness'))
#isExtended = (catalogs[f][i].get('flux.kron') > 0.8*catalogs[f][i].get('flux.psf')) | \
# (ref.get("shape.sdss").getDeterminantRadius() > 1.1*ref.get("shape.sdss.psf").getDeterminantRadius())
#if not (isExtended == isExtended):
# isExtended = 0
#record.set(mergedSchema['isExtended'].asKey(), int(isExtended))
"""dust correction
"""
EB_V = self.getDustCorrection(self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey()))
record.set(mergedSchema['EB_V'].asKey(), EB_V)
"""flux in micro Jansky
"""
for f, dc in zip(filters, dustCoefs):
record.set(mergedSchema[("EB_V_corr_"+f).replace(".", "_").replace("-", "_")].asKey(), pow(10.0, +0.4* dc * EB_V))
for p in photo:
if p == "flux.aperture":
for a in aperId:
flux = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p)[a] / fluxMag0[f]
flux_err = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p+".err")[a] / fluxMag0[f]
keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_")
record.set(mergedSchema[keyName].asKey(), flux)
record.set(mergedSchema[keyName+"_err"].asKey(), flux_err)
else:
flux = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p)/fluxMag0[f]
flux_err = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p+".err")/fluxMag0[f]
keyName = (p+"_"+f).replace(".", "_").replace("-", "_")
record.set(mergedSchema[keyName].asKey(), flux)
record.set(mergedSchema[keyName+"_err"].asKey(), flux_err)
# count += 1
"""write catalog
"""
self.log.info("Writing {0:s}".format(fileOutName))
self.mkdir_p(os.path.dirname(fileOutName))
merged.writeFits(fileOutName)
return
def getSchemaCatalogs(self):
"""!Return a dict containing an empty catalog representative of this task's output."""
transformedSrc = afwTable.BaseCatalog(self.mapper.getOutputSchema())
return {self.outputDataset: transformedSrc}
def fgcmMakeAllStarObservations(self,
groupedDataRefs,
visitCat,
calibFluxApertureRadius=None,
visitCatDataRef=None,
starObsDataRef=None,
inStarObsCat=None):
startTime = time.time()
# If both dataRefs are None, then we assume the caller does not
# want to store checkpoint files. If both are set, we will
# do checkpoint files. And if only one is set, this is potentially
# unintentional and we will warn.
if (visitCatDataRef is not None and starObsDataRef is None
or visitCatDataRef is None and starObsDataRef is not None):
self.log.warn(
"Only one of visitCatDataRef and starObsDataRef are set, so "
"no checkpoint files will be persisted.")
if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True."
)
# create our source schema. Use the first valid dataRef
dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
sourceSchema = dataRef.get('src_schema', immediate=True).schema
# Construct a mapping from ccd number to index
camera = dataRef.get('camera')
ccdMapping = {}
for ccdIndex, detector in enumerate(camera):
ccdMapping[detector.getId()] = ccdIndex
approxPixelAreaFields = computeApproxPixelAreaFields(camera)
sourceMapper = self._makeSourceMapper(sourceSchema)
# We also have a temporary catalog that will accumulate aperture measurements
aperMapper = self._makeAperMapper(sourceSchema)
outputSchema = sourceMapper.getOutputSchema()
if inStarObsCat is not None:
fullCatalog = inStarObsCat
comp1 = fullCatalog.schema.compare(outputSchema,
outputSchema.EQUAL_KEYS)
comp2 = fullCatalog.schema.compare(outputSchema,
outputSchema.EQUAL_NAMES)
if not comp1 or not comp2:
raise RuntimeError(
"Existing fgcmStarObservations file found with mismatched schema."
)
else:
fullCatalog = afwTable.BaseCatalog(outputSchema)
# FGCM will provide relative calibration for the flux in config.instFluxField
instFluxKey = sourceSchema[self.config.instFluxField].asKey()
instFluxErrKey = sourceSchema[self.config.instFluxField +
'Err'].asKey()
visitKey = outputSchema['visit'].asKey()
ccdKey = outputSchema['ccd'].asKey()
instMagKey = outputSchema['instMag'].asKey()
instMagErrKey = outputSchema['instMagErr'].asKey()
deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey()
# Prepare local background if desired
if self.config.doSubtractLocalBackground:
localBackgroundFluxKey = sourceSchema[
self.config.localBackgroundFluxField].asKey()
localBackgroundArea = np.pi * calibFluxApertureRadius**2.
aperOutputSchema = aperMapper.getOutputSchema()
instFluxAperInKey = sourceSchema[
self.config.apertureInnerInstFluxField].asKey()
instFluxErrAperInKey = sourceSchema[
self.config.apertureInnerInstFluxField + 'Err'].asKey()
instFluxAperOutKey = sourceSchema[
self.config.apertureOuterInstFluxField].asKey()
instFluxErrAperOutKey = sourceSchema[
self.config.apertureOuterInstFluxField + 'Err'].asKey()
instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
k = 2.5 / np.log(10.)
# loop over visits
for ctr, visit in enumerate(visitCat):
if visit['sources_read']:
continue
expTime = visit['exptime']
nStarInVisit = 0
# Reset the aperture catalog (per visit)
aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
for dataRef in groupedDataRefs[visit['visit']]:
ccdId = dataRef.dataId[self.config.ccdDataRefName]
sources = dataRef.get(datasetType='src',
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
goodSrc = self.sourceSelector.selectSources(sources)
# Need to add a selection based on the local background correction
# if necessary
if self.config.doSubtractLocalBackground:
localBackground = localBackgroundArea * sources[
localBackgroundFluxKey]
bad, = np.where(
(sources[instFluxKey] - localBackground) <= 0.0)
goodSrc.selected[bad] = False
tempCat = afwTable.BaseCatalog(fullCatalog.schema)
tempCat.reserve(goodSrc.selected.sum())
tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
tempCat[visitKey][:] = visit['visit']
tempCat[ccdKey][:] = ccdId
# Compute "instrumental magnitude" by scaling flux with exposure time.
scaledInstFlux = (sources[instFluxKey][goodSrc.selected] *
visit['scaling'][ccdMapping[ccdId]])
tempCat[instMagKey][:] = (-2.5 * np.log10(scaledInstFlux) +
2.5 * np.log10(expTime))
# Compute the change in magnitude from the background offset
if self.config.doSubtractLocalBackground:
# At the moment we only adjust the flux and not the flux
# error by the background because the error on
# base_LocalBackground_instFlux is the rms error in the
# background annulus, not the error on the mean in the
# background estimate (which is much smaller, by sqrt(n)
# pixels used to estimate the background, which we do not
# have access to in this task). In the default settings,
# the annulus is sufficiently large such that these
# additional errors are are negligibly small (much less
# than a mmag in quadrature).
# This is the difference between the mag with background correction
# and the mag without background correction.
tempCat[deltaMagBkgKey][:] = (
-2.5 *
np.log10(sources[instFluxKey][goodSrc.selected] -
localBackground[goodSrc.selected]) - -2.5 *
np.log10(sources[instFluxKey][goodSrc.selected]))
else:
tempCat[deltaMagBkgKey][:] = 0.0
# Compute instMagErr from instFluxErr/instFlux, any scaling
# will cancel out.
tempCat[instMagErrKey][:] = k * (
sources[instFluxErrKey][goodSrc.selected] /
sources[instFluxKey][goodSrc.selected])
# Compute the jacobian from an approximate PixelAreaBoundedField
tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate(
tempCat['x'], tempCat['y'])
# Apply the jacobian if configured
if self.config.doApplyWcsJacobian:
tempCat[instMagKey][:] -= 2.5 * np.log10(
tempCat['jacobian'][:])
fullCatalog.extend(tempCat)
# And the aperture information
# This does not need the jacobian because it is all locally relative
tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
tempAperCat.reserve(goodSrc.selected.sum())
tempAperCat.extend(sources[goodSrc.selected],
mapper=aperMapper)
with np.warnings.catch_warnings():
# Ignore warnings, we will filter infinities and
# nans below.
np.warnings.simplefilter("ignore")
tempAperCat[instMagInKey][:] = -2.5 * np.log10(
sources[instFluxAperInKey][goodSrc.selected])
tempAperCat[instMagErrInKey][:] = k * (
sources[instFluxErrAperInKey][goodSrc.selected] /
sources[instFluxAperInKey][goodSrc.selected])
tempAperCat[instMagOutKey][:] = -2.5 * np.log10(
sources[instFluxAperOutKey][goodSrc.selected])
tempAperCat[instMagErrOutKey][:] = k * (
sources[instFluxErrAperOutKey][goodSrc.selected] /
sources[instFluxAperOutKey][goodSrc.selected])
aperVisitCatalog.extend(tempAperCat)
nStarInVisit += len(tempCat)
# Compute the median delta-aper
if not aperVisitCatalog.isContiguous():
aperVisitCatalog = aperVisitCatalog.copy(deep=True)
instMagIn = aperVisitCatalog[instMagInKey]
instMagErrIn = aperVisitCatalog[instMagErrInKey]
instMagOut = aperVisitCatalog[instMagOutKey]
instMagErrOut = aperVisitCatalog[instMagErrOutKey]
ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
& np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
visit['sources_read'] = True
self.log.info(
" Found %d good stars in visit %d (deltaAper = %.3f)" %
(nStarInVisit, visit['visit'], visit['deltaAper']))
if ((ctr % self.config.nVisitsPerCheckpoint) == 0
and starObsDataRef is not None
and visitCatDataRef is not None):
# We need to persist both the stars and the visit catalog which gets
# additional metadata from each visit.
starObsDataRef.put(fullCatalog)
visitCatDataRef.put(visitCat)
self.log.info("Found all good star observations in %.2f s" %
(time.time() - startTime))
return fullCatalog
def fgcmMakeAllStarObservations(self,
groupedHandles,
visitCat,
sourceSchema,
camera,
calibFluxApertureRadius=None):
startTime = time.time()
if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
raise RuntimeError(
"Must set calibFluxApertureRadius if doSubtractLocalBackground is True."
)
# To get the correct output schema, we use the legacy code.
# We are not actually using this mapper, except to grab the outputSchema
sourceMapper = self._makeSourceMapper(sourceSchema)
outputSchema = sourceMapper.getOutputSchema()
# Construct mapping from ccd number to index
ccdMapping = {}
for ccdIndex, detector in enumerate(camera):
ccdMapping[detector.getId()] = ccdIndex
approxPixelAreaFields = computeApproxPixelAreaFields(camera)
fullCatalog = afwTable.BaseCatalog(outputSchema)
visitKey = outputSchema['visit'].asKey()
ccdKey = outputSchema['ccd'].asKey()
instMagKey = outputSchema['instMag'].asKey()
instMagErrKey = outputSchema['instMagErr'].asKey()
deltaMagAperKey = outputSchema['deltaMagAper'].asKey()
# Prepare local background if desired
if self.config.doSubtractLocalBackground:
localBackgroundArea = np.pi * calibFluxApertureRadius**2.
columns = None
k = 2.5 / np.log(10.)
for counter, visit in enumerate(visitCat):
expTime = visit['exptime']
handle = groupedHandles[visit['visit']][-1]
if columns is None:
inColumns = handle.get(component='columns')
columns, detColumn = self._get_sourceTable_visit_columns(
inColumns)
df = handle.get(parameters={'columns': columns})
goodSrc = self.sourceSelector.selectSources(df)
# Need to add a selection based on the local background correction
# if necessary
if self.config.doSubtractLocalBackground:
localBackground = localBackgroundArea * df[
self.config.localBackgroundFluxField].values
use, = np.where((goodSrc.selected)
& ((df[self.config.instFluxField].values -
localBackground) > 0.0))
else:
use, = np.where(goodSrc.selected)
tempCat = afwTable.BaseCatalog(fullCatalog.schema)
tempCat.resize(use.size)
tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
tempCat['x'][:] = df['x'].values[use]
tempCat['y'][:] = df['y'].values[use]
# The "visit" name in the parquet table is hard-coded.
tempCat[visitKey][:] = df['visit'].values[use]
tempCat[ccdKey][:] = df[detColumn].values[use]
tempCat['psf_candidate'] = df[
self.config.psfCandidateName].values[use]
with np.warnings.catch_warnings():
# Ignore warnings, we will filter infinites and nans below
np.warnings.simplefilter("ignore")
instMagInner = -2.5 * np.log10(
df[self.config.apertureInnerInstFluxField].values[use])
instMagErrInner = k * (
df[self.config.apertureInnerInstFluxField +
'Err'].values[use] /
df[self.config.apertureInnerInstFluxField].values[use])
instMagOuter = -2.5 * np.log10(
df[self.config.apertureOuterInstFluxField].values[use])
instMagErrOuter = k * (
df[self.config.apertureOuterInstFluxField +
'Err'].values[use] /
df[self.config.apertureOuterInstFluxField].values[use])
tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter
# Set bad values to illegal values for fgcm.
tempCat[deltaMagAperKey][
~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0
if self.config.doSubtractLocalBackground:
# At the moment we only adjust the flux and not the flux
# error by the background because the error on
# base_LocalBackground_instFlux is the rms error in the
# background annulus, not the error on the mean in the
# background estimate (which is much smaller, by sqrt(n)
# pixels used to estimate the background, which we do not
# have access to in this task). In the default settings,
# the annulus is sufficiently large such that these
# additional errors are are negligibly small (much less
# than a mmag in quadrature).
# This is the difference between the mag with local background correction
# and the mag without local background correction.
tempCat['deltaMagBkg'] = (
-2.5 * np.log10(df[self.config.instFluxField].values[use] -
localBackground[use]) -
-2.5 * np.log10(df[self.config.instFluxField].values[use]))
else:
tempCat['deltaMagBkg'][:] = 0.0
# Need to loop over ccds here
for detector in camera:
ccdId = detector.getId()
# used index for all observations with a given ccd
use2 = (tempCat[ccdKey] == ccdId)
tempCat['jacobian'][use2] = approxPixelAreaFields[
ccdId].evaluate(tempCat['x'][use2], tempCat['y'][use2])
scaledInstFlux = (
df[self.config.instFluxField].values[use[use2]] *
visit['scaling'][ccdMapping[ccdId]])
tempCat[instMagKey][use2] = (-2.5 * np.log10(scaledInstFlux) +
2.5 * np.log10(expTime))
# Compute instMagErr from instFluxErr/instFlux, any scaling
# will cancel out.
tempCat[instMagErrKey][:] = k * (
df[self.config.instFluxField + 'Err'].values[use] /
df[self.config.instFluxField].values[use])
# Apply the jacobian if configured
if self.config.doApplyWcsJacobian:
tempCat[instMagKey][:] -= 2.5 * np.log10(
tempCat['jacobian'][:])
fullCatalog.extend(tempCat)
deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner)
& np.isfinite(instMagOuter)
& np.isfinite(instMagErrOuter))
visit['deltaAper'] = np.median(instMagInner[deltaOk] -
instMagOuter[deltaOk])
visit['sources_read'] = True
self.log.info(
" Found %d good stars in visit %d (deltaAper = %0.3f)",
use.size, visit['visit'], visit['deltaAper'])
self.log.info("Found all good star observations in %.2f s" %
(time.time() - startTime))
return fullCatalog
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = {}
handleDict['camera'] = butlerQC.get(inputRefs.camera)
handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable)
handleDict['fgcmVisitCatalog'] = butlerQC.get(
inputRefs.fgcmVisitCatalog)
handleDict['fgcmStandardStars'] = butlerQC.get(
inputRefs.fgcmStandardStars)
if self.config.doZeropointOutput:
handleDict['fgcmZeropoints'] = butlerQC.get(
inputRefs.fgcmZeropoints)
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
if self.config.doAtmosphereOutput:
handleDict['fgcmAtmosphereParameters'] = butlerQC.get(
inputRefs.fgcmAtmosphereParameters)
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
if self.config.doReferenceCalibration:
refConfig = LoadReferenceObjectsConfig()
self.refObjLoader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
log=self.log,
config=refConfig)
else:
self.refObjLoader = None
struct = self.run(handleDict, self.config.physicalFilterMap)
# Output the photoCalib exposure catalogs
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
# Output the atmospheres
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
if self.config.doReferenceCalibration:
# Turn offset into simple catalog for persistence if necessary
schema = afwTable.Schema()
schema.addField('offset',
type=np.float64,
doc="Post-process calibration offset (mag)")
offsetCat = afwTable.BaseCatalog(schema)
offsetCat.resize(len(struct.offsets))
offsetCat['offset'][:] = struct.offsets
butlerQC.put(offsetCat, outputRefs.fgcmOffsets)
return
def setUp(self):
self.schema = afwTable.Schema()
self.schema.addField('test1', type='ArrayF', size=430000025)
self.schema.addField('test2', type='ArrayF', size=430000025)
self.cat = afwTable.BaseCatalog(self.schema)
def _makeLutCat(self, lutSchema, physicalFilterString,
stdPhysicalFilterString, atmosphereTableName):
"""
Make the LUT schema
Parameters
----------
lutSchema: `afwTable.schema`
Lut catalog schema
physicalFilterString: `str`
Combined string of all the physicalFilters
stdPhysicalFilterString: `str`
Combined string of all the standard physicalFilters
atmosphereTableName: `str`
Name of the atmosphere table used to generate LUT
Returns
-------
lutCat: `afwTable.BaseCatalog`
Lut catalog for persistence
"""
# The somewhat strange format is to make sure that
# the rows of the afwTable do not get too large
# (see DM-11419)
lutCat = afwTable.BaseCatalog(lutSchema)
lutCat.table.preallocate(14)
# first fill the first index
rec = lutCat.addNew()
rec['tablename'] = atmosphereTableName
rec['elevation'] = self.fgcmLutMaker.atmosphereTable.elevation
rec['physicalFilters'] = physicalFilterString
rec['stdPhysicalFilters'] = stdPhysicalFilterString
rec['pmb'][:] = self.fgcmLutMaker.pmb
rec['pmbFactor'][:] = self.fgcmLutMaker.pmbFactor
rec['pmbElevation'] = self.fgcmLutMaker.pmbElevation
rec['pwv'][:] = self.fgcmLutMaker.pwv
rec['o3'][:] = self.fgcmLutMaker.o3
rec['tau'][:] = self.fgcmLutMaker.tau
rec['lambdaNorm'] = self.fgcmLutMaker.lambdaNorm
rec['alpha'][:] = self.fgcmLutMaker.alpha
rec['zenith'][:] = self.fgcmLutMaker.zenith
rec['nCcd'] = self.fgcmLutMaker.nCCD
rec['pmbStd'] = self.fgcmLutMaker.pmbStd
rec['pwvStd'] = self.fgcmLutMaker.pwvStd
rec['o3Std'] = self.fgcmLutMaker.o3Std
rec['tauStd'] = self.fgcmLutMaker.tauStd
rec['alphaStd'] = self.fgcmLutMaker.alphaStd
rec['zenithStd'] = self.fgcmLutMaker.zenithStd
rec['lambdaRange'][:] = self.fgcmLutMaker.lambdaRange
rec['lambdaStep'] = self.fgcmLutMaker.lambdaStep
rec['lambdaStd'][:] = self.fgcmLutMaker.lambdaStd
rec['lambdaStdFilter'][:] = self.fgcmLutMaker.lambdaStdFilter
rec['i0Std'][:] = self.fgcmLutMaker.I0Std
rec['i1Std'][:] = self.fgcmLutMaker.I1Std
rec['i10Std'][:] = self.fgcmLutMaker.I10Std
rec['i2Std'][:] = self.fgcmLutMaker.I2Std
rec['lambdaB'][:] = self.fgcmLutMaker.lambdaB
rec['atmLambda'][:] = self.fgcmLutMaker.atmLambda
rec['atmStdTrans'][:] = self.fgcmLutMaker.atmStdTrans
rec['luttype'] = 'I0'
rec['lut'][:] = self.fgcmLutMaker.lut['I0'].flatten()
# and add the rest
rec = lutCat.addNew()
rec['luttype'] = 'I1'
rec['lut'][:] = self.fgcmLutMaker.lut['I1'].flatten()
derivTypes = [
'D_PMB', 'D_LNPWV', 'D_O3', 'D_LNTAU', 'D_ALPHA', 'D_SECZENITH',
'D_PMB_I1', 'D_LNPWV_I1', 'D_O3_I1', 'D_LNTAU_I1', 'D_ALPHA_I1',
'D_SECZENITH_I1'
]
for derivType in derivTypes:
rec = lutCat.addNew()
rec['luttype'] = derivType
rec['lut'][:] = self.fgcmLutMaker.lutDeriv[derivType].flatten()
return lutCat
def _fgcmMatchStars(self, butler, visitCat):
"""
Use FGCM code to match observations into unique stars.
Parameters
----------
butler: `lsst.daf.persistence.Butler`
visitCat: `afw.table.BaseCatalog`
Catalog with visit data for FGCM
Outputs
-------
Butler will output fgcmStarIds (matched star identifiers) and
fgcmStarIndices (index values linking fgcmStarObservations to
fgcmStarIds).
"""
obsCat = butler.get('fgcmStarObservations')
# get filter names into a numpy array...
# This is the type that is expected by the fgcm code
visitFilterNames = np.zeros(len(visitCat), dtype='a2')
for i in range(len(visitCat)):
visitFilterNames[i] = visitCat[i]['filtername']
# match to put filterNames with observations
visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit'])
obsFilterNames = visitFilterNames[visitIndex]
if self.config.doReferenceMatches:
# Get the reference filter names, using the LUT
lutCat = butler.get('fgcmLookUpTable')
stdFilterDict = {
filterName: stdFilter
for (filterName,
stdFilter) in zip(lutCat[0]['filterNames'].split(','),
lutCat[0]['stdFilterNames'].split(','))
}
stdLambdaDict = {
stdFilter: stdLambda
for (stdFilter,
stdLambda) in zip(lutCat[0]['stdFilterNames'].split(','),
lutCat[0]['lambdaStdFilter'])
}
del lutCat
referenceFilterNames = self._getReferenceFilterNames(
visitCat, stdFilterDict, stdLambdaDict)
self.log.info("Using the following reference filters: %s" %
(', '.join(referenceFilterNames)))
else:
# This should be an empty list
referenceFilterNames = []
# make the fgcm starConfig dict
starConfig = {
'logger': self.log,
'filterToBand': self.config.filterMap,
'requiredBands': self.config.requiredBands,
'minPerBand': self.config.minPerBand,
'matchRadius': self.config.matchRadius,
'isolationRadius': self.config.isolationRadius,
'matchNSide': self.config.matchNside,
'coarseNSide': self.config.coarseNside,
'densNSide': self.config.densityCutNside,
'densMaxPerPixel': self.config.densityCutMaxPerPixel,
'primaryBands': self.config.primaryBands,
'referenceFilterNames': referenceFilterNames
}
# initialize the FgcmMakeStars object
fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
# make the primary stars
# note that the ra/dec native Angle format is radians
# We determine the conversion from the native units (typically
# radians) to degrees for the first observation. This allows us
# to treate ra/dec as numpy arrays rather than Angles, which would
# be approximately 600x slower.
conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
obsCat['dec'] * conv,
filterNameArray=obsFilterNames,
bandSelected=False)
# and match all the stars
fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
obsCat['dec'] * conv, obsFilterNames)
if self.config.doReferenceMatches:
fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
# now persist
objSchema = self._makeFgcmObjSchema()
# make catalog and records
fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
for i in range(fgcmMakeStars.objIndexCat.size):
fgcmStarIdCat.addNew()
# fill the catalog
fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat[
'obsarrindex']
fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
butler.put(fgcmStarIdCat, 'fgcmStarIds')
obsSchema = self._makeFgcmObsSchema()
fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
for i in range(fgcmMakeStars.obsIndexCat.size):
fgcmStarIndicesCat.addNew()
fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat[
'obsindex']
butler.put(fgcmStarIndicesCat, 'fgcmStarIndices')
if self.config.doReferenceMatches:
refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
fgcmRefCat = afwTable.BaseCatalog(refSchema)
fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
for i in range(fgcmMakeStars.referenceCat.size):
fgcmRefCat.addNew()
fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat[
'refMagErr']
butler.put(fgcmRefCat, 'fgcmReferenceStars')
def _fgcmMakeAllStarObservations(self, butler, visitCat):
"""
Compile all good star observations from visits in visitCat
Parameters
----------
butler: `lsst.daf.persistence.Butler`
visitCat: `afw.table.BaseCatalog`
Catalog with visit data for FGCM
"""
startTime = time.time()
# create our source schema
sourceSchema = butler.get('src_schema', immediate=True).schema
sourceMapper = self._makeSourceMapper(sourceSchema)
# We also have a temporary catalog that will accumulate aperture measurements
aperMapper = self._makeAperMapper(sourceSchema)
# we need to know the ccds...
camera = butler.get('camera')
outputSchema = sourceMapper.getOutputSchema()
fullCatalog = afwTable.BaseCatalog(outputSchema)
# FGCM will provide relative calibration for the flux in config.instFluxField
instFluxKey = sourceSchema[self.config.instFluxField].asKey()
instFluxErrKey = sourceSchema[self.config.instFluxField +
'Err'].asKey()
visitKey = outputSchema['visit'].asKey()
ccdKey = outputSchema['ccd'].asKey()
instMagKey = outputSchema['instMag'].asKey()
instMagErrKey = outputSchema['instMagErr'].asKey()
aperOutputSchema = aperMapper.getOutputSchema()
instFluxAperInKey = sourceSchema[
self.config.apertureInnerInstFluxField].asKey()
instFluxErrAperInKey = sourceSchema[
self.config.apertureInnerInstFluxField + 'Err'].asKey()
instFluxAperOutKey = sourceSchema[
self.config.apertureOuterInstFluxField].asKey()
instFluxErrAperOutKey = sourceSchema[
self.config.apertureOuterInstFluxField + 'Err'].asKey()
instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey()
instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey()
instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey()
instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey()
# loop over visits
for visit in visitCat:
expTime = visit['exptime']
nStarInVisit = 0
# Reset the aperture catalog (per visit)
aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema)
# loop over CCDs
for ccdIndex, detector in enumerate(camera):
ccdId = detector.getId()
try:
sources = butler.get(
'src',
dataId={
self.config.visitDataRefName: visit['visit'],
self.config.ccdDataRefName: ccdId
},
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
except butlerExceptions.NoResults:
# this is not a problem if this ccd isn't there
continue
goodSrc = self.sourceSelector.selectSources(sources)
tempCat = afwTable.BaseCatalog(fullCatalog.schema)
tempCat.reserve(goodSrc.selected.sum())
tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper)
tempCat[visitKey][:] = visit['visit']
tempCat[ccdKey][:] = ccdId
# Compute "magnitude" by scaling flux with exposure time.
# Add an arbitrary zeropoint that needs to be investigated.
scaledInstFlux = sources[instFluxKey][
goodSrc.selected] * visit['scaling'][ccdIndex]
tempCat[instMagKey][:] = (-2.5 * np.log10(scaledInstFlux) +
2.5 * np.log10(expTime))
# instMagErr is computed with original (unscaled) flux
k = 2.5 / np.log(10.)
tempCat[instMagErrKey][:] = k * (
sources[instFluxErrKey][goodSrc.selected] /
sources[instFluxKey][goodSrc.selected])
if self.config.applyJacobian:
tempCat[instMagKey][:] -= 2.5 * np.log10(
tempCat['jacobian'][:])
fullCatalog.extend(tempCat)
# And the aperture information
tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema)
tempAperCat.reserve(goodSrc.selected.sum())
tempAperCat.extend(sources[goodSrc.selected],
mapper=aperMapper)
with np.warnings.catch_warnings():
# Ignore warnings, we will filter infinities and
# nans below.
np.warnings.simplefilter("ignore")
tempAperCat[instMagInKey][:] = -2.5 * np.log10(
sources[instFluxAperInKey][goodSrc.selected])
tempAperCat[instMagErrInKey][:] = (2.5 / np.log(10.)) * (
sources[instFluxErrAperInKey][goodSrc.selected] /
sources[instFluxAperInKey][goodSrc.selected])
tempAperCat[instMagOutKey][:] = -2.5 * np.log10(
sources[instFluxAperOutKey][goodSrc.selected])
tempAperCat[instMagErrOutKey][:] = (2.5 / np.log(10.)) * (
sources[instFluxErrAperOutKey][goodSrc.selected] /
sources[instFluxAperOutKey][goodSrc.selected])
aperVisitCatalog.extend(tempAperCat)
nStarInVisit += len(tempCat)
# Compute the median delta-aper
if not aperVisitCatalog.isContiguous():
aperVisitCatalog = aperVisitCatalog.copy(deep=True)
instMagIn = aperVisitCatalog[instMagInKey]
instMagErrIn = aperVisitCatalog[instMagErrInKey]
instMagOut = aperVisitCatalog[instMagOutKey]
instMagErrOut = aperVisitCatalog[instMagErrOutKey]
ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn)
& np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
self.log.info(
" Found %d good stars in visit %d (deltaAper = %.3f)" %
(nStarInVisit, visit['visit'], visit['deltaAper']))
self.log.info("Found all good star observations in %.2f s" %
(time.time() - startTime))
# Write all the observations
butler.put(fullCatalog, 'fgcmStarObservations')
# And overwrite the visitCatalog with delta_aper info
butler.put(visitCat, 'fgcmVisitCatalog')
self.log.info("Done with all stars in %.2f s" %
(time.time() - startTime))
def run(self, dataRef, selectDataList=[]):
self.log.info("Processing %s" % (dataRef.dataId))
filters = self.config.filters.split("^")
catalogs = dict(self.readCatalog(dataRef, f) for f in filters)
coadds = dict(self.readCoadd(dataRef, f) for f in filters)
wcs = coadds[filters[0]].getWcs()
pixel_scale = wcs.pixelScale().asDegrees() * 3600.0
ref = catalogs[self.config.refFilterName]
# create new table table
mergedSchema = afwTable.Schema()
fields = []
# define table fields
fields.append(mergedSchema.addField("id", type="L", doc="Unique id"))
fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]"))
fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]"))
fields.append(
mergedSchema.addField(
"countInputs",
type="I",
doc="Number of input single exposures for the reference filter"
))
fields.append(
mergedSchema.addField(
"PSFDetRadius",
type="F",
doc=
"Determinant radius for the PSF at the object position = sigma if gaussian [arcsec]"
))
fields.append(
mergedSchema.addField("EB_V",
type="F",
doc="Milky Way dust E(B-V) [mag]"))
fields.append(
mergedSchema.addField("isDuplicated",
type="I",
doc="1 if outside the inner tract or patch"))
fields.append(
mergedSchema.addField(
"isEdge",
type="I",
doc="1 if offImage or in region masked EDGE or NO_DATA"))
fields.append(
mergedSchema.addField(
"hasBadPhotometry",
type="I",
doc=
"1 if interpolated, saturated, suspect, has CR at center or near bright object"
))
fields.append(
mergedSchema.addField("isClean",
type="I",
doc="1 if none of other flags is set"))
if self.config.hasDepthInfo:
fields.append(
mergedSchema.addField(
"isFullDepthColor",
type="I",
doc="1 if point located in full depth and color area"))
# create table object
merged = afwTable.BaseCatalog(mergedSchema)
N = len(ref)
for i in range(N):
# for i in range(100,110):
# create new record
record = merged.addNew()
coord = ref[i].get('coord')
# record if any of the filter is flagged as bad photometry
for f in filters:
hasBadPhotometry = (catalogs[f][i].get('flags.pixel.interpolated.center')) \
| (catalogs[f][i].get('flags.pixel.saturated.center')) \
| (catalogs[f][i].get('flags.pixel.suspect.center')) \
| (catalogs[f][i].get('flags.pixel.cr.center')) \
| (catalogs[f][i].get('flags.pixel.bad')) \
| (catalogs[f][i].get('flags.pixel.bright.object.center'))
if hasBadPhotometry:
break
isDuplicated = not ref[i].get('detect.is-primary')
isEdge = (ref[i].get('flags.pixel.offimage')) | (
ref[i].get('flags.pixel.edge'))
isClean = (not hasBadPhotometry) & (not isDuplicated) & (
not isEdge)
# record common info from reference filter
record.set(mergedSchema['id'].asKey(), ref[i].get('id'))
record.set(mergedSchema['ra'].asKey(),
coord.toFk5().getRa().asDegrees())
record.set(mergedSchema['dec'].asKey(),
coord.toFk5().getDec().asDegrees())
record.set(mergedSchema['countInputs'].asKey(),
ref[i].get('countInputs'))
record.set(
mergedSchema['PSFDetRadius'].asKey(),
ref[i].get("shape.sdss.psf").getDeterminantRadius() *
pixel_scale)
record.set(mergedSchema['isDuplicated'].asKey(), int(isDuplicated))
record.set(mergedSchema['isEdge'].asKey(), int(isEdge))
record.set(mergedSchema['hasBadPhotometry'].asKey(),
int(hasBadPhotometry))
record.set(mergedSchema['isClean'].asKey(), int(isClean))
if self.config.hasDepthInfo:
record.set(mergedSchema['isFullDepthColor'].asKey(),
int(ref[i].get('isFullDepthColor')))
# dust correction
EB_V = self.getDustCorrection(
self.dustMap, record.get(mergedSchema['ra'].asKey()),
record.get(mergedSchema['dec'].asKey()))
record.set(mergedSchema['EB_V'].asKey(), EB_V)
# write catalog
if self.config.fileOutName == "":
# get output dir
# TO DO: create new PAF
# see /Users/coupon/local/source/hscPipe/install/DarwinX86/solvetansip/6.5.1p_hsc/python/hsc/meas/tansip/utils.py
# and /Users/coupon/local/source/hscPipe/install/DarwinX86/pex_policy/HSC-4.0.0/tests/Policy_1.py
if self.config.dirOutName == "":
dirOutName = dataRef.getButler(
).mapper.root + "/" + self.config.coaddName + "Coadd-results"
self.log.info(
"WARNING: the output file will be written in {0:s}.".
format(dirOutName))
else:
dirOutName = self.config.dirOutName
fileOutName = "{0}/{1}/{2}/{3}/multiRanCat-{2}-{3}.fits".format(
dirOutName, "merged", dataRef.dataId["tract"],
dataRef.dataId["patch"])
else:
fileOutName = self.config.fileOutName
self.log.info("Writing {0:s}".format(fileOutName))
self.mkdir_p(os.path.dirname(fileOutName))
merged.writeFits(fileOutName)
# write catalog
#self.log.info("Writing {0:s}".format(self.config.fileOutName))
#if self.config.fileOutName == "":
# fileOutName = "{0}/{1}/{2}/{3}/MultiRanCat-{2}-{3}.fits".format(self.config.dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"])
# self.mkdir_p(os.path.dirname(fileOutName))
#else:
# fileOutName = self.config.fileOutName
#merged.writeFits(fileOutName)
return
def runDataRef(self, files):
"""Run this task via CmdLineTask and Gen2 Butler.
Parameters
----------
patchRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
A list of DataRefs for all filters in a single patch.
"""
self.prep()
for file in files:
cat = afwTable.BaseCatalog.readFits(file)
mask = ((cat['moments_r_even'][:, 0] >= self.fluxMin) &
(cat['moments_r_even'][:, 0] < self.fluxMax) &
(cat['moments_r_cov_even'][:, 0] >= self.varMin) &
(cat['moments_r_cov_even'][:, 0] < self.varMax) &
(cat['z'] >= self.zMin) &
(cat['z'] < self.zMax))
tgs = []
for rec in cat[mask]:
cov_even = rec.get('moments_r_cov_even')
cov_odd = rec.get('moments_r_cov_odd')
full_cov_even = np.zeros((self.n_even, self.n_even), dtype=np.float32)
full_cov_odd = np.zeros((self.n_odd, self.n_odd), dtype=np.float32)
start = 0
for i in range(self.n_even):
full_cov_even[i][i:] = cov_even[start:start + self.n_even - i]
start += self.n_even - i
for i in range(self.n_even):
for j in range(i):
full_cov_even[i, j] = full_cov_even[j, i]
start = 0
for i in range(self.n_odd):
full_cov_odd[i][i:] = cov_odd[start:start + self.n_odd - i]
start += self.n_odd - i
for i in range(self.n_odd):
for j in range(i):
full_cov_odd[i, j] = full_cov_odd[j, i]
cov = self.bfd.MomentCov(full_cov_even, full_cov_odd)
#mom_odd = np.array([0, 0]
moment = self.bfd.Moment(rec.get('moments_r_even'), rec.get('moments_r_odd'))
pos = np.array([rec.get('ra'), rec.get('dec')])
tg = self.bfd.TargetGalaxy(moment, cov, pos, rec.get('id'))
tgs.append(tg)
results = self.prior.getPqrCatalog(tgs[:10], self.config.threads, self.config.chunk)
outcat = afwTable.BaseCatalog(self.schema)
raKey = self.schema['coord_ra'].asKey()
decKey = self.schema['coord_ra'].asKey()
idKey = self.schema['id'].asKey()
for r, rec in zip(results, cat[mask][:10]):
out = outcat.addNew()
out.set(self.pqrKey, r[0]._pqr)
out.set(self.numKey, r[1])
out.set(self.uniqKey, r[2])
out.set(self.momKey, rec.get('moments_r_even'))
out.set(self.momCovKey, rec.get('moments_r_cov_even'))
out.set(self.zKey, rec.get('z'))
out.set(self.g1Key, rec.get('g1'))
out.set(self.g1Key, rec.get('g2'))
out.set(self.kappaKey, rec.get('kappa'))
out.set(self.magKey, rec.get('mag'))
out.set(self.labelKey, self.config.label)
out.set(raKey, rec.get('ra')*afwGeom.degrees)
out.set(decKey, rec.get('dec')*afwGeom.degrees)
out.set(idKey, rec.get('id'))
outfile = file.replace('moment', 'pqr')
outcat.writeFits(outfile)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat,
calibFluxApertureRadius=None,
visitCatDataRef=None,
starObsDataRef=None,
inStarObsCat=None):
startTime = time.time()
# If both dataRefs are None, then we assume the caller does not
# want to store checkpoint files. If both are set, we will
# do checkpoint files. And if only one is set, this is potentially
# unintentional and we will warn.
if (visitCatDataRef is not None and starObsDataRef is None or
visitCatDataRef is None and starObsDataRef is not None):
self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so "
"no checkpoint files will be persisted.")
if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None:
raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.")
# To get the correct output schema, we use similar code as fgcmBuildStarsTask
# We are not actually using this mapper, except to grab the outputSchema
dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0]
sourceSchema = dataRef.get('src_schema', immediate=True).schema
sourceMapper = self._makeSourceMapper(sourceSchema)
outputSchema = sourceMapper.getOutputSchema()
# Construct mapping from ccd number to index
camera = dataRef.get('camera')
ccdMapping = {}
for ccdIndex, detector in enumerate(camera):
ccdMapping[detector.getId()] = ccdIndex
approxPixelAreaFields = computeApproxPixelAreaFields(camera)
if inStarObsCat is not None:
fullCatalog = inStarObsCat
comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS)
comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES)
if not comp1 or not comp2:
raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.")
else:
fullCatalog = afwTable.BaseCatalog(outputSchema)
visitKey = outputSchema['visit'].asKey()
ccdKey = outputSchema['ccd'].asKey()
instMagKey = outputSchema['instMag'].asKey()
instMagErrKey = outputSchema['instMagErr'].asKey()
# Prepare local background if desired
if self.config.doSubtractLocalBackground:
localBackgroundArea = np.pi*calibFluxApertureRadius**2.
# Determine which columns we need from the sourceTable_visit catalogs
columns = self._get_sourceTable_visit_columns()
k = 2.5/np.log(10.)
for counter, visit in enumerate(visitCat):
# Check if these sources have already been read and stored in the checkpoint file
if visit['sources_read']:
continue
expTime = visit['exptime']
dataRef = groupedDataRefs[visit['visit']][-1]
srcTable = dataRef.get()
df = srcTable.toDataFrame(columns)
goodSrc = self.sourceSelector.selectSources(df)
# Need to add a selection based on the local background correction
# if necessary
if self.config.doSubtractLocalBackground:
localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values
use, = np.where((goodSrc.selected) &
((df[self.config.instFluxField].values - localBackground) > 0.0))
else:
use, = np.where(goodSrc.selected)
tempCat = afwTable.BaseCatalog(fullCatalog.schema)
tempCat.resize(use.size)
tempCat['ra'][:] = np.deg2rad(df['ra'].values[use])
tempCat['dec'][:] = np.deg2rad(df['decl'].values[use])
tempCat['x'][:] = df['x'].values[use]
tempCat['y'][:] = df['y'].values[use]
tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use]
tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use]
tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use]
if self.config.doSubtractLocalBackground:
# At the moment we only adjust the flux and not the flux
# error by the background because the error on
# base_LocalBackground_instFlux is the rms error in the
# background annulus, not the error on the mean in the
# background estimate (which is much smaller, by sqrt(n)
# pixels used to estimate the background, which we do not
# have access to in this task). In the default settings,
# the annulus is sufficiently large such that these
# additional errors are are negligibly small (much less
# than a mmag in quadrature).
# This is the difference between the mag with local background correction
# and the mag without local background correction.
tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] -
localBackground[use]) -
-2.5*np.log10(df[self.config.instFluxField].values[use]))
else:
tempCat['deltaMagBkg'][:] = 0.0
# Need to loop over ccds here
for detector in camera:
ccdId = detector.getId()
# used index for all observations with a given ccd
use2 = (tempCat[ccdKey] == ccdId)
tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2],
tempCat['y'][use2])
scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] *
visit['scaling'][ccdMapping[ccdId]])
tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime))
# Compute instMagErr from instFluxErr/instFlux, any scaling
# will cancel out.
tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] /
df[self.config.instFluxField].values[use])
# Apply the jacobian if configured
if self.config.doApplyWcsJacobian:
tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:])
fullCatalog.extend(tempCat)
# Now do the aperture information
with np.warnings.catch_warnings():
# Ignore warnings, we will filter infinites and nans below
np.warnings.simplefilter("ignore")
instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use])
instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] /
df[self.config.apertureInnerInstFluxField].values[use])
instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use])
instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] /
df[self.config.apertureOuterInstFluxField].values[use])
ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) &
np.isfinite(instMagOut) & np.isfinite(instMagErrOut))
visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok])
visit['sources_read'] = True
self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)",
use.size, visit['visit'], visit['deltaAper'])
if ((counter % self.config.nVisitsPerCheckpoint) == 0 and
starObsDataRef is not None and visitCatDataRef is not None):
# We need to persist both the stars and the visit catalog which gets
# additional metadata from each visit.
starObsDataRef.put(fullCatalog)
visitCatDataRef.put(visitCat)
self.log.info("Found all good star observations in %.2f s" %
(time.time() - startTime))
return fullCatalog
def runQuantum(self, butlerQC, inputRefs, outputRefs):
handleDict = butlerQC.get(inputRefs)
self.log.info("Running with %d sourceTable_visit handles",
(len(handleDict['source_catalogs'])))
# Run the build stars tasks
tract = butlerQC.quantum.dataId['tract']
handleDict['sourceSchema'] = self.sourceSchema
sourceTableHandles = handleDict['source_catalogs']
sourceTableHandleDict = {
sourceTableHandle.dataId['visit']: sourceTableHandle
for sourceTableHandle in sourceTableHandles
}
visitSummaryHandles = handleDict['visitSummary']
visitSummaryHandleDict = {
visitSummaryHandle.dataId['visit']: visitSummaryHandle
for visitSummaryHandle in visitSummaryHandles
}
handleDict['sourceTableHandleDict'] = sourceTableHandleDict
handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict
# And the outputs
if self.config.fgcmOutputProducts.doZeropointOutput:
photoCalibRefDict = {
photoCalibRef.dataId.byName()['visit']: photoCalibRef
for photoCalibRef in outputRefs.fgcmPhotoCalib
}
handleDict['fgcmPhotoCalibs'] = photoCalibRefDict
if self.config.fgcmOutputProducts.doAtmosphereOutput:
atmRefDict = {
atmRef.dataId.byName()['visit']: atmRef
for atmRef in outputRefs.fgcmTransmissionAtmosphere
}
handleDict['fgcmTransmissionAtmospheres'] = atmRefDict
if self.config.fgcmBuildStars.doReferenceMatches:
refConfig = LoadReferenceObjectsConfig()
refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
buildStarsRefObjLoader = loader
else:
buildStarsRefObjLoader = None
if self.config.fgcmOutputProducts.doReferenceCalibration:
refConfig = self.config.fgcmOutputProducts.refObjLoader
loader = ReferenceObjectLoader(
dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat],
refCats=butlerQC.get(inputRefs.refCat),
config=refConfig,
log=self.log)
self.fgcmOutputProducts.refObjLoader = loader
struct = self.run(handleDict,
tract,
buildStarsRefObjLoader=buildStarsRefObjLoader)
if struct.photoCalibCatalogs is not None:
self.log.info("Outputting photoCalib catalogs.")
for visit, expCatalog in struct.photoCalibCatalogs:
butlerQC.put(expCatalog, photoCalibRefDict[visit])
self.log.info("Done outputting photoCalib catalogs.")
if struct.atmospheres is not None:
self.log.info("Outputting atmosphere transmission files.")
for visit, atm in struct.atmospheres:
butlerQC.put(atm, atmRefDict[visit])
self.log.info("Done outputting atmosphere files.")
# Turn raw repeatability into simple catalog for persistence
schema = afwTable.Schema()
schema.addField('rawRepeatability',
type=np.float64,
doc="Per-band raw repeatability in FGCM calibration.")
repeatabilityCat = afwTable.BaseCatalog(schema)
repeatabilityCat.resize(len(struct.repeatability))
repeatabilityCat['rawRepeatability'][:] = struct.repeatability
butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability)
return
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None):
"""
Use FGCM code to match observations into unique stars.
Parameters
----------
visitCat: `afw.table.BaseCatalog`
Catalog with visit data for fgcm
obsCat: `afw.table.BaseCatalog`
Full catalog of star observations for fgcm
lutHandle: `lsst.daf.butler.DeferredDatasetHandle`, optional
Data reference to fgcm look-up table (used if matching reference stars).
Returns
-------
fgcmStarIdCat: `afw.table.BaseCatalog`
Catalog of unique star identifiers and index keys
fgcmStarIndicesCat: `afwTable.BaseCatalog`
Catalog of unique star indices
fgcmRefCat: `afw.table.BaseCatalog`
Catalog of matched reference stars.
Will be None if `config.doReferenceMatches` is False.
"""
# get filter names into a numpy array...
# This is the type that is expected by the fgcm code
visitFilterNames = np.zeros(len(visitCat), dtype='a30')
for i in range(len(visitCat)):
visitFilterNames[i] = visitCat[i]['physicalFilter']
# match to put filterNames with observations
visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit'])
obsFilterNames = visitFilterNames[visitIndex]
if self.config.doReferenceMatches:
# Get the reference filter names, using the LUT
lutCat = lutHandle.get()
stdFilterDict = {
filterName: stdFilter
for (filterName, stdFilter
) in zip(lutCat[0]['physicalFilters'].split(','),
lutCat[0]['stdPhysicalFilters'].split(','))
}
stdLambdaDict = {
stdFilter: stdLambda
for (stdFilter, stdLambda
) in zip(lutCat[0]['stdPhysicalFilters'].split(','),
lutCat[0]['lambdaStdFilter'])
}
del lutCat
referenceFilterNames = self._getReferenceFilterNames(
visitCat, stdFilterDict, stdLambdaDict)
self.log.info("Using the following reference filters: %s" %
(', '.join(referenceFilterNames)))
else:
# This should be an empty list
referenceFilterNames = []
# make the fgcm starConfig dict
starConfig = {
'logger': self.log,
'useHtm': True,
'filterToBand': self.config.physicalFilterMap,
'requiredBands': self.config.requiredBands,
'minPerBand': self.config.minPerBand,
'matchRadius': self.config.matchRadius,
'isolationRadius': self.config.isolationRadius,
'matchNSide': self.config.matchNside,
'coarseNSide': self.config.coarseNside,
'densNSide': self.config.densityCutNside,
'densMaxPerPixel': self.config.densityCutMaxPerPixel,
'randomSeed': self.config.randomSeed,
'primaryBands': self.config.primaryBands,
'referenceFilterNames': referenceFilterNames
}
# initialize the FgcmMakeStars object
fgcmMakeStars = fgcm.FgcmMakeStars(starConfig)
# make the primary stars
# note that the ra/dec native Angle format is radians
# We determine the conversion from the native units (typically
# radians) to degrees for the first observation. This allows us
# to treate ra/dec as numpy arrays rather than Angles, which would
# be approximately 600x slower.
conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra'])
fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv,
obsCat['dec'] * conv,
filterNameArray=obsFilterNames,
bandSelected=False)
# and match all the stars
fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv,
obsCat['dec'] * conv, obsFilterNames)
if self.config.doReferenceMatches:
fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog)
# now persist
objSchema = self._makeFgcmObjSchema()
# make catalog and records
fgcmStarIdCat = afwTable.BaseCatalog(objSchema)
fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size)
for i in range(fgcmMakeStars.objIndexCat.size):
fgcmStarIdCat.addNew()
# fill the catalog
fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id']
fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra']
fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec']
fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat[
'obsarrindex']
fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs']
obsSchema = self._makeFgcmObsSchema()
fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema)
fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size)
for i in range(fgcmMakeStars.obsIndexCat.size):
fgcmStarIndicesCat.addNew()
fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat[
'obsindex']
if self.config.doReferenceMatches:
refSchema = self._makeFgcmRefSchema(len(referenceFilterNames))
fgcmRefCat = afwTable.BaseCatalog(refSchema)
fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size)
for i in range(fgcmMakeStars.referenceCat.size):
fgcmRefCat.addNew()
fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id']
fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag']
fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat[
'refMagErr']
md = PropertyList()
md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION)
md.set("FILTERNAMES", referenceFilterNames)
fgcmRefCat.setMetadata(md)
else:
fgcmRefCat = None
return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat
