def toTableDataTable(self, metadata): """Produce linearity catalog from table data Parameters ---------- metadata : `lsst.daf.base.PropertyList` Linearizer metadata Returns ------- catalog : `lsst.afw.table.BaseCatalog` Catalog to write """ schema = afwTable.Schema() dimensions = self.tableData.shape lut = schema.addField("LOOKUP_VALUES", type='ArrayF', size=dimensions[1], doc="linearity lookup data") catalog = afwTable.BaseCatalog(schema) catalog.resize(dimensions[0]) for ii in range(dimensions[0]): catalog[ii][lut] = np.array(self.tableData[ii], dtype=np.float32) metadata["LINEARITY_LOOKUP"] = True catalog.setMetadata(metadata) return catalog
def makeAtmCat(atmSchema, atmStruct): """ Make the atmosphere catalog for persistence Parameters ---------- atmSchema: `lsst.afw.table.Schema` Atmosphere catalog schema atmStruct: `numpy.ndarray` Atmosphere structure from fgcm Returns ------- atmCat: `lsst.afw.table.BaseCatalog` Atmosphere catalog for persistence """ atmCat = afwTable.BaseCatalog(atmSchema) atmCat.resize(atmStruct.size) atmCat['visit'][:] = atmStruct['VISIT'] atmCat['pmb'][:] = atmStruct['PMB'] atmCat['pwv'][:] = atmStruct['PWV'] atmCat['tau'][:] = atmStruct['TAU'] atmCat['alpha'][:] = atmStruct['ALPHA'] atmCat['o3'][:] = atmStruct['O3'] atmCat['secZenith'][:] = atmStruct['SECZENITH'] atmCat['cTrans'][:] = atmStruct['CTRANS'] atmCat['lamStd'][:] = atmStruct['LAMSTD'] return atmCat
def _wrap_result(data, column_names, table=None, index_col=None, coerce_float=True, column_dtypes=None, parse_dates=None): """Wrap result set of query in a afw table """ result_size = len(data) # Turn into columns first from pandas import lib data = list(lib.to_object_array_tuples(data).T) arrays = [lib.maybe_convert_objects(arr, try_float=True) for arr in data] _harmonize_columns(arrays, column_names, table, column_dtypes, parse_dates) schema = afw_table.Schema() # build schema for i, column_name in enumerate(column_names): column_type = arrays[i].dtype schema.addField(column_name, type=column_type.type) catalog = afw_table.BaseCatalog(schema) # Preallocate rows based on first column length catalog.preallocate(result_size) for i in range(result_size): record = catalog.addNew() for column_i in range(len(column_names)): record.set(column_names[column_i], arrays[column_i][i]) return catalog
def _fgcmMakeVisitCatalog(self, butler, dataRefs): """ Make a visit catalog with all the key data from each visit Parameters ---------- butler: `lsst.daf.persistence.Butler` dataRefs: `list` of `lsst.daf.persistence.ButlerDataRef` Data references for the input visits. If this is an empty list, all visits with src catalogs in the repository are used. Only one individual dataRef from a visit need be specified and the code will find the other source catalogs from each visit. Returns ------- visitCat: `afw.table.BaseCatalog` """ startTime = time.time() camera = butler.get('camera') nCcd = len(camera) # TODO: related to DM-13730, this dance of looking for source visits # will be unnecessary with Gen3 Butler. This should be part of # DM-13730. if len(dataRefs) == 0: srcVisits, srcCcds = self._findSourceVisits(butler, nCcd) else: # get the visits from the datarefs, only for referenceCCD srcVisits = [ d.dataId[self.config.visitDataRefName] for d in dataRefs if d.dataId[self.config.ccdDataRefName] == self.config.referenceCCD ] srcCcds = [self.config.referenceCCD] * len(srcVisits) # Sort the visits for searching/indexing srcVisits.sort() self.log.info("Found %d visits in %.2f s" % (len(srcVisits), time.time() - startTime)) schema = self._makeFgcmVisitSchema(nCcd) visitCat = afwTable.BaseCatalog(schema) visitCat.table.preallocate(len(srcVisits)) startTime = time.time() self._fillVisitCatalog(butler, visitCat, srcVisits, srcCcds) self.log.info("Found all VisitInfo in %.2f s" % (time.time() - startTime)) return visitCat
def _performTransform(self, transformClass, inCat, doExtend=True): """Operate on inCat with a transform of class transformClass""" mapper = afwTable.SchemaMapper(inCat.schema) config = SillyCentroidConfig() transform = transformClass(config, self.pluginName, mapper) outCat = afwTable.BaseCatalog(mapper.getOutputSchema()) if doExtend: outCat.extend(inCat, mapper=mapper) transform(inCat, outCat, makeWcs(), afwImage.Calib()) return outCat
def testApplyCppTransform(self): """Test that we can apply a simple C++ transform""" inCat = self._generateCatalog() sillyControl = testLib.SillyCentroidControl() mapper = afwTable.SchemaMapper(inCat.schema) sillyTransform = testLib.SillyTransform(sillyControl, self.pluginName, mapper) outCat = afwTable.BaseCatalog(mapper.getOutputSchema()) outCat.extend(inCat, mapper=mapper) self.assertEqual(len(inCat), len(outCat)) sillyTransform(inCat, outCat, makeWcs(), afwImage.Calib()) self._checkSillyOutputs(inCat, outCat)
def makeZptCat(zptSchema, zpStruct): """ Make the zeropoint catalog for persistence Parameters ---------- zptSchema: `lsst.afw.table.Schema` Zeropoint catalog schema zpStruct: `numpy.ndarray` Zeropoint structure from fgcm Returns ------- zptCat: `afwTable.BaseCatalog` Zeropoint catalog for persistence """ zptCat = afwTable.BaseCatalog(zptSchema) zptCat.reserve(zpStruct.size) for filterName in zpStruct['FILTERNAME']: rec = zptCat.addNew() rec['filtername'] = filterName.decode('utf-8') zptCat['visit'][:] = zpStruct['VISIT'] zptCat['ccd'][:] = zpStruct['CCD'] zptCat['fgcmFlag'][:] = zpStruct['FGCM_FLAG'] zptCat['fgcmZpt'][:] = zpStruct['FGCM_ZPT'] zptCat['fgcmZptErr'][:] = zpStruct['FGCM_ZPTERR'] zptCat['fgcmfZptChebXyMax'][:, :] = zpStruct['FGCM_FZPT_XYMAX'] zptCat['fgcmfZptCheb'][:, :] = zpStruct['FGCM_FZPT_CHEB'] zptCat['fgcmfZptSstarCheb'][:, :] = zpStruct['FGCM_FZPT_SSTAR_CHEB'] zptCat['fgcmI0'][:] = zpStruct['FGCM_I0'] zptCat['fgcmI10'][:] = zpStruct['FGCM_I10'] zptCat['fgcmR0'][:] = zpStruct['FGCM_R0'] zptCat['fgcmR10'][:] = zpStruct['FGCM_R10'] zptCat['fgcmGry'][:] = zpStruct['FGCM_GRY'] zptCat['fgcmDeltaChrom'][:] = zpStruct['FGCM_DELTACHROM'] zptCat['fgcmZptVar'][:] = zpStruct['FGCM_ZPTVAR'] zptCat['fgcmTilings'][:] = zpStruct['FGCM_TILINGS'] zptCat['fgcmFpGry'][:] = zpStruct['FGCM_FPGRY'] zptCat['fgcmFpGryBlue'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 0] zptCat['fgcmFpGryBlueErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 0] zptCat['fgcmFpGryRed'][:] = zpStruct['FGCM_FPGRY_CSPLIT'][:, 2] zptCat['fgcmFpGryRedErr'][:] = zpStruct['FGCM_FPGRY_CSPLITERR'][:, 2] zptCat['fgcmFpVar'][:] = zpStruct['FGCM_FPVAR'] zptCat['fgcmDust'][:] = zpStruct['FGCM_DUST'] zptCat['fgcmFlat'][:] = zpStruct['FGCM_FLAT'] zptCat['fgcmAperCorr'][:] = zpStruct['FGCM_APERCORR'] zptCat['fgcmDeltaMagBkg'][:] = zpStruct['FGCM_DELTAMAGBKG'] zptCat['exptime'][:] = zpStruct['EXPTIME'] return zptCat
def _get_afw_table(): schema = afw_table.Schema() aa = schema.addField("a", type=np.int64, doc="a") bb = schema.addField("b", type=np.float64, doc="b") cat = afw_table.BaseCatalog(schema) row = cat.addNew() row.set(aa, 12345) row.set(bb, 1.2345) row = cat.addNew() row.set(aa, 4321) row.set(bb, 4.123) return cat
def fgcmMakeVisitCatalog(self, camera, groupedDataRefs, visitCatDataRef=None, inVisitCat=None): """ Make a visit catalog with all the keys from each visit Parameters ---------- camera: `lsst.afw.cameraGeom.Camera` Camera from the butler groupedDataRefs: `dict` Dictionary with visit keys, and `list`s of `lsst.daf.persistence.ButlerDataRef` visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional Dataref to write visitCat for checkpoints inVisitCat: `afw.table.BaseCatalog` Input (possibly incomplete) visit catalog Returns ------- visitCat: `afw.table.BaseCatalog` """ self.log.info("Assembling visitCatalog from %d %ss" % (len(groupedDataRefs), self.config.visitDataRefName)) nCcd = len(camera) if inVisitCat is None: schema = self._makeFgcmVisitSchema(nCcd) visitCat = afwTable.BaseCatalog(schema) visitCat.reserve(len(groupedDataRefs)) for i, visit in enumerate(sorted(groupedDataRefs)): rec = visitCat.addNew() rec['visit'] = visit rec['used'] = 0 rec['sources_read'] = 0 else: visitCat = inVisitCat # No matter what, fill the catalog. This will check if it was # already read. self._fillVisitCatalog(visitCat, groupedDataRefs, visitCatDataRef=visitCatDataRef) return visitCat
def buildCatalog(self, momentPrior): outCat = afwTable.BaseCatalog(self.schema) for temp in momentPrior.templates: rec = outCat.addNew() rec.set(self.mKey, temp.m) rec.set(self.dmKey, temp.dm.flatten()) rec.set(self.dxyKey, temp.dxy.flatten()) rec.set(self.ndaKey, temp.nda) rec.set(self.idKey, temp.id) if self.config.zFile: rec.set(self.zKey, self.z_list[temp.id]) # rec.set(self.zIdKey, self.z_id_list[temp.id]) return outCat
def concatenate(catalogList): """Concatenate multiple catalogs (FITS tables from lsst.afw.table)""" catalogList = [afwTable.BaseCatalog.readFits(c) if isinstance(c, basestring) else c for c in catalogList] schema = catalogList[0].schema for i, c in enumerate(catalogList[1:]): if c.schema != schema: raise RuntimeError("Schema for catalog %d not consistent" % (i+1)) out = afwTable.BaseCatalog(schema) num = reduce(lambda n, c: n + len(c), catalogList, 0) out.preallocate(num) for catalog in catalogList: for record in catalog: out.append(out.table.copyRecord(record)) return out
def __init__(self, *args, **kwargs): """ """ super(BuildCovarianceTask, self).__init__(*args, **kwargs) self.ncolors = 0 self.bfd = dbfd.BFDConfig(use_conc=self.config.use_conc, use_mag=self.config.use_mag, ncolors=self.ncolors) self.n_even = self.bfd.BFDConfig.MSIZE self.n_odd = self.bfd.BFDConfig.XYSIZE self.size_even = self.n_even * (self.n_even + 1) // 2 self.size_odd = self.n_odd * (self.n_odd + 1) // 2 self.schema = afwTable.Schema() self.labelKey = self.schema.addField("label", type=str, doc="name of bin", size=10) self.minKey = self.schema.addField("min", type=float, doc="minimum value of the variance") self.maxKey = self.schema.addField("max", type=float, doc="maximum value of the variance") self.isoCovEvenKey = self.schema.addField( "isoCovEven", doc="isotropized moment covariance matrix", type="ArrayF", size=self.size_even) self.isoCovOddKey = self.schema.addField( "isoCovOdd", doc="isotropized moment covariance matrix", type="ArrayF", size=self.size_odd) self.covEvenKey = self.schema.addField("covEven", doc="moment covariance matrix", type="ArrayF", size=self.size_even) self.covOddKey = self.schema.addField("covOdd", doc="moment covariance matrix", type="ArrayF", size=self.size_odd) self.catalog = afwTable.BaseCatalog(self.schema)
def _makeForcedSourceCatalog(objects): """Make a catalog containing a bunch of DiaFourceSources associated with the input diaObjects. """ # make some sources schema = afwTable.Schema() schema.addField("diaObjectId", "L") schema.addField("ccdVisitId", "L") schema.addField("flags", "L") catalog = afwTable.BaseCatalog(schema) oids = [] for obj in objects: record = catalog.addNew() record.set("diaObjectId", obj["id"]) record.set("ccdVisitId", 1) record.set("flags", 0) oids.append(obj["id"]) return catalog, oids
def run(self, inputCat, wcs, calib): """!Transform raw source measurements to calibrated quantities. @param[in] inputCat SourceCatalog of sources to transform. @param[in] wcs The world coordinate system under which transformations will take place. @param[in] calib The calibration under which transformations will take place. @return A BaseCatalog containing the transformed measurements. """ outputCat = afwTable.BaseCatalog(self.mapper.getOutputSchema()) outputCat.extend(inputCat, mapper=self.mapper) # Transforms may use a ColumnView on the input and output catalogs, # which requires that the data be contiguous in memory. inputCat = makeContiguous(inputCat) outputCat = makeContiguous(outputCat) for transform in self.transforms: transform(inputCat, outputCat, wcs, calib) return outputCat
def _makeSourceCatalog(objects): """Make a catalog containing a bunch of DiaSources associated with the input diaObjects. """ # make some sources schema = make_minimal_dia_source_schema() catalog = afwTable.BaseCatalog(schema) oids = [] for sid, obj in enumerate(objects): record = catalog.addNew() record.set("id", sid) record.set("ccdVisitId", 1) record.set("diaObjectId", obj["id"]) record.set("parent", 0) record.set("coord_ra", obj["coord_ra"]) record.set("coord_dec", obj["coord_dec"]) record.set("flags", 0) record.set("pixelId", obj["pixelId"]) oids.append(obj["id"]) return catalog, oids
def fgcmMakeVisitCatalog(self, camera, groupedHandles, bkgHandleDict=None): """ Make a visit catalog with all the keys from each visit Parameters ---------- camera: `lsst.afw.cameraGeom.Camera` Camera from the butler groupedHandles: `dict` [`list` [`lsst.daf.butler.DeferredDatasetHandle`]] Dataset handles, grouped by visit. bkgHandleDict: `dict`, optional Dictionary of `lsst.daf.butler.DeferredDatasetHandle` for background info. Returns ------- visitCat: `afw.table.BaseCatalog` """ self.log.info("Assembling visitCatalog from %d visits", len(groupedHandles)) nCcd = len(camera) schema = self._makeFgcmVisitSchema(nCcd) visitCat = afwTable.BaseCatalog(schema) visitCat.reserve(len(groupedHandles)) visitCat.resize(len(groupedHandles)) visitCat['visit'] = list(groupedHandles.keys()) visitCat['used'] = 0 visitCat['sources_read'] = False # No matter what, fill the catalog. This will check if it was # already read. self._fillVisitCatalog(visitCat, groupedHandles, bkgHandleDict=bkgHandleDict) return visitCat
def run(self, dataRef, selectDataList=[]): if self.config.fileOutName == "": if self.config.dirOutName == "" : dirOutName = dataRef.getButler().mapper.root+"/"+self.config.coaddName+"Coadd-results" self.log.info("WARNING: the output file will be written in {0:s}.".format(dirOutName)) else: dirOutName = self.config.dirOutName fileOutName = "{0}/{1}/{2}/{3}/multiCat-{2}-{3}.fits".format(dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"]) else: fileOutName = self.config.fileOutName if os.path.isfile(fileOutName) and not self.config.clobber: self.log.info("File for %s exists. Exiting..." % (dataRef.dataId)) return self.log.info("Processing %s" % (dataRef.dataId)) filters = self.config.filters.split("^") dustCoefs = [float(c) for c in self.config.dustCoefs.split("^") ] ref = dataRef.get("deepCoadd_ref") catalogs = dict(self.readCatalog(dataRef, f) for f in filters) coadds = dict(self.readCoadd(dataRef, f) for f in filters) ghostFilters = [] if self.config.ghostFilters != "": ghostFilters = self.config.ghostFilters.split("^") self.log.info("Ghost filters: %s" % ghostFilters) # print ref.schema.getOrderedNames() # print dir(ref.schema) # print catalogs[filters[0]].schema.getOrderedNames() # return fluxMag0 = {} for f in filters: fluxMag0[f] = coadds[f].getCalib().getFluxMag0()[0] self.log.info("Mag ZP for filter {0:s}: {1:f}".format(f, 2.5*np.log10(fluxMag0[f]))) wcs = coadds[filters[0]].getWcs() pixel_scale = wcs.pixelScale().asDegrees()*3600.0 aperId = [int(a) for a in self.config.aperId.split(",")] """display which aperture diameter size will be used """ aperSize = [] for j, a in enumerate(aperId): aperSize.append(catalogs[filters[0]].getMetadata().get("flux_aperture_radii")[a] * 2.0 * pixel_scale) self.log.info("Diameter of flux apertures: {0:f}\"".format(aperSize[j])) """create new table table """ mergedSchema = afwTable.Schema() """define table fields """ fields=[] fields.append(mergedSchema.addField("id", type="L", doc="Unique id")) fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]")) fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]")) fields.append(mergedSchema.addField("tract", type="I", doc="tract number")) fields.append(mergedSchema.addField("patch", type="String", size=3, doc="patch number")) fields.append(mergedSchema.addField("refFilter", type="String", size=10, doc="Name of the filter used as reference")) fields.append(mergedSchema.addField("countInputs", type="I", doc="Number of input single exposures for the reference filter")) fields.append(mergedSchema.addField("detRadius", type="F", doc="Determinant radius for the object in the reference filter = sigma if gaussian [arcsec]")) fields.append(mergedSchema.addField("PSFDetRadius", type="F", doc="Determinant radius for the PSF in the reference filter at the object position = sigma if gaussian [arcsec]")) fields.append(mergedSchema.addField("cmodel_fracDev", type="F", doc="fraction of flux in de Vaucouleur component")) fields.append(mergedSchema.addField("blendedness", type="F", doc="Ranges from 0 (unblended) to 1 (blended)")) fields.append(mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]")) fields.append(mergedSchema.addField("extendedness", type="F", doc="probability of being extended from PSF/cmodel flux difference")) fields.append(mergedSchema.addField("hasBadCentroid", type="I", doc="1 if has bad centroid (but not used in islean")) fields.append(mergedSchema.addField("isSky", type="I", doc="1 if sky object")) fields.append(mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch")) fields.append(mergedSchema.addField("isParent", type="I", doc="1 if parent of a deblended object")) fields.append(mergedSchema.addField("isClean_refFilter", type="I", doc="1 if none of other flags is set for reference filter")) for f in filters+ghostFilters: fields.append(mergedSchema.addField("hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if interpolated, saturated, suspect, has CR at center or near bright object for filter {0:s}".format(f))) for f in filters+ghostFilters: fields.append(mergedSchema.addField("isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if offImage or in region masked EDGE or NO_DATA for filter {0:s}".format(f))) """photometry estimates """ photo = ["flux.aperture", "flux.kron", "flux.psf", "cmodel.flux"] for p in photo: for f in filters+ghostFilters: if p == "flux.aperture": for a in aperSize: keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_") fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s} within {2:f}\" diameter aperture".format(p,f,a))) fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f))) else: keyName = (p+"_"+f).replace(".", "_").replace("-", "_") fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s}".format(p,f))) fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f))) fields.append(mergedSchema.addField(p+"_flag".replace(".", "_"), type="I", doc="Highest flag value among filters for {0:s}".format(p))) """dust corrections """ for f in filters+ghostFilters: fields.append(mergedSchema.addField(("EB_V_corr_"+f).replace(".", "_").replace("-", "_"), type="F", doc="Milky way dust flux correction for filter {0:s}".format(f))) """create table object """ merged = afwTable.BaseCatalog(mergedSchema) N = len(ref) # count = 0 for i in range(N): # for i in range(10000,10200): # for i in range(1,100): """create new record """ record = merged.addNew() coord = ref[i].get('coord') """record common info from reference filter """ record.set(mergedSchema['id'].asKey(), ref[i].get('id')) record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees()) record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees()) record.set(mergedSchema['tract'].asKey(), dataRef.dataId["tract"]) record.set(mergedSchema['patch'].asKey(), dataRef.dataId["patch"]) record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs')) record.set(mergedSchema['detRadius'].asKey(), ref[i].get("shape.sdss").getDeterminantRadius()*pixel_scale) record.set(mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius()*pixel_scale) record.set(mergedSchema['cmodel_fracDev'].asKey(), ref[i].get('cmodel.fracDev')) record.set(mergedSchema['blendedness'].asKey(), ref[i].get('blendedness.abs.flux')) record.set(mergedSchema['extendedness'].asKey(), ref[i].get('classification.extendedness')) record.set(mergedSchema['hasBadCentroid'].asKey(), int(ref[i].get('centroid.sdss.flags'))) record.set(mergedSchema['isSky'].asKey(), int(ref[i].get('merge.footprint.sky'))) record.set(mergedSchema['isDuplicated'].asKey(), int(not ref[i].get('detect.is-primary'))) record.set(mergedSchema['isParent'].asKey(), int(ref[i].get('deblend.nchild') != 0)) """record the name of the filter used as reference """ for f in filters: name = afwImage.Filter(afwImage.Filter(f).getId()).getName() if ref[i].get("merge.measurement."+name): refName = f.replace(".", "_").replace("-", "_") record.set(mergedSchema["refFilter"].asKey(), refName) break """photometry measurement flags """ for p in photo: flag = 0 for f in filters: if catalogs[f][i].get(p+".flags") > flag: flag = catalogs[f][i].get(p+".flags") record.set(mergedSchema[p+"_flag".replace(".", "_")].asKey(), flag) """record bad photometry flag for each filter """ for f in filters: record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int((catalogs[f][i].get('flags.pixel.offimage')) | (catalogs[f][i].get('flags.pixel.edge')))) record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int( (catalogs[f][i].get('flags.pixel.interpolated.center')) \ | (catalogs[f][i].get('flags.pixel.saturated.center')) \ | (catalogs[f][i].get('flags.pixel.suspect.center')) \ | (catalogs[f][i].get('flags.pixel.cr.center')) \ | (catalogs[f][i].get('flags.pixel.bad')) \ | (catalogs[f][i].get('flags.pixel.bright.object.center')))) for f in ghostFilters: record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1) record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1) """record isClean flag for reference filter """ hasBadPhotometry_refFilter = \ (ref[i].get('flags.pixel.interpolated.center')) \ | (ref[i].get('flags.pixel.saturated.center')) \ | (ref[i].get('flags.pixel.suspect.center')) \ | (ref[i].get('flags.pixel.cr.center')) \ | (ref[i].get('flags.pixel.bad')) \ | (ref[i].get('flags.pixel.bright.object.center')) isNoData_refFilter = (ref[i].get('flags.pixel.offimage')) | (ref[i].get('flags.pixel.edge')) isSky_refFilter = ref[i].get('merge.footprint.sky') isDuplicated_refFilter = not ref[i].get('detect.is-primary') isParent_refFilter = ref[i].get('deblend.nchild') != 0 record.set(mergedSchema["isClean_refFilter"].asKey(), int( (not hasBadPhotometry_refFilter) \ & (not isNoData_refFilter) \ & (not isSky_refFilter) \ & (not isDuplicated_refFilter) \ & (not isParent_refFilter))) #isExtended = (ref[i].get('classification.extendedness')) #isExtended = (catalogs[f][i].get('flux.kron') > 0.8*catalogs[f][i].get('flux.psf')) | \ # (ref.get("shape.sdss").getDeterminantRadius() > 1.1*ref.get("shape.sdss.psf").getDeterminantRadius()) #if not (isExtended == isExtended): # isExtended = 0 #record.set(mergedSchema['isExtended'].asKey(), int(isExtended)) """dust correction """ EB_V = self.getDustCorrection(self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey())) record.set(mergedSchema['EB_V'].asKey(), EB_V) """flux in micro Jansky """ for f, dc in zip(filters, dustCoefs): record.set(mergedSchema[("EB_V_corr_"+f).replace(".", "_").replace("-", "_")].asKey(), pow(10.0, +0.4* dc * EB_V)) for p in photo: if p == "flux.aperture": for a in aperId: flux = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p)[a] / fluxMag0[f] flux_err = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p+".err")[a] / fluxMag0[f] keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_") record.set(mergedSchema[keyName].asKey(), flux) record.set(mergedSchema[keyName+"_err"].asKey(), flux_err) else: flux = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p)/fluxMag0[f] flux_err = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p+".err")/fluxMag0[f] keyName = (p+"_"+f).replace(".", "_").replace("-", "_") record.set(mergedSchema[keyName].asKey(), flux) record.set(mergedSchema[keyName+"_err"].asKey(), flux_err) # count += 1 """write catalog """ self.log.info("Writing {0:s}".format(fileOutName)) self.mkdir_p(os.path.dirname(fileOutName)) merged.writeFits(fileOutName) return
def getSchemaCatalogs(self): """!Return a dict containing an empty catalog representative of this task's output.""" transformedSrc = afwTable.BaseCatalog(self.mapper.getOutputSchema()) return {self.outputDataset: transformedSrc}
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None): startTime = time.time() # If both dataRefs are None, then we assume the caller does not # want to store checkpoint files. If both are set, we will # do checkpoint files. And if only one is set, this is potentially # unintentional and we will warn. if (visitCatDataRef is not None and starObsDataRef is None or visitCatDataRef is None and starObsDataRef is not None): self.log.warn( "Only one of visitCatDataRef and starObsDataRef are set, so " "no checkpoint files will be persisted.") if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: raise RuntimeError( "Must set calibFluxApertureRadius if doSubtractLocalBackground is True." ) # create our source schema. Use the first valid dataRef dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] sourceSchema = dataRef.get('src_schema', immediate=True).schema # Construct a mapping from ccd number to index camera = dataRef.get('camera') ccdMapping = {} for ccdIndex, detector in enumerate(camera): ccdMapping[detector.getId()] = ccdIndex approxPixelAreaFields = computeApproxPixelAreaFields(camera) sourceMapper = self._makeSourceMapper(sourceSchema) # We also have a temporary catalog that will accumulate aperture measurements aperMapper = self._makeAperMapper(sourceSchema) outputSchema = sourceMapper.getOutputSchema() if inStarObsCat is not None: fullCatalog = inStarObsCat comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) if not comp1 or not comp2: raise RuntimeError( "Existing fgcmStarObservations file found with mismatched schema." ) else: fullCatalog = afwTable.BaseCatalog(outputSchema) # FGCM will provide relative calibration for the flux in config.instFluxField instFluxKey = sourceSchema[self.config.instFluxField].asKey() instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() visitKey = outputSchema['visit'].asKey() ccdKey = outputSchema['ccd'].asKey() instMagKey = outputSchema['instMag'].asKey() instMagErrKey = outputSchema['instMagErr'].asKey() deltaMagBkgKey = outputSchema['deltaMagBkg'].asKey() # Prepare local background if desired if self.config.doSubtractLocalBackground: localBackgroundFluxKey = sourceSchema[ self.config.localBackgroundFluxField].asKey() localBackgroundArea = np.pi * calibFluxApertureRadius**2. aperOutputSchema = aperMapper.getOutputSchema() instFluxAperInKey = sourceSchema[ self.config.apertureInnerInstFluxField].asKey() instFluxErrAperInKey = sourceSchema[ self.config.apertureInnerInstFluxField + 'Err'].asKey() instFluxAperOutKey = sourceSchema[ self.config.apertureOuterInstFluxField].asKey() instFluxErrAperOutKey = sourceSchema[ self.config.apertureOuterInstFluxField + 'Err'].asKey() instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() k = 2.5 / np.log(10.) # loop over visits for ctr, visit in enumerate(visitCat): if visit['sources_read']: continue expTime = visit['exptime'] nStarInVisit = 0 # Reset the aperture catalog (per visit) aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) for dataRef in groupedDataRefs[visit['visit']]: ccdId = dataRef.dataId[self.config.ccdDataRefName] sources = dataRef.get(datasetType='src', flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) goodSrc = self.sourceSelector.selectSources(sources) # Need to add a selection based on the local background correction # if necessary if self.config.doSubtractLocalBackground: localBackground = localBackgroundArea * sources[ localBackgroundFluxKey] bad, = np.where( (sources[instFluxKey] - localBackground) <= 0.0) goodSrc.selected[bad] = False tempCat = afwTable.BaseCatalog(fullCatalog.schema) tempCat.reserve(goodSrc.selected.sum()) tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) tempCat[visitKey][:] = visit['visit'] tempCat[ccdKey][:] = ccdId # Compute "instrumental magnitude" by scaling flux with exposure time. scaledInstFlux = (sources[instFluxKey][goodSrc.selected] * visit['scaling'][ccdMapping[ccdId]]) tempCat[instMagKey][:] = (-2.5 * np.log10(scaledInstFlux) + 2.5 * np.log10(expTime)) # Compute the change in magnitude from the background offset if self.config.doSubtractLocalBackground: # At the moment we only adjust the flux and not the flux # error by the background because the error on # base_LocalBackground_instFlux is the rms error in the # background annulus, not the error on the mean in the # background estimate (which is much smaller, by sqrt(n) # pixels used to estimate the background, which we do not # have access to in this task). In the default settings, # the annulus is sufficiently large such that these # additional errors are are negligibly small (much less # than a mmag in quadrature). # This is the difference between the mag with background correction # and the mag without background correction. tempCat[deltaMagBkgKey][:] = ( -2.5 * np.log10(sources[instFluxKey][goodSrc.selected] - localBackground[goodSrc.selected]) - -2.5 * np.log10(sources[instFluxKey][goodSrc.selected])) else: tempCat[deltaMagBkgKey][:] = 0.0 # Compute instMagErr from instFluxErr/instFlux, any scaling # will cancel out. tempCat[instMagErrKey][:] = k * ( sources[instFluxErrKey][goodSrc.selected] / sources[instFluxKey][goodSrc.selected]) # Compute the jacobian from an approximate PixelAreaBoundedField tempCat['jacobian'] = approxPixelAreaFields[ccdId].evaluate( tempCat['x'], tempCat['y']) # Apply the jacobian if configured if self.config.doApplyWcsJacobian: tempCat[instMagKey][:] -= 2.5 * np.log10( tempCat['jacobian'][:]) fullCatalog.extend(tempCat) # And the aperture information # This does not need the jacobian because it is all locally relative tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) tempAperCat.reserve(goodSrc.selected.sum()) tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) with np.warnings.catch_warnings(): # Ignore warnings, we will filter infinities and # nans below. np.warnings.simplefilter("ignore") tempAperCat[instMagInKey][:] = -2.5 * np.log10( sources[instFluxAperInKey][goodSrc.selected]) tempAperCat[instMagErrInKey][:] = k * ( sources[instFluxErrAperInKey][goodSrc.selected] / sources[instFluxAperInKey][goodSrc.selected]) tempAperCat[instMagOutKey][:] = -2.5 * np.log10( sources[instFluxAperOutKey][goodSrc.selected]) tempAperCat[instMagErrOutKey][:] = k * ( sources[instFluxErrAperOutKey][goodSrc.selected] / sources[instFluxAperOutKey][goodSrc.selected]) aperVisitCatalog.extend(tempAperCat) nStarInVisit += len(tempCat) # Compute the median delta-aper if not aperVisitCatalog.isContiguous(): aperVisitCatalog = aperVisitCatalog.copy(deep=True) instMagIn = aperVisitCatalog[instMagInKey] instMagErrIn = aperVisitCatalog[instMagErrInKey] instMagOut = aperVisitCatalog[instMagOutKey] instMagErrOut = aperVisitCatalog[instMagErrOutKey] ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) visit['sources_read'] = True self.log.info( " Found %d good stars in visit %d (deltaAper = %.3f)" % (nStarInVisit, visit['visit'], visit['deltaAper'])) if ((ctr % self.config.nVisitsPerCheckpoint) == 0 and starObsDataRef is not None and visitCatDataRef is not None): # We need to persist both the stars and the visit catalog which gets # additional metadata from each visit. starObsDataRef.put(fullCatalog) visitCatDataRef.put(visitCat) self.log.info("Found all good star observations in %.2f s" % (time.time() - startTime)) return fullCatalog
def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, sourceSchema, camera, calibFluxApertureRadius=None): startTime = time.time() if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: raise RuntimeError( "Must set calibFluxApertureRadius if doSubtractLocalBackground is True." ) # To get the correct output schema, we use the legacy code. # We are not actually using this mapper, except to grab the outputSchema sourceMapper = self._makeSourceMapper(sourceSchema) outputSchema = sourceMapper.getOutputSchema() # Construct mapping from ccd number to index ccdMapping = {} for ccdIndex, detector in enumerate(camera): ccdMapping[detector.getId()] = ccdIndex approxPixelAreaFields = computeApproxPixelAreaFields(camera) fullCatalog = afwTable.BaseCatalog(outputSchema) visitKey = outputSchema['visit'].asKey() ccdKey = outputSchema['ccd'].asKey() instMagKey = outputSchema['instMag'].asKey() instMagErrKey = outputSchema['instMagErr'].asKey() deltaMagAperKey = outputSchema['deltaMagAper'].asKey() # Prepare local background if desired if self.config.doSubtractLocalBackground: localBackgroundArea = np.pi * calibFluxApertureRadius**2. columns = None k = 2.5 / np.log(10.) for counter, visit in enumerate(visitCat): expTime = visit['exptime'] handle = groupedHandles[visit['visit']][-1] if columns is None: inColumns = handle.get(component='columns') columns, detColumn = self._get_sourceTable_visit_columns( inColumns) df = handle.get(parameters={'columns': columns}) goodSrc = self.sourceSelector.selectSources(df) # Need to add a selection based on the local background correction # if necessary if self.config.doSubtractLocalBackground: localBackground = localBackgroundArea * df[ self.config.localBackgroundFluxField].values use, = np.where((goodSrc.selected) & ((df[self.config.instFluxField].values - localBackground) > 0.0)) else: use, = np.where(goodSrc.selected) tempCat = afwTable.BaseCatalog(fullCatalog.schema) tempCat.resize(use.size) tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) tempCat['x'][:] = df['x'].values[use] tempCat['y'][:] = df['y'].values[use] # The "visit" name in the parquet table is hard-coded. tempCat[visitKey][:] = df['visit'].values[use] tempCat[ccdKey][:] = df[detColumn].values[use] tempCat['psf_candidate'] = df[ self.config.psfCandidateName].values[use] with np.warnings.catch_warnings(): # Ignore warnings, we will filter infinites and nans below np.warnings.simplefilter("ignore") instMagInner = -2.5 * np.log10( df[self.config.apertureInnerInstFluxField].values[use]) instMagErrInner = k * ( df[self.config.apertureInnerInstFluxField + 'Err'].values[use] / df[self.config.apertureInnerInstFluxField].values[use]) instMagOuter = -2.5 * np.log10( df[self.config.apertureOuterInstFluxField].values[use]) instMagErrOuter = k * ( df[self.config.apertureOuterInstFluxField + 'Err'].values[use] / df[self.config.apertureOuterInstFluxField].values[use]) tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter # Set bad values to illegal values for fgcm. tempCat[deltaMagAperKey][ ~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 if self.config.doSubtractLocalBackground: # At the moment we only adjust the flux and not the flux # error by the background because the error on # base_LocalBackground_instFlux is the rms error in the # background annulus, not the error on the mean in the # background estimate (which is much smaller, by sqrt(n) # pixels used to estimate the background, which we do not # have access to in this task). In the default settings, # the annulus is sufficiently large such that these # additional errors are are negligibly small (much less # than a mmag in quadrature). # This is the difference between the mag with local background correction # and the mag without local background correction. tempCat['deltaMagBkg'] = ( -2.5 * np.log10(df[self.config.instFluxField].values[use] - localBackground[use]) - -2.5 * np.log10(df[self.config.instFluxField].values[use])) else: tempCat['deltaMagBkg'][:] = 0.0 # Need to loop over ccds here for detector in camera: ccdId = detector.getId() # used index for all observations with a given ccd use2 = (tempCat[ccdKey] == ccdId) tempCat['jacobian'][use2] = approxPixelAreaFields[ ccdId].evaluate(tempCat['x'][use2], tempCat['y'][use2]) scaledInstFlux = ( df[self.config.instFluxField].values[use[use2]] * visit['scaling'][ccdMapping[ccdId]]) tempCat[instMagKey][use2] = (-2.5 * np.log10(scaledInstFlux) + 2.5 * np.log10(expTime)) # Compute instMagErr from instFluxErr/instFlux, any scaling # will cancel out. tempCat[instMagErrKey][:] = k * ( df[self.config.instFluxField + 'Err'].values[use] / df[self.config.instFluxField].values[use]) # Apply the jacobian if configured if self.config.doApplyWcsJacobian: tempCat[instMagKey][:] -= 2.5 * np.log10( tempCat['jacobian'][:]) fullCatalog.extend(tempCat) deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) visit['sources_read'] = True self.log.info( " Found %d good stars in visit %d (deltaAper = %0.3f)", use.size, visit['visit'], visit['deltaAper']) self.log.info("Found all good star observations in %.2f s" % (time.time() - startTime)) return fullCatalog
def runQuantum(self, butlerQC, inputRefs, outputRefs): handleDict = {} handleDict['camera'] = butlerQC.get(inputRefs.camera) handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) handleDict['fgcmVisitCatalog'] = butlerQC.get( inputRefs.fgcmVisitCatalog) handleDict['fgcmStandardStars'] = butlerQC.get( inputRefs.fgcmStandardStars) if self.config.doZeropointOutput: handleDict['fgcmZeropoints'] = butlerQC.get( inputRefs.fgcmZeropoints) photoCalibRefDict = { photoCalibRef.dataId.byName()['visit']: photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib } if self.config.doAtmosphereOutput: handleDict['fgcmAtmosphereParameters'] = butlerQC.get( inputRefs.fgcmAtmosphereParameters) atmRefDict = { atmRef.dataId.byName()['visit']: atmRef for atmRef in outputRefs.fgcmTransmissionAtmosphere } if self.config.doReferenceCalibration: refConfig = LoadReferenceObjectsConfig() self.refObjLoader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), log=self.log, config=refConfig) else: self.refObjLoader = None struct = self.run(handleDict, self.config.physicalFilterMap) # Output the photoCalib exposure catalogs if struct.photoCalibCatalogs is not None: self.log.info("Outputting photoCalib catalogs.") for visit, expCatalog in struct.photoCalibCatalogs: butlerQC.put(expCatalog, photoCalibRefDict[visit]) self.log.info("Done outputting photoCalib catalogs.") # Output the atmospheres if struct.atmospheres is not None: self.log.info("Outputting atmosphere transmission files.") for visit, atm in struct.atmospheres: butlerQC.put(atm, atmRefDict[visit]) self.log.info("Done outputting atmosphere files.") if self.config.doReferenceCalibration: # Turn offset into simple catalog for persistence if necessary schema = afwTable.Schema() schema.addField('offset', type=np.float64, doc="Post-process calibration offset (mag)") offsetCat = afwTable.BaseCatalog(schema) offsetCat.resize(len(struct.offsets)) offsetCat['offset'][:] = struct.offsets butlerQC.put(offsetCat, outputRefs.fgcmOffsets) return
def setUp(self): self.schema = afwTable.Schema() self.schema.addField('test1', type='ArrayF', size=430000025) self.schema.addField('test2', type='ArrayF', size=430000025) self.cat = afwTable.BaseCatalog(self.schema)
def _makeLutCat(self, lutSchema, physicalFilterString, stdPhysicalFilterString, atmosphereTableName): """ Make the LUT schema Parameters ---------- lutSchema: `afwTable.schema` Lut catalog schema physicalFilterString: `str` Combined string of all the physicalFilters stdPhysicalFilterString: `str` Combined string of all the standard physicalFilters atmosphereTableName: `str` Name of the atmosphere table used to generate LUT Returns ------- lutCat: `afwTable.BaseCatalog` Lut catalog for persistence """ # The somewhat strange format is to make sure that # the rows of the afwTable do not get too large # (see DM-11419) lutCat = afwTable.BaseCatalog(lutSchema) lutCat.table.preallocate(14) # first fill the first index rec = lutCat.addNew() rec['tablename'] = atmosphereTableName rec['elevation'] = self.fgcmLutMaker.atmosphereTable.elevation rec['physicalFilters'] = physicalFilterString rec['stdPhysicalFilters'] = stdPhysicalFilterString rec['pmb'][:] = self.fgcmLutMaker.pmb rec['pmbFactor'][:] = self.fgcmLutMaker.pmbFactor rec['pmbElevation'] = self.fgcmLutMaker.pmbElevation rec['pwv'][:] = self.fgcmLutMaker.pwv rec['o3'][:] = self.fgcmLutMaker.o3 rec['tau'][:] = self.fgcmLutMaker.tau rec['lambdaNorm'] = self.fgcmLutMaker.lambdaNorm rec['alpha'][:] = self.fgcmLutMaker.alpha rec['zenith'][:] = self.fgcmLutMaker.zenith rec['nCcd'] = self.fgcmLutMaker.nCCD rec['pmbStd'] = self.fgcmLutMaker.pmbStd rec['pwvStd'] = self.fgcmLutMaker.pwvStd rec['o3Std'] = self.fgcmLutMaker.o3Std rec['tauStd'] = self.fgcmLutMaker.tauStd rec['alphaStd'] = self.fgcmLutMaker.alphaStd rec['zenithStd'] = self.fgcmLutMaker.zenithStd rec['lambdaRange'][:] = self.fgcmLutMaker.lambdaRange rec['lambdaStep'] = self.fgcmLutMaker.lambdaStep rec['lambdaStd'][:] = self.fgcmLutMaker.lambdaStd rec['lambdaStdFilter'][:] = self.fgcmLutMaker.lambdaStdFilter rec['i0Std'][:] = self.fgcmLutMaker.I0Std rec['i1Std'][:] = self.fgcmLutMaker.I1Std rec['i10Std'][:] = self.fgcmLutMaker.I10Std rec['i2Std'][:] = self.fgcmLutMaker.I2Std rec['lambdaB'][:] = self.fgcmLutMaker.lambdaB rec['atmLambda'][:] = self.fgcmLutMaker.atmLambda rec['atmStdTrans'][:] = self.fgcmLutMaker.atmStdTrans rec['luttype'] = 'I0' rec['lut'][:] = self.fgcmLutMaker.lut['I0'].flatten() # and add the rest rec = lutCat.addNew() rec['luttype'] = 'I1' rec['lut'][:] = self.fgcmLutMaker.lut['I1'].flatten() derivTypes = [ 'D_PMB', 'D_LNPWV', 'D_O3', 'D_LNTAU', 'D_ALPHA', 'D_SECZENITH', 'D_PMB_I1', 'D_LNPWV_I1', 'D_O3_I1', 'D_LNTAU_I1', 'D_ALPHA_I1', 'D_SECZENITH_I1' ] for derivType in derivTypes: rec = lutCat.addNew() rec['luttype'] = derivType rec['lut'][:] = self.fgcmLutMaker.lutDeriv[derivType].flatten() return lutCat
def _fgcmMatchStars(self, butler, visitCat): """ Use FGCM code to match observations into unique stars. Parameters ---------- butler: `lsst.daf.persistence.Butler` visitCat: `afw.table.BaseCatalog` Catalog with visit data for FGCM Outputs ------- Butler will output fgcmStarIds (matched star identifiers) and fgcmStarIndices (index values linking fgcmStarObservations to fgcmStarIds). """ obsCat = butler.get('fgcmStarObservations') # get filter names into a numpy array... # This is the type that is expected by the fgcm code visitFilterNames = np.zeros(len(visitCat), dtype='a2') for i in range(len(visitCat)): visitFilterNames[i] = visitCat[i]['filtername'] # match to put filterNames with observations visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit']) obsFilterNames = visitFilterNames[visitIndex] if self.config.doReferenceMatches: # Get the reference filter names, using the LUT lutCat = butler.get('fgcmLookUpTable') stdFilterDict = { filterName: stdFilter for (filterName, stdFilter) in zip(lutCat[0]['filterNames'].split(','), lutCat[0]['stdFilterNames'].split(',')) } stdLambdaDict = { stdFilter: stdLambda for (stdFilter, stdLambda) in zip(lutCat[0]['stdFilterNames'].split(','), lutCat[0]['lambdaStdFilter']) } del lutCat referenceFilterNames = self._getReferenceFilterNames( visitCat, stdFilterDict, stdLambdaDict) self.log.info("Using the following reference filters: %s" % (', '.join(referenceFilterNames))) else: # This should be an empty list referenceFilterNames = [] # make the fgcm starConfig dict starConfig = { 'logger': self.log, 'filterToBand': self.config.filterMap, 'requiredBands': self.config.requiredBands, 'minPerBand': self.config.minPerBand, 'matchRadius': self.config.matchRadius, 'isolationRadius': self.config.isolationRadius, 'matchNSide': self.config.matchNside, 'coarseNSide': self.config.coarseNside, 'densNSide': self.config.densityCutNside, 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 'primaryBands': self.config.primaryBands, 'referenceFilterNames': referenceFilterNames } # initialize the FgcmMakeStars object fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) # make the primary stars # note that the ra/dec native Angle format is radians # We determine the conversion from the native units (typically # radians) to degrees for the first observation. This allows us # to treate ra/dec as numpy arrays rather than Angles, which would # be approximately 600x slower. conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, obsCat['dec'] * conv, filterNameArray=obsFilterNames, bandSelected=False) # and match all the stars fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, obsCat['dec'] * conv, obsFilterNames) if self.config.doReferenceMatches: fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) # now persist objSchema = self._makeFgcmObjSchema() # make catalog and records fgcmStarIdCat = afwTable.BaseCatalog(objSchema) fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) for i in range(fgcmMakeStars.objIndexCat.size): fgcmStarIdCat.addNew() # fill the catalog fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat[ 'obsarrindex'] fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] butler.put(fgcmStarIdCat, 'fgcmStarIds') obsSchema = self._makeFgcmObsSchema() fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) for i in range(fgcmMakeStars.obsIndexCat.size): fgcmStarIndicesCat.addNew() fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat[ 'obsindex'] butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') if self.config.doReferenceMatches: refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) fgcmRefCat = afwTable.BaseCatalog(refSchema) fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) for i in range(fgcmMakeStars.referenceCat.size): fgcmRefCat.addNew() fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat[ 'refMagErr'] butler.put(fgcmRefCat, 'fgcmReferenceStars')
def _fgcmMakeAllStarObservations(self, butler, visitCat): """ Compile all good star observations from visits in visitCat Parameters ---------- butler: `lsst.daf.persistence.Butler` visitCat: `afw.table.BaseCatalog` Catalog with visit data for FGCM """ startTime = time.time() # create our source schema sourceSchema = butler.get('src_schema', immediate=True).schema sourceMapper = self._makeSourceMapper(sourceSchema) # We also have a temporary catalog that will accumulate aperture measurements aperMapper = self._makeAperMapper(sourceSchema) # we need to know the ccds... camera = butler.get('camera') outputSchema = sourceMapper.getOutputSchema() fullCatalog = afwTable.BaseCatalog(outputSchema) # FGCM will provide relative calibration for the flux in config.instFluxField instFluxKey = sourceSchema[self.config.instFluxField].asKey() instFluxErrKey = sourceSchema[self.config.instFluxField + 'Err'].asKey() visitKey = outputSchema['visit'].asKey() ccdKey = outputSchema['ccd'].asKey() instMagKey = outputSchema['instMag'].asKey() instMagErrKey = outputSchema['instMagErr'].asKey() aperOutputSchema = aperMapper.getOutputSchema() instFluxAperInKey = sourceSchema[ self.config.apertureInnerInstFluxField].asKey() instFluxErrAperInKey = sourceSchema[ self.config.apertureInnerInstFluxField + 'Err'].asKey() instFluxAperOutKey = sourceSchema[ self.config.apertureOuterInstFluxField].asKey() instFluxErrAperOutKey = sourceSchema[ self.config.apertureOuterInstFluxField + 'Err'].asKey() instMagInKey = aperOutputSchema['instMag_aper_inner'].asKey() instMagErrInKey = aperOutputSchema['instMagErr_aper_inner'].asKey() instMagOutKey = aperOutputSchema['instMag_aper_outer'].asKey() instMagErrOutKey = aperOutputSchema['instMagErr_aper_outer'].asKey() # loop over visits for visit in visitCat: expTime = visit['exptime'] nStarInVisit = 0 # Reset the aperture catalog (per visit) aperVisitCatalog = afwTable.BaseCatalog(aperOutputSchema) # loop over CCDs for ccdIndex, detector in enumerate(camera): ccdId = detector.getId() try: sources = butler.get( 'src', dataId={ self.config.visitDataRefName: visit['visit'], self.config.ccdDataRefName: ccdId }, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) except butlerExceptions.NoResults: # this is not a problem if this ccd isn't there continue goodSrc = self.sourceSelector.selectSources(sources) tempCat = afwTable.BaseCatalog(fullCatalog.schema) tempCat.reserve(goodSrc.selected.sum()) tempCat.extend(sources[goodSrc.selected], mapper=sourceMapper) tempCat[visitKey][:] = visit['visit'] tempCat[ccdKey][:] = ccdId # Compute "magnitude" by scaling flux with exposure time. # Add an arbitrary zeropoint that needs to be investigated. scaledInstFlux = sources[instFluxKey][ goodSrc.selected] * visit['scaling'][ccdIndex] tempCat[instMagKey][:] = (-2.5 * np.log10(scaledInstFlux) + 2.5 * np.log10(expTime)) # instMagErr is computed with original (unscaled) flux k = 2.5 / np.log(10.) tempCat[instMagErrKey][:] = k * ( sources[instFluxErrKey][goodSrc.selected] / sources[instFluxKey][goodSrc.selected]) if self.config.applyJacobian: tempCat[instMagKey][:] -= 2.5 * np.log10( tempCat['jacobian'][:]) fullCatalog.extend(tempCat) # And the aperture information tempAperCat = afwTable.BaseCatalog(aperVisitCatalog.schema) tempAperCat.reserve(goodSrc.selected.sum()) tempAperCat.extend(sources[goodSrc.selected], mapper=aperMapper) with np.warnings.catch_warnings(): # Ignore warnings, we will filter infinities and # nans below. np.warnings.simplefilter("ignore") tempAperCat[instMagInKey][:] = -2.5 * np.log10( sources[instFluxAperInKey][goodSrc.selected]) tempAperCat[instMagErrInKey][:] = (2.5 / np.log(10.)) * ( sources[instFluxErrAperInKey][goodSrc.selected] / sources[instFluxAperInKey][goodSrc.selected]) tempAperCat[instMagOutKey][:] = -2.5 * np.log10( sources[instFluxAperOutKey][goodSrc.selected]) tempAperCat[instMagErrOutKey][:] = (2.5 / np.log(10.)) * ( sources[instFluxErrAperOutKey][goodSrc.selected] / sources[instFluxAperOutKey][goodSrc.selected]) aperVisitCatalog.extend(tempAperCat) nStarInVisit += len(tempCat) # Compute the median delta-aper if not aperVisitCatalog.isContiguous(): aperVisitCatalog = aperVisitCatalog.copy(deep=True) instMagIn = aperVisitCatalog[instMagInKey] instMagErrIn = aperVisitCatalog[instMagErrInKey] instMagOut = aperVisitCatalog[instMagOutKey] instMagErrOut = aperVisitCatalog[instMagErrOutKey] ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) self.log.info( " Found %d good stars in visit %d (deltaAper = %.3f)" % (nStarInVisit, visit['visit'], visit['deltaAper'])) self.log.info("Found all good star observations in %.2f s" % (time.time() - startTime)) # Write all the observations butler.put(fullCatalog, 'fgcmStarObservations') # And overwrite the visitCatalog with delta_aper info butler.put(visitCat, 'fgcmVisitCatalog') self.log.info("Done with all stars in %.2f s" % (time.time() - startTime))
def run(self, dataRef, selectDataList=[]): self.log.info("Processing %s" % (dataRef.dataId)) filters = self.config.filters.split("^") catalogs = dict(self.readCatalog(dataRef, f) for f in filters) coadds = dict(self.readCoadd(dataRef, f) for f in filters) wcs = coadds[filters[0]].getWcs() pixel_scale = wcs.pixelScale().asDegrees() * 3600.0 ref = catalogs[self.config.refFilterName] # create new table table mergedSchema = afwTable.Schema() fields = [] # define table fields fields.append(mergedSchema.addField("id", type="L", doc="Unique id")) fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]")) fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]")) fields.append( mergedSchema.addField( "countInputs", type="I", doc="Number of input single exposures for the reference filter" )) fields.append( mergedSchema.addField( "PSFDetRadius", type="F", doc= "Determinant radius for the PSF at the object position = sigma if gaussian [arcsec]" )) fields.append( mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]")) fields.append( mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch")) fields.append( mergedSchema.addField( "isEdge", type="I", doc="1 if offImage or in region masked EDGE or NO_DATA")) fields.append( mergedSchema.addField( "hasBadPhotometry", type="I", doc= "1 if interpolated, saturated, suspect, has CR at center or near bright object" )) fields.append( mergedSchema.addField("isClean", type="I", doc="1 if none of other flags is set")) if self.config.hasDepthInfo: fields.append( mergedSchema.addField( "isFullDepthColor", type="I", doc="1 if point located in full depth and color area")) # create table object merged = afwTable.BaseCatalog(mergedSchema) N = len(ref) for i in range(N): # for i in range(100,110): # create new record record = merged.addNew() coord = ref[i].get('coord') # record if any of the filter is flagged as bad photometry for f in filters: hasBadPhotometry = (catalogs[f][i].get('flags.pixel.interpolated.center')) \ | (catalogs[f][i].get('flags.pixel.saturated.center')) \ | (catalogs[f][i].get('flags.pixel.suspect.center')) \ | (catalogs[f][i].get('flags.pixel.cr.center')) \ | (catalogs[f][i].get('flags.pixel.bad')) \ | (catalogs[f][i].get('flags.pixel.bright.object.center')) if hasBadPhotometry: break isDuplicated = not ref[i].get('detect.is-primary') isEdge = (ref[i].get('flags.pixel.offimage')) | ( ref[i].get('flags.pixel.edge')) isClean = (not hasBadPhotometry) & (not isDuplicated) & ( not isEdge) # record common info from reference filter record.set(mergedSchema['id'].asKey(), ref[i].get('id')) record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees()) record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees()) record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs')) record.set( mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius() * pixel_scale) record.set(mergedSchema['isDuplicated'].asKey(), int(isDuplicated)) record.set(mergedSchema['isEdge'].asKey(), int(isEdge)) record.set(mergedSchema['hasBadPhotometry'].asKey(), int(hasBadPhotometry)) record.set(mergedSchema['isClean'].asKey(), int(isClean)) if self.config.hasDepthInfo: record.set(mergedSchema['isFullDepthColor'].asKey(), int(ref[i].get('isFullDepthColor'))) # dust correction EB_V = self.getDustCorrection( self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey())) record.set(mergedSchema['EB_V'].asKey(), EB_V) # write catalog if self.config.fileOutName == "": # get output dir # TO DO: create new PAF # see /Users/coupon/local/source/hscPipe/install/DarwinX86/solvetansip/6.5.1p_hsc/python/hsc/meas/tansip/utils.py # and /Users/coupon/local/source/hscPipe/install/DarwinX86/pex_policy/HSC-4.0.0/tests/Policy_1.py if self.config.dirOutName == "": dirOutName = dataRef.getButler( ).mapper.root + "/" + self.config.coaddName + "Coadd-results" self.log.info( "WARNING: the output file will be written in {0:s}.". format(dirOutName)) else: dirOutName = self.config.dirOutName fileOutName = "{0}/{1}/{2}/{3}/multiRanCat-{2}-{3}.fits".format( dirOutName, "merged", dataRef.dataId["tract"], dataRef.dataId["patch"]) else: fileOutName = self.config.fileOutName self.log.info("Writing {0:s}".format(fileOutName)) self.mkdir_p(os.path.dirname(fileOutName)) merged.writeFits(fileOutName) # write catalog #self.log.info("Writing {0:s}".format(self.config.fileOutName)) #if self.config.fileOutName == "": # fileOutName = "{0}/{1}/{2}/{3}/MultiRanCat-{2}-{3}.fits".format(self.config.dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"]) # self.mkdir_p(os.path.dirname(fileOutName)) #else: # fileOutName = self.config.fileOutName #merged.writeFits(fileOutName) return
def runDataRef(self, files): """Run this task via CmdLineTask and Gen2 Butler. Parameters ---------- patchRefList : `list` of `lsst.daf.persistence.ButlerDataRef` A list of DataRefs for all filters in a single patch. """ self.prep() for file in files: cat = afwTable.BaseCatalog.readFits(file) mask = ((cat['moments_r_even'][:, 0] >= self.fluxMin) & (cat['moments_r_even'][:, 0] < self.fluxMax) & (cat['moments_r_cov_even'][:, 0] >= self.varMin) & (cat['moments_r_cov_even'][:, 0] < self.varMax) & (cat['z'] >= self.zMin) & (cat['z'] < self.zMax)) tgs = [] for rec in cat[mask]: cov_even = rec.get('moments_r_cov_even') cov_odd = rec.get('moments_r_cov_odd') full_cov_even = np.zeros((self.n_even, self.n_even), dtype=np.float32) full_cov_odd = np.zeros((self.n_odd, self.n_odd), dtype=np.float32) start = 0 for i in range(self.n_even): full_cov_even[i][i:] = cov_even[start:start + self.n_even - i] start += self.n_even - i for i in range(self.n_even): for j in range(i): full_cov_even[i, j] = full_cov_even[j, i] start = 0 for i in range(self.n_odd): full_cov_odd[i][i:] = cov_odd[start:start + self.n_odd - i] start += self.n_odd - i for i in range(self.n_odd): for j in range(i): full_cov_odd[i, j] = full_cov_odd[j, i] cov = self.bfd.MomentCov(full_cov_even, full_cov_odd) #mom_odd = np.array([0, 0] moment = self.bfd.Moment(rec.get('moments_r_even'), rec.get('moments_r_odd')) pos = np.array([rec.get('ra'), rec.get('dec')]) tg = self.bfd.TargetGalaxy(moment, cov, pos, rec.get('id')) tgs.append(tg) results = self.prior.getPqrCatalog(tgs[:10], self.config.threads, self.config.chunk) outcat = afwTable.BaseCatalog(self.schema) raKey = self.schema['coord_ra'].asKey() decKey = self.schema['coord_ra'].asKey() idKey = self.schema['id'].asKey() for r, rec in zip(results, cat[mask][:10]): out = outcat.addNew() out.set(self.pqrKey, r[0]._pqr) out.set(self.numKey, r[1]) out.set(self.uniqKey, r[2]) out.set(self.momKey, rec.get('moments_r_even')) out.set(self.momCovKey, rec.get('moments_r_cov_even')) out.set(self.zKey, rec.get('z')) out.set(self.g1Key, rec.get('g1')) out.set(self.g1Key, rec.get('g2')) out.set(self.kappaKey, rec.get('kappa')) out.set(self.magKey, rec.get('mag')) out.set(self.labelKey, self.config.label) out.set(raKey, rec.get('ra')*afwGeom.degrees) out.set(decKey, rec.get('dec')*afwGeom.degrees) out.set(idKey, rec.get('id')) outfile = file.replace('moment', 'pqr') outcat.writeFits(outfile)
def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, calibFluxApertureRadius=None, visitCatDataRef=None, starObsDataRef=None, inStarObsCat=None): startTime = time.time() # If both dataRefs are None, then we assume the caller does not # want to store checkpoint files. If both are set, we will # do checkpoint files. And if only one is set, this is potentially # unintentional and we will warn. if (visitCatDataRef is not None and starObsDataRef is None or visitCatDataRef is None and starObsDataRef is not None): self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so " "no checkpoint files will be persisted.") if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") # To get the correct output schema, we use similar code as fgcmBuildStarsTask # We are not actually using this mapper, except to grab the outputSchema dataRef = groupedDataRefs[list(groupedDataRefs.keys())[0]][0] sourceSchema = dataRef.get('src_schema', immediate=True).schema sourceMapper = self._makeSourceMapper(sourceSchema) outputSchema = sourceMapper.getOutputSchema() # Construct mapping from ccd number to index camera = dataRef.get('camera') ccdMapping = {} for ccdIndex, detector in enumerate(camera): ccdMapping[detector.getId()] = ccdIndex approxPixelAreaFields = computeApproxPixelAreaFields(camera) if inStarObsCat is not None: fullCatalog = inStarObsCat comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) if not comp1 or not comp2: raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.") else: fullCatalog = afwTable.BaseCatalog(outputSchema) visitKey = outputSchema['visit'].asKey() ccdKey = outputSchema['ccd'].asKey() instMagKey = outputSchema['instMag'].asKey() instMagErrKey = outputSchema['instMagErr'].asKey() # Prepare local background if desired if self.config.doSubtractLocalBackground: localBackgroundArea = np.pi*calibFluxApertureRadius**2. # Determine which columns we need from the sourceTable_visit catalogs columns = self._get_sourceTable_visit_columns() k = 2.5/np.log(10.) for counter, visit in enumerate(visitCat): # Check if these sources have already been read and stored in the checkpoint file if visit['sources_read']: continue expTime = visit['exptime'] dataRef = groupedDataRefs[visit['visit']][-1] srcTable = dataRef.get() df = srcTable.toDataFrame(columns) goodSrc = self.sourceSelector.selectSources(df) # Need to add a selection based on the local background correction # if necessary if self.config.doSubtractLocalBackground: localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values use, = np.where((goodSrc.selected) & ((df[self.config.instFluxField].values - localBackground) > 0.0)) else: use, = np.where(goodSrc.selected) tempCat = afwTable.BaseCatalog(fullCatalog.schema) tempCat.resize(use.size) tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) tempCat['x'][:] = df['x'].values[use] tempCat['y'][:] = df['y'].values[use] tempCat[visitKey][:] = df[self.config.visitDataRefName].values[use] tempCat[ccdKey][:] = df[self.config.ccdDataRefName].values[use] tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use] if self.config.doSubtractLocalBackground: # At the moment we only adjust the flux and not the flux # error by the background because the error on # base_LocalBackground_instFlux is the rms error in the # background annulus, not the error on the mean in the # background estimate (which is much smaller, by sqrt(n) # pixels used to estimate the background, which we do not # have access to in this task). In the default settings, # the annulus is sufficiently large such that these # additional errors are are negligibly small (much less # than a mmag in quadrature). # This is the difference between the mag with local background correction # and the mag without local background correction. tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] - localBackground[use]) - -2.5*np.log10(df[self.config.instFluxField].values[use])) else: tempCat['deltaMagBkg'][:] = 0.0 # Need to loop over ccds here for detector in camera: ccdId = detector.getId() # used index for all observations with a given ccd use2 = (tempCat[ccdKey] == ccdId) tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], tempCat['y'][use2]) scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] * visit['scaling'][ccdMapping[ccdId]]) tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) # Compute instMagErr from instFluxErr/instFlux, any scaling # will cancel out. tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] / df[self.config.instFluxField].values[use]) # Apply the jacobian if configured if self.config.doApplyWcsJacobian: tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) fullCatalog.extend(tempCat) # Now do the aperture information with np.warnings.catch_warnings(): # Ignore warnings, we will filter infinites and nans below np.warnings.simplefilter("ignore") instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] / df[self.config.apertureInnerInstFluxField].values[use]) instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] / df[self.config.apertureOuterInstFluxField].values[use]) ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) visit['sources_read'] = True self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", use.size, visit['visit'], visit['deltaAper']) if ((counter % self.config.nVisitsPerCheckpoint) == 0 and starObsDataRef is not None and visitCatDataRef is not None): # We need to persist both the stars and the visit catalog which gets # additional metadata from each visit. starObsDataRef.put(fullCatalog) visitCatDataRef.put(visitCat) self.log.info("Found all good star observations in %.2f s" % (time.time() - startTime)) return fullCatalog
def runQuantum(self, butlerQC, inputRefs, outputRefs): handleDict = butlerQC.get(inputRefs) self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs']))) # Run the build stars tasks tract = butlerQC.quantum.dataId['tract'] handleDict['sourceSchema'] = self.sourceSchema sourceTableHandles = handleDict['source_catalogs'] sourceTableHandleDict = { sourceTableHandle.dataId['visit']: sourceTableHandle for sourceTableHandle in sourceTableHandles } visitSummaryHandles = handleDict['visitSummary'] visitSummaryHandleDict = { visitSummaryHandle.dataId['visit']: visitSummaryHandle for visitSummaryHandle in visitSummaryHandles } handleDict['sourceTableHandleDict'] = sourceTableHandleDict handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict # And the outputs if self.config.fgcmOutputProducts.doZeropointOutput: photoCalibRefDict = { photoCalibRef.dataId.byName()['visit']: photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib } handleDict['fgcmPhotoCalibs'] = photoCalibRefDict if self.config.fgcmOutputProducts.doAtmosphereOutput: atmRefDict = { atmRef.dataId.byName()['visit']: atmRef for atmRef in outputRefs.fgcmTransmissionAtmosphere } handleDict['fgcmTransmissionAtmospheres'] = atmRefDict if self.config.fgcmBuildStars.doReferenceMatches: refConfig = LoadReferenceObjectsConfig() refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap loader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), config=refConfig, log=self.log) buildStarsRefObjLoader = loader else: buildStarsRefObjLoader = None if self.config.fgcmOutputProducts.doReferenceCalibration: refConfig = self.config.fgcmOutputProducts.refObjLoader loader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), config=refConfig, log=self.log) self.fgcmOutputProducts.refObjLoader = loader struct = self.run(handleDict, tract, buildStarsRefObjLoader=buildStarsRefObjLoader) if struct.photoCalibCatalogs is not None: self.log.info("Outputting photoCalib catalogs.") for visit, expCatalog in struct.photoCalibCatalogs: butlerQC.put(expCatalog, photoCalibRefDict[visit]) self.log.info("Done outputting photoCalib catalogs.") if struct.atmospheres is not None: self.log.info("Outputting atmosphere transmission files.") for visit, atm in struct.atmospheres: butlerQC.put(atm, atmRefDict[visit]) self.log.info("Done outputting atmosphere files.") # Turn raw repeatability into simple catalog for persistence schema = afwTable.Schema() schema.addField('rawRepeatability', type=np.float64, doc="Per-band raw repeatability in FGCM calibration.") repeatabilityCat = afwTable.BaseCatalog(schema) repeatabilityCat.resize(len(struct.repeatability)) repeatabilityCat['rawRepeatability'][:] = struct.repeatability butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) return
def fgcmMatchStars(self, visitCat, obsCat, lutHandle=None): """ Use FGCM code to match observations into unique stars. Parameters ---------- visitCat: `afw.table.BaseCatalog` Catalog with visit data for fgcm obsCat: `afw.table.BaseCatalog` Full catalog of star observations for fgcm lutHandle: `lsst.daf.butler.DeferredDatasetHandle`, optional Data reference to fgcm look-up table (used if matching reference stars). Returns ------- fgcmStarIdCat: `afw.table.BaseCatalog` Catalog of unique star identifiers and index keys fgcmStarIndicesCat: `afwTable.BaseCatalog` Catalog of unique star indices fgcmRefCat: `afw.table.BaseCatalog` Catalog of matched reference stars. Will be None if `config.doReferenceMatches` is False. """ # get filter names into a numpy array... # This is the type that is expected by the fgcm code visitFilterNames = np.zeros(len(visitCat), dtype='a30') for i in range(len(visitCat)): visitFilterNames[i] = visitCat[i]['physicalFilter'] # match to put filterNames with observations visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit']) obsFilterNames = visitFilterNames[visitIndex] if self.config.doReferenceMatches: # Get the reference filter names, using the LUT lutCat = lutHandle.get() stdFilterDict = { filterName: stdFilter for (filterName, stdFilter ) in zip(lutCat[0]['physicalFilters'].split(','), lutCat[0]['stdPhysicalFilters'].split(',')) } stdLambdaDict = { stdFilter: stdLambda for (stdFilter, stdLambda ) in zip(lutCat[0]['stdPhysicalFilters'].split(','), lutCat[0]['lambdaStdFilter']) } del lutCat referenceFilterNames = self._getReferenceFilterNames( visitCat, stdFilterDict, stdLambdaDict) self.log.info("Using the following reference filters: %s" % (', '.join(referenceFilterNames))) else: # This should be an empty list referenceFilterNames = [] # make the fgcm starConfig dict starConfig = { 'logger': self.log, 'useHtm': True, 'filterToBand': self.config.physicalFilterMap, 'requiredBands': self.config.requiredBands, 'minPerBand': self.config.minPerBand, 'matchRadius': self.config.matchRadius, 'isolationRadius': self.config.isolationRadius, 'matchNSide': self.config.matchNside, 'coarseNSide': self.config.coarseNside, 'densNSide': self.config.densityCutNside, 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 'randomSeed': self.config.randomSeed, 'primaryBands': self.config.primaryBands, 'referenceFilterNames': referenceFilterNames } # initialize the FgcmMakeStars object fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) # make the primary stars # note that the ra/dec native Angle format is radians # We determine the conversion from the native units (typically # radians) to degrees for the first observation. This allows us # to treate ra/dec as numpy arrays rather than Angles, which would # be approximately 600x slower. conv = obsCat[0]['ra'].asDegrees() / float(obsCat[0]['ra']) fgcmMakeStars.makePrimaryStars(obsCat['ra'] * conv, obsCat['dec'] * conv, filterNameArray=obsFilterNames, bandSelected=False) # and match all the stars fgcmMakeStars.makeMatchedStars(obsCat['ra'] * conv, obsCat['dec'] * conv, obsFilterNames) if self.config.doReferenceMatches: fgcmMakeStars.makeReferenceMatches(self.fgcmLoadReferenceCatalog) # now persist objSchema = self._makeFgcmObjSchema() # make catalog and records fgcmStarIdCat = afwTable.BaseCatalog(objSchema) fgcmStarIdCat.reserve(fgcmMakeStars.objIndexCat.size) for i in range(fgcmMakeStars.objIndexCat.size): fgcmStarIdCat.addNew() # fill the catalog fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] fgcmStarIdCat['obsArrIndex'][:] = fgcmMakeStars.objIndexCat[ 'obsarrindex'] fgcmStarIdCat['nObs'][:] = fgcmMakeStars.objIndexCat['nobs'] obsSchema = self._makeFgcmObsSchema() fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) fgcmStarIndicesCat.reserve(fgcmMakeStars.obsIndexCat.size) for i in range(fgcmMakeStars.obsIndexCat.size): fgcmStarIndicesCat.addNew() fgcmStarIndicesCat['obsIndex'][:] = fgcmMakeStars.obsIndexCat[ 'obsindex'] if self.config.doReferenceMatches: refSchema = self._makeFgcmRefSchema(len(referenceFilterNames)) fgcmRefCat = afwTable.BaseCatalog(refSchema) fgcmRefCat.reserve(fgcmMakeStars.referenceCat.size) for i in range(fgcmMakeStars.referenceCat.size): fgcmRefCat.addNew() fgcmRefCat['fgcm_id'][:] = fgcmMakeStars.referenceCat['fgcm_id'] fgcmRefCat['refMag'][:, :] = fgcmMakeStars.referenceCat['refMag'] fgcmRefCat['refMagErr'][:, :] = fgcmMakeStars.referenceCat[ 'refMagErr'] md = PropertyList() md.set("REFSTARS_FORMAT_VERSION", REFSTARS_FORMAT_VERSION) md.set("FILTERNAMES", referenceFilterNames) fgcmRefCat.setMetadata(md) else: fgcmRefCat = None return fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat