def __init__(self, butler=None, schema=None, refObjLoader=None, **kwargs): """! @param[in] butler: the butler can be used to retrieve schema or passed to the refObjLoader constructor in case it is needed. @param[in] schema: the schema of the source detection catalog used as input. @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference catalog. May be None if the butler argument is provided or all steps requiring a reference catalog are disabled. """ BatchPoolTask.__init__(self, **kwargs) if schema is None: assert butler is not None, "Butler not provided" schema = butler.get(self.config.coaddName + "Coadd_det_schema", immediate=True).schema self.butler = butler self.makeSubtask("detectCoaddSources") self.makeSubtask("mergeCoaddDetections", schema=schema) self.makeSubtask("measureCoaddSources", schema=afwTable.Schema( self.mergeCoaddDetections.schema), peakSchema=afwTable.Schema( self.mergeCoaddDetections.merged.getPeakSchema()), refObjLoader=refObjLoader, butler=butler) self.makeSubtask("mergeCoaddMeasurements", schema=afwTable.Schema( self.measureCoaddSources.schema)) self.makeSubtask("forcedPhotCoadd", refSchema=afwTable.Schema( self.mergeCoaddMeasurements.schema))
def testCatalogExtras(self): butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) schema = afwTable.Schema() aa = schema.addField("a", type=np.int32, doc="a") bb = schema.addField("b", type=np.float64, doc="b") catalog = lsst.afw.table.BaseCatalog(schema) row = catalog.addNew() row.set(aa, 12345) row.set(bb, 1.2345) size = len(catalog) dataId = dict(visit=123, ccd=45) butler.put(catalog, "someCatalog", dataId) filename = butler.get("someCatalog_filename", dataId)[0] try: self.assertTrue(os.path.exists(filename)) self.assertEqual(butler.get("someCatalog_schema", dataId), schema) self.assertEqual(butler.get("someCatalog_len", dataId), size) header = butler.get("someCatalog_md", dataId) self.assertEqual(header.getScalar("NAXIS2"), size) finally: try: os.remove(filename) except OSError as exc: print("Warning: could not remove file %r: %s" % (filename, exc))
def _wrap_result(data, column_names, table=None, index_col=None, coerce_float=True, column_dtypes=None, parse_dates=None): """Wrap result set of query in a afw table """ result_size = len(data) # Turn into columns first from pandas import lib data = list(lib.to_object_array_tuples(data).T) arrays = [lib.maybe_convert_objects(arr, try_float=True) for arr in data] _harmonize_columns(arrays, column_names, table, column_dtypes, parse_dates) schema = afw_table.Schema() # build schema for i, column_name in enumerate(column_names): column_type = arrays[i].dtype schema.addField(column_name, type=column_type.type) catalog = afw_table.BaseCatalog(schema) # Preallocate rows based on first column length catalog.preallocate(result_size) for i in range(result_size): record = catalog.addNew() for column_i in range(len(column_names)): record.set(column_names[column_i], arrays[column_i][i]) return catalog
def toTableDataTable(self, metadata): """Produce linearity catalog from table data Parameters ---------- metadata : `lsst.daf.base.PropertyList` Linearizer metadata Returns ------- catalog : `lsst.afw.table.BaseCatalog` Catalog to write """ schema = afwTable.Schema() dimensions = self.tableData.shape lut = schema.addField("LOOKUP_VALUES", type='ArrayF', size=dimensions[1], doc="linearity lookup data") catalog = afwTable.BaseCatalog(schema) catalog.resize(dimensions[0]) for ii in range(dimensions[0]): catalog[ii][lut] = np.array(self.tableData[ii], dtype=np.float32) metadata["LINEARITY_LOOKUP"] = True catalog.setMetadata(metadata) return catalog
def _makeFgcmRefSchema(self, nReferenceBands): """ Make a schema for the referenceCat from fgcmMakeStars Parameters ---------- nReferenceBands: `int` Number of reference bands Returns ------- schema: `afwTable.Schema` """ refSchema = afwTable.Schema() refSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') refSchema.addField('refMag', type='ArrayF', doc='Reference magnitude array (AB)', size=nReferenceBands) refSchema.addField('refMagErr', type='ArrayF', doc='Reference magnitude error array', size=nReferenceBands) return refSchema
def _get_afw_table(): schema = afw_table.Schema() aa = schema.addField("a", type=np.int64, doc="a") bb = schema.addField("b", type=np.float64, doc="b") cat = afw_table.BaseCatalog(schema) row = cat.addNew() row.set(aa, 12345) row.set(bb, 1.2345) row = cat.addNew() row.set(aa, 4321) row.set(bb, 4.123) return cat
def _makeFgcmVisitSchema(self, nCcd): """ Make a schema for an fgcmVisitCatalog Parameters ---------- nCcd: `int` Number of CCDs in the camera Returns ------- schema: `afwTable.Schema` """ schema = afwTable.Schema() schema.addField('visit', type=np.int64, doc="Visit number") schema.addField('physicalFilter', type=str, size=30, doc="Physical filter") schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") schema.addField('telrot', type=np.float64, doc="Camera rotation (deg)") schema.addField('mjd', type=np.float64, doc="MJD of visit") schema.addField('exptime', type=np.float32, doc="Exposure time") schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") # the following field is not used yet schema.addField('deepFlag', type=np.int32, doc="Deep observation") schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", size=nCcd) schema.addField('used', type=np.int32, doc="This visit has been ingested.") schema.addField('sources_read', type='Flag', doc="This visit had sources read.") return schema
def _makeFgcmObsSchema(self): """ Make a schema for the obsIndexCat from fgcmMakeStars Returns ------- schema: `afwTable.Schema` """ obsSchema = afwTable.Schema() obsSchema.addField('obsIndex', type=np.int32, doc='Index in observation table') return obsSchema
def __init__(self, *args, **kwargs): """ """ super(BuildCovarianceTask, self).__init__(*args, **kwargs) self.ncolors = 0 self.bfd = dbfd.BFDConfig(use_conc=self.config.use_conc, use_mag=self.config.use_mag, ncolors=self.ncolors) self.n_even = self.bfd.BFDConfig.MSIZE self.n_odd = self.bfd.BFDConfig.XYSIZE self.size_even = self.n_even * (self.n_even + 1) // 2 self.size_odd = self.n_odd * (self.n_odd + 1) // 2 self.schema = afwTable.Schema() self.labelKey = self.schema.addField("label", type=str, doc="name of bin", size=10) self.minKey = self.schema.addField("min", type=float, doc="minimum value of the variance") self.maxKey = self.schema.addField("max", type=float, doc="maximum value of the variance") self.isoCovEvenKey = self.schema.addField( "isoCovEven", doc="isotropized moment covariance matrix", type="ArrayF", size=self.size_even) self.isoCovOddKey = self.schema.addField( "isoCovOdd", doc="isotropized moment covariance matrix", type="ArrayF", size=self.size_odd) self.covEvenKey = self.schema.addField("covEven", doc="moment covariance matrix", type="ArrayF", size=self.size_even) self.covOddKey = self.schema.addField("covOdd", doc="moment covariance matrix", type="ArrayF", size=self.size_odd) self.catalog = afwTable.BaseCatalog(self.schema)
def _makeForcedSourceCatalog(objects): """Make a catalog containing a bunch of DiaFourceSources associated with the input diaObjects. """ # make some sources schema = afwTable.Schema() schema.addField("diaObjectId", "L") schema.addField("ccdVisitId", "L") schema.addField("flags", "L") catalog = afwTable.BaseCatalog(schema) oids = [] for obj in objects: record = catalog.addNew() record.set("diaObjectId", obj["id"]) record.set("ccdVisitId", 1) record.set("flags", 0) oids.append(obj["id"]) return catalog, oids
def _makeFgcmVisitSchema(self, nCcd): """ Make a schema for an fgcmVisitCatalog Parameters ---------- nCcd: `int` Number of CCDs in the camera Returns ------- schema: `afwTable.Schema` """ schema = afwTable.Schema() schema.addField('visit', type=np.int32, doc="Visit number") # Note that the FGCM code currently handles filternames up to 2 characters long schema.addField('filtername', type=str, size=2, doc="Filter name") schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") schema.addField('mjd', type=np.float64, doc="MJD of visit") schema.addField('exptime', type=np.float32, doc="Exposure time") schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") schema.addField('psfSigma', type=np.float32, doc="PSF sigma (reference CCD)") schema.addField('deltaAper', type=np.float32, doc="Delta-aperture") schema.addField('skyBackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") # the following field is not used yet schema.addField('deepFlag', type=np.int32, doc="Deep observation") schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", size=nCcd) return schema
def _makeFgcmObjSchema(self): """ Make a schema for the objIndexCat from fgcmMakeStars Returns ------- schema: `afwTable.Schema` """ objSchema = afwTable.Schema() objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') # Will investigate making these angles... objSchema.addField('ra', type=np.float64, doc='Mean object RA (deg)') objSchema.addField('dec', type=np.float64, doc='Mean object Dec (deg)') objSchema.addField('obsArrIndex', type=np.int32, doc='Index in obsIndexTable for first observation') objSchema.addField('nObs', type=np.int32, doc='Total number of observations') return objSchema
def makeAtmSchema(): """ Make the atmosphere schema Returns ------- atmSchema: `lsst.afw.table.Schema` """ atmSchema = afwTable.Schema() atmSchema.addField('visit', type=np.int32, doc='Visit number') atmSchema.addField('pmb', type=np.float64, doc='Barometric pressure (mb)') atmSchema.addField('pwv', type=np.float64, doc='Water vapor (mm)') atmSchema.addField('tau', type=np.float64, doc='Aerosol optical depth') atmSchema.addField('alpha', type=np.float64, doc='Aerosol slope') atmSchema.addField('o3', type=np.float64, doc='Ozone (dobson)') atmSchema.addField('secZenith', type=np.float64, doc='Secant(zenith) (~ airmass)') atmSchema.addField('cTrans', type=np.float64, doc='Transmission correction factor') atmSchema.addField('lamStd', type=np.float64, doc='Wavelength for transmission correction') return atmSchema
def run(self, dataRef, selectDataList=[]): if self.config.fileOutName == "": if self.config.dirOutName == "" : dirOutName = dataRef.getButler().mapper.root+"/"+self.config.coaddName+"Coadd-results" self.log.info("WARNING: the output file will be written in {0:s}.".format(dirOutName)) else: dirOutName = self.config.dirOutName fileOutName = "{0}/{1}/{2}/{3}/multiCat-{2}-{3}.fits".format(dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"]) else: fileOutName = self.config.fileOutName if os.path.isfile(fileOutName) and not self.config.clobber: self.log.info("File for %s exists. Exiting..." % (dataRef.dataId)) return self.log.info("Processing %s" % (dataRef.dataId)) filters = self.config.filters.split("^") dustCoefs = [float(c) for c in self.config.dustCoefs.split("^") ] ref = dataRef.get("deepCoadd_ref") catalogs = dict(self.readCatalog(dataRef, f) for f in filters) coadds = dict(self.readCoadd(dataRef, f) for f in filters) ghostFilters = [] if self.config.ghostFilters != "": ghostFilters = self.config.ghostFilters.split("^") self.log.info("Ghost filters: %s" % ghostFilters) # print ref.schema.getOrderedNames() # print dir(ref.schema) # print catalogs[filters[0]].schema.getOrderedNames() # return fluxMag0 = {} for f in filters: fluxMag0[f] = coadds[f].getCalib().getFluxMag0()[0] self.log.info("Mag ZP for filter {0:s}: {1:f}".format(f, 2.5*np.log10(fluxMag0[f]))) wcs = coadds[filters[0]].getWcs() pixel_scale = wcs.pixelScale().asDegrees()*3600.0 aperId = [int(a) for a in self.config.aperId.split(",")] """display which aperture diameter size will be used """ aperSize = [] for j, a in enumerate(aperId): aperSize.append(catalogs[filters[0]].getMetadata().get("flux_aperture_radii")[a] * 2.0 * pixel_scale) self.log.info("Diameter of flux apertures: {0:f}\"".format(aperSize[j])) """create new table table """ mergedSchema = afwTable.Schema() """define table fields """ fields=[] fields.append(mergedSchema.addField("id", type="L", doc="Unique id")) fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]")) fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]")) fields.append(mergedSchema.addField("tract", type="I", doc="tract number")) fields.append(mergedSchema.addField("patch", type="String", size=3, doc="patch number")) fields.append(mergedSchema.addField("refFilter", type="String", size=10, doc="Name of the filter used as reference")) fields.append(mergedSchema.addField("countInputs", type="I", doc="Number of input single exposures for the reference filter")) fields.append(mergedSchema.addField("detRadius", type="F", doc="Determinant radius for the object in the reference filter = sigma if gaussian [arcsec]")) fields.append(mergedSchema.addField("PSFDetRadius", type="F", doc="Determinant radius for the PSF in the reference filter at the object position = sigma if gaussian [arcsec]")) fields.append(mergedSchema.addField("cmodel_fracDev", type="F", doc="fraction of flux in de Vaucouleur component")) fields.append(mergedSchema.addField("blendedness", type="F", doc="Ranges from 0 (unblended) to 1 (blended)")) fields.append(mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]")) fields.append(mergedSchema.addField("extendedness", type="F", doc="probability of being extended from PSF/cmodel flux difference")) fields.append(mergedSchema.addField("hasBadCentroid", type="I", doc="1 if has bad centroid (but not used in islean")) fields.append(mergedSchema.addField("isSky", type="I", doc="1 if sky object")) fields.append(mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch")) fields.append(mergedSchema.addField("isParent", type="I", doc="1 if parent of a deblended object")) fields.append(mergedSchema.addField("isClean_refFilter", type="I", doc="1 if none of other flags is set for reference filter")) for f in filters+ghostFilters: fields.append(mergedSchema.addField("hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if interpolated, saturated, suspect, has CR at center or near bright object for filter {0:s}".format(f))) for f in filters+ghostFilters: fields.append(mergedSchema.addField("isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_")), type="I", doc="1 if offImage or in region masked EDGE or NO_DATA for filter {0:s}".format(f))) """photometry estimates """ photo = ["flux.aperture", "flux.kron", "flux.psf", "cmodel.flux"] for p in photo: for f in filters+ghostFilters: if p == "flux.aperture": for a in aperSize: keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_") fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s} within {2:f}\" diameter aperture".format(p,f,a))) fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f))) else: keyName = (p+"_"+f).replace(".", "_").replace("-", "_") fields.append(mergedSchema.addField(keyName, type="F", doc="{0:s} for filter {1:s}".format(p,f))) fields.append(mergedSchema.addField(keyName+"_err", type="F", doc="{0:s} error for filter {1:s}".format(p,f))) fields.append(mergedSchema.addField(p+"_flag".replace(".", "_"), type="I", doc="Highest flag value among filters for {0:s}".format(p))) """dust corrections """ for f in filters+ghostFilters: fields.append(mergedSchema.addField(("EB_V_corr_"+f).replace(".", "_").replace("-", "_"), type="F", doc="Milky way dust flux correction for filter {0:s}".format(f))) """create table object """ merged = afwTable.BaseCatalog(mergedSchema) N = len(ref) # count = 0 for i in range(N): # for i in range(10000,10200): # for i in range(1,100): """create new record """ record = merged.addNew() coord = ref[i].get('coord') """record common info from reference filter """ record.set(mergedSchema['id'].asKey(), ref[i].get('id')) record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees()) record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees()) record.set(mergedSchema['tract'].asKey(), dataRef.dataId["tract"]) record.set(mergedSchema['patch'].asKey(), dataRef.dataId["patch"]) record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs')) record.set(mergedSchema['detRadius'].asKey(), ref[i].get("shape.sdss").getDeterminantRadius()*pixel_scale) record.set(mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius()*pixel_scale) record.set(mergedSchema['cmodel_fracDev'].asKey(), ref[i].get('cmodel.fracDev')) record.set(mergedSchema['blendedness'].asKey(), ref[i].get('blendedness.abs.flux')) record.set(mergedSchema['extendedness'].asKey(), ref[i].get('classification.extendedness')) record.set(mergedSchema['hasBadCentroid'].asKey(), int(ref[i].get('centroid.sdss.flags'))) record.set(mergedSchema['isSky'].asKey(), int(ref[i].get('merge.footprint.sky'))) record.set(mergedSchema['isDuplicated'].asKey(), int(not ref[i].get('detect.is-primary'))) record.set(mergedSchema['isParent'].asKey(), int(ref[i].get('deblend.nchild') != 0)) """record the name of the filter used as reference """ for f in filters: name = afwImage.Filter(afwImage.Filter(f).getId()).getName() if ref[i].get("merge.measurement."+name): refName = f.replace(".", "_").replace("-", "_") record.set(mergedSchema["refFilter"].asKey(), refName) break """photometry measurement flags """ for p in photo: flag = 0 for f in filters: if catalogs[f][i].get(p+".flags") > flag: flag = catalogs[f][i].get(p+".flags") record.set(mergedSchema[p+"_flag".replace(".", "_")].asKey(), flag) """record bad photometry flag for each filter """ for f in filters: record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int((catalogs[f][i].get('flags.pixel.offimage')) | (catalogs[f][i].get('flags.pixel.edge')))) record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), int( (catalogs[f][i].get('flags.pixel.interpolated.center')) \ | (catalogs[f][i].get('flags.pixel.saturated.center')) \ | (catalogs[f][i].get('flags.pixel.suspect.center')) \ | (catalogs[f][i].get('flags.pixel.cr.center')) \ | (catalogs[f][i].get('flags.pixel.bad')) \ | (catalogs[f][i].get('flags.pixel.bright.object.center')))) for f in ghostFilters: record.set(mergedSchema["isNoData_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1) record.set(mergedSchema["hasBadPhotometry_{0:s}".format(f.replace(".", "_").replace("-", "_"))].asKey(), 1) """record isClean flag for reference filter """ hasBadPhotometry_refFilter = \ (ref[i].get('flags.pixel.interpolated.center')) \ | (ref[i].get('flags.pixel.saturated.center')) \ | (ref[i].get('flags.pixel.suspect.center')) \ | (ref[i].get('flags.pixel.cr.center')) \ | (ref[i].get('flags.pixel.bad')) \ | (ref[i].get('flags.pixel.bright.object.center')) isNoData_refFilter = (ref[i].get('flags.pixel.offimage')) | (ref[i].get('flags.pixel.edge')) isSky_refFilter = ref[i].get('merge.footprint.sky') isDuplicated_refFilter = not ref[i].get('detect.is-primary') isParent_refFilter = ref[i].get('deblend.nchild') != 0 record.set(mergedSchema["isClean_refFilter"].asKey(), int( (not hasBadPhotometry_refFilter) \ & (not isNoData_refFilter) \ & (not isSky_refFilter) \ & (not isDuplicated_refFilter) \ & (not isParent_refFilter))) #isExtended = (ref[i].get('classification.extendedness')) #isExtended = (catalogs[f][i].get('flux.kron') > 0.8*catalogs[f][i].get('flux.psf')) | \ # (ref.get("shape.sdss").getDeterminantRadius() > 1.1*ref.get("shape.sdss.psf").getDeterminantRadius()) #if not (isExtended == isExtended): # isExtended = 0 #record.set(mergedSchema['isExtended'].asKey(), int(isExtended)) """dust correction """ EB_V = self.getDustCorrection(self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey())) record.set(mergedSchema['EB_V'].asKey(), EB_V) """flux in micro Jansky """ for f, dc in zip(filters, dustCoefs): record.set(mergedSchema[("EB_V_corr_"+f).replace(".", "_").replace("-", "_")].asKey(), pow(10.0, +0.4* dc * EB_V)) for p in photo: if p == "flux.aperture": for a in aperId: flux = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p)[a] / fluxMag0[f] flux_err = pow(10.0, 23.9/2.5) * catalogs[f][i].get(p+".err")[a] / fluxMag0[f] keyName = (p+"_"+str(int(a))+"arcsec_"+f).replace(".", "_").replace("-", "_") record.set(mergedSchema[keyName].asKey(), flux) record.set(mergedSchema[keyName+"_err"].asKey(), flux_err) else: flux = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p)/fluxMag0[f] flux_err = pow(10.0, 23.9/2.5)*catalogs[f][i].get(p+".err")/fluxMag0[f] keyName = (p+"_"+f).replace(".", "_").replace("-", "_") record.set(mergedSchema[keyName].asKey(), flux) record.set(mergedSchema[keyName+"_err"].asKey(), flux_err) # count += 1 """write catalog """ self.log.info("Writing {0:s}".format(fileOutName)) self.mkdir_p(os.path.dirname(fileOutName)) merged.writeFits(fileOutName) return
def _makeLutSchema(self, physicalFilterString, stdPhysicalFilterString, atmosphereTableName): """ Make the LUT schema Parameters ---------- physicalFilterString: `str` Combined string of all the physicalFilters stdPhysicalFilterString: `str` Combined string of all the standard physicalFilters atmosphereTableName: `str` Name of the atmosphere table used to generate LUT Returns ------- lutSchema: `afwTable.schema` """ lutSchema = afwTable.Schema() lutSchema.addField('tablename', type=str, doc='Atmosphere table name', size=len(atmosphereTableName)) lutSchema.addField('elevation', type=float, doc="Telescope elevation used for LUT") lutSchema.addField('physicalFilters', type=str, doc='physicalFilters in LUT', size=len(physicalFilterString)) lutSchema.addField('stdPhysicalFilters', type=str, doc='Standard physicalFilters in LUT', size=len(stdPhysicalFilterString)) lutSchema.addField('pmb', type='ArrayD', doc='Barometric Pressure', size=self.fgcmLutMaker.pmb.size) lutSchema.addField('pmbFactor', type='ArrayD', doc='PMB scaling factor', size=self.fgcmLutMaker.pmb.size) lutSchema.addField('pmbElevation', type=np.float64, doc='PMB Scaling at elevation') lutSchema.addField('pwv', type='ArrayD', doc='Preciptable Water Vapor', size=self.fgcmLutMaker.pwv.size) lutSchema.addField('o3', type='ArrayD', doc='Ozone', size=self.fgcmLutMaker.o3.size) lutSchema.addField('tau', type='ArrayD', doc='Aerosol optical depth', size=self.fgcmLutMaker.tau.size) lutSchema.addField('lambdaNorm', type=np.float64, doc='AOD wavelength') lutSchema.addField('alpha', type='ArrayD', doc='Aerosol alpha', size=self.fgcmLutMaker.alpha.size) lutSchema.addField('zenith', type='ArrayD', doc='Zenith angle', size=self.fgcmLutMaker.zenith.size) lutSchema.addField('nCcd', type=np.int32, doc='Number of CCDs') # and the standard values lutSchema.addField('pmbStd', type=np.float64, doc='PMB Standard') lutSchema.addField('pwvStd', type=np.float64, doc='PWV Standard') lutSchema.addField('o3Std', type=np.float64, doc='O3 Standard') lutSchema.addField('tauStd', type=np.float64, doc='Tau Standard') lutSchema.addField('alphaStd', type=np.float64, doc='Alpha Standard') lutSchema.addField('zenithStd', type=np.float64, doc='Zenith angle Standard') lutSchema.addField('lambdaRange', type='ArrayD', doc='Wavelength range', size=2) lutSchema.addField('lambdaStep', type=np.float64, doc='Wavelength step') lutSchema.addField('lambdaStd', type='ArrayD', doc='Standard Wavelength', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('lambdaStdFilter', type='ArrayD', doc='Standard Wavelength (raw)', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i0Std', type='ArrayD', doc='I0 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i1Std', type='ArrayD', doc='I1 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i10Std', type='ArrayD', doc='I10 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i2Std', type='ArrayD', doc='I2 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('lambdaB', type='ArrayD', doc='Wavelength for passband (no atm)', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('atmLambda', type='ArrayD', doc='Atmosphere wavelengths (Angstrom)', size=self.fgcmLutMaker.atmLambda.size) lutSchema.addField('atmStdTrans', type='ArrayD', doc='Standard Atmosphere Throughput', size=self.fgcmLutMaker.atmStdTrans.size) # and the look-up-tables lutSchema.addField('luttype', type=str, size=20, doc='Look-up table type') lutSchema.addField('lut', type='ArrayF', doc='Look-up table for luttype', size=self.fgcmLutMaker.lut['I0'].size) return lutSchema
def runQuantum(self, butlerQC, inputRefs, outputRefs): handleDict = butlerQC.get(inputRefs) self.log.info("Running with %d sourceTable_visit handles", (len(handleDict['source_catalogs']))) # Run the build stars tasks tract = butlerQC.quantum.dataId['tract'] handleDict['sourceSchema'] = self.sourceSchema sourceTableHandles = handleDict['source_catalogs'] sourceTableHandleDict = { sourceTableHandle.dataId['visit']: sourceTableHandle for sourceTableHandle in sourceTableHandles } visitSummaryHandles = handleDict['visitSummary'] visitSummaryHandleDict = { visitSummaryHandle.dataId['visit']: visitSummaryHandle for visitSummaryHandle in visitSummaryHandles } handleDict['sourceTableHandleDict'] = sourceTableHandleDict handleDict['visitSummaryHandleDict'] = visitSummaryHandleDict # And the outputs if self.config.fgcmOutputProducts.doZeropointOutput: photoCalibRefDict = { photoCalibRef.dataId.byName()['visit']: photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib } handleDict['fgcmPhotoCalibs'] = photoCalibRefDict if self.config.fgcmOutputProducts.doAtmosphereOutput: atmRefDict = { atmRef.dataId.byName()['visit']: atmRef for atmRef in outputRefs.fgcmTransmissionAtmosphere } handleDict['fgcmTransmissionAtmospheres'] = atmRefDict if self.config.fgcmBuildStars.doReferenceMatches: refConfig = LoadReferenceObjectsConfig() refConfig.filterMap = self.config.fgcmBuildStars.fgcmLoadReferenceCatalog.filterMap loader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), config=refConfig, log=self.log) buildStarsRefObjLoader = loader else: buildStarsRefObjLoader = None if self.config.fgcmOutputProducts.doReferenceCalibration: refConfig = self.config.fgcmOutputProducts.refObjLoader loader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), config=refConfig, log=self.log) self.fgcmOutputProducts.refObjLoader = loader struct = self.run(handleDict, tract, buildStarsRefObjLoader=buildStarsRefObjLoader) if struct.photoCalibCatalogs is not None: self.log.info("Outputting photoCalib catalogs.") for visit, expCatalog in struct.photoCalibCatalogs: butlerQC.put(expCatalog, photoCalibRefDict[visit]) self.log.info("Done outputting photoCalib catalogs.") if struct.atmospheres is not None: self.log.info("Outputting atmosphere transmission files.") for visit, atm in struct.atmospheres: butlerQC.put(atm, atmRefDict[visit]) self.log.info("Done outputting atmosphere files.") # Turn raw repeatability into simple catalog for persistence schema = afwTable.Schema() schema.addField('rawRepeatability', type=np.float64, doc="Per-band raw repeatability in FGCM calibration.") repeatabilityCat = afwTable.BaseCatalog(schema) repeatabilityCat.resize(len(struct.repeatability)) repeatabilityCat['rawRepeatability'][:] = struct.repeatability butlerQC.put(repeatabilityCat, outputRefs.fgcmRepeatability) return
true_y.append(gtrue.y) for filter_name, filter_ in filters.items(): tmp = cgal.drawImage(filter_, image=images[filter_name], add_to_image=True, offset=(offsetx, offsety)) gtrue.fluxes[filter_name] = tmp.added_flux true_gals.append(gtrue) for filter_name, filter_ in filters.items(): tot_images[filter_name][bounds] = images[filter_name] # Write out truth catalogs true_schema = afwTable.Schema() xKey = true_schema.addField("x", type=float, doc="x position") yKey = true_schema.addField("y", type=float, doc="y position") radKey = true_schema.addField("size", type=float, doc="size") zKey = true_schema.addField("redshift", type=float, doc="redshift") fluxKeys = {} for filter_ in filter_names: fluxKeys[filter_] = true_schema.addField("flux_%s" % filter_, type=float, doc="flux in %s" % filter_) w_vals = np.arange(100, 1500, 5) wKey = true_schema.addField("wave", type="ArrayF", doc="wavelengths", size=len(w_vals)) sedKey = true_schema.addField("sed",
def toAmpTable(self, metadata): """Produce linearity catalog Parameters ---------- metadata : `lsst.daf.base.PropertyList` Linearizer metadata Returns ------- catalog : `lsst.afw.table.BaseCatalog` Catalog to write """ metadata["LINEARITY_SCHEMA"] = "Linearity table" metadata["LINEARITY_VERSION"] = 1 # Now pack it into a fits table. length = max([ len(self.linearityCoeffs[x]) for x in self.linearityCoeffs.keys() ]) schema = afwTable.Schema() names = schema.addField("AMPLIFIER_NAME", type="String", size=16, doc="linearity amplifier name") types = schema.addField("TYPE", type="String", size=16, doc="linearity type names") coeffs = schema.addField("COEFFS", type="ArrayD", size=length, doc="linearity coefficients") boxX = schema.addField("BBOX_X0", type="I", doc="linearity bbox minimum x") boxY = schema.addField("BBOX_Y0", type="I", doc="linearity bbox minimum y") boxDx = schema.addField("BBOX_DX", type="I", doc="linearity bbox x dimension") boxDy = schema.addField("BBOX_DY", type="I", doc="linearity bbox y dimension") if (self.fitParams): lengthFitParams = max( [len(self.fitParams[x]) for x in self.fitParams.keys()]) fitParams = schema.addField( "FIT_PARAMS", type="ArrayD", size=lengthFitParams, doc="parameters of linearity polynomial fit") fitParamsErr = schema.addField( "FIT_PARAMS_ERR", type="ArrayD", size=lengthFitParams, doc="errors of parameters of linearity polynomial fit") reducedChiSquared = schema.addField( "RED_CHI_SQ", type="D", doc="unweighted reduced chi sq. from linearity pol. fit") catalog = afwTable.BaseCatalog(schema) catalog.resize(len(self.linearityCoeffs.keys())) for ii, ampName in enumerate(self.linearityType): catalog[ii][names] = ampName catalog[ii][types] = self.linearityType[ampName] catalog[ii][coeffs] = np.array(self.linearityCoeffs[ampName], dtype=float) if (self.fitParams): catalog[ii][fitParams] = np.array(self.fitParams[ampName], dtype=float) catalog[ii][fitParamsErr] = np.array( self.fitParamsErr[ampName], dtype=float) catalog[ii][ reducedChiSquared] = self.linearityFitReducedChiSquared[ ampName] bbox = self.linearityBBox[ampName] catalog[ii][boxX], catalog[ii][boxY] = bbox.getMin() catalog[ii][boxDx], catalog[ii][boxDy] = bbox.getDimensions() catalog.setMetadata(metadata) return catalog
def _fgcmMatchStars(self, butler, visitCat): """ Use FGCM code to match observations into unique stars. Parameters ---------- butler: lsst.daf.persistence.Butler visitCat: afw.table.BaseCatalog Catalog with visit data for FGCM Returns ------- None """ obsCat = butler.get('fgcmStarObservations') # get filter names into a numpy array... visitFilterNames = np.zeros(len(visitCat), dtype='a2') for i in xrange(len(visitCat)): visitFilterNames[i] = visitCat[i]['filtername'] # match to put filterNames with observations visitIndex = np.searchsorted(visitCat['visit'], obsCat['visit']) obsFilterNames = visitFilterNames[visitIndex] # make the fgcm starConfig dict starConfig = { 'logger': self.log, 'filterToBand': self.config.filterToBand, 'requiredBands': self.config.requiredBands, 'minPerBand': self.config.minPerBand, 'matchRadius': self.config.matchRadius, 'isolationRadius': self.config.isolationRadius, 'matchNSide': self.config.matchNside, 'coarseNSide': self.config.coarseNside, 'densNSide': self.config.densityCutNside, 'densMaxPerPixel': self.config.densityCutMaxPerPixel, 'referenceBands': self.config.referenceBands } # initialize the FgcmMakeStars object fgcmMakeStars = fgcm.FgcmMakeStars(starConfig) # make the reference stars # note that the ra/dec native Angle format is radians fgcmMakeStars.makeReferenceStars(np.rad2deg(obsCat['ra']), np.rad2deg(obsCat['dec']), filterNameArray=obsFilterNames, bandSelected=False) # and match all the stars fgcmMakeStars.makeMatchedStars(np.rad2deg(obsCat['ra']), np.rad2deg(obsCat['dec']), obsFilterNames) # now persist # afwTable for objects objSchema = afwTable.Schema() objSchema.addField('fgcm_id', type=np.int32, doc='FGCM Unique ID') # FIXME: should be angle? objSchema.addField('ra', type=np.float64, doc='Mean object RA') objSchema.addField('dec', type=np.float64, doc='Mean object Dec') objSchema.addField('obsarrindex', type=np.int32, doc='Index in obsIndexTable for first observation') objSchema.addField('nobs', type=np.int32, doc='Total number of observations') # make catalog and records fgcmStarIdCat = afwTable.BaseCatalog(objSchema) fgcmStarIdCat.table.preallocate(fgcmMakeStars.objIndexCat.size) for i in xrange(fgcmMakeStars.objIndexCat.size): fgcmStarIdCat.addNew() # fill the catalog fgcmStarIdCat['fgcm_id'][:] = fgcmMakeStars.objIndexCat['fgcm_id'] fgcmStarIdCat['ra'][:] = fgcmMakeStars.objIndexCat['ra'] fgcmStarIdCat['dec'][:] = fgcmMakeStars.objIndexCat['dec'] fgcmStarIdCat['obsarrindex'][:] = fgcmMakeStars.objIndexCat[ 'obsarrindex'] fgcmStarIdCat['nobs'][:] = fgcmMakeStars.objIndexCat['nobs'] butler.put(fgcmStarIdCat, 'fgcmStarIds') # afwTable for observation indices obsSchema = afwTable.Schema() obsSchema.addField('obsindex', type=np.int32, doc='Index in observation table') fgcmStarIndicesCat = afwTable.BaseCatalog(obsSchema) fgcmStarIndicesCat.table.preallocate(fgcmMakeStars.obsIndexCat.size) for i in xrange(fgcmMakeStars.obsIndexCat.size): fgcmStarIndicesCat.addNew() fgcmStarIndicesCat['obsindex'][:] = fgcmMakeStars.obsIndexCat[ 'obsindex'] butler.put(fgcmStarIndicesCat, 'fgcmStarIndices') # and we're done with the stars return None
def runQuantum(self, butlerQC, inputRefs, outputRefs): handleDict = {} handleDict['camera'] = butlerQC.get(inputRefs.camera) handleDict['fgcmLookUpTable'] = butlerQC.get(inputRefs.fgcmLookUpTable) handleDict['fgcmVisitCatalog'] = butlerQC.get( inputRefs.fgcmVisitCatalog) handleDict['fgcmStandardStars'] = butlerQC.get( inputRefs.fgcmStandardStars) if self.config.doZeropointOutput: handleDict['fgcmZeropoints'] = butlerQC.get( inputRefs.fgcmZeropoints) photoCalibRefDict = { photoCalibRef.dataId.byName()['visit']: photoCalibRef for photoCalibRef in outputRefs.fgcmPhotoCalib } if self.config.doAtmosphereOutput: handleDict['fgcmAtmosphereParameters'] = butlerQC.get( inputRefs.fgcmAtmosphereParameters) atmRefDict = { atmRef.dataId.byName()['visit']: atmRef for atmRef in outputRefs.fgcmTransmissionAtmosphere } if self.config.doReferenceCalibration: refConfig = LoadReferenceObjectsConfig() self.refObjLoader = ReferenceObjectLoader( dataIds=[ref.datasetRef.dataId for ref in inputRefs.refCat], refCats=butlerQC.get(inputRefs.refCat), log=self.log, config=refConfig) else: self.refObjLoader = None struct = self.run(handleDict, self.config.physicalFilterMap) # Output the photoCalib exposure catalogs if struct.photoCalibCatalogs is not None: self.log.info("Outputting photoCalib catalogs.") for visit, expCatalog in struct.photoCalibCatalogs: butlerQC.put(expCatalog, photoCalibRefDict[visit]) self.log.info("Done outputting photoCalib catalogs.") # Output the atmospheres if struct.atmospheres is not None: self.log.info("Outputting atmosphere transmission files.") for visit, atm in struct.atmospheres: butlerQC.put(atm, atmRefDict[visit]) self.log.info("Done outputting atmosphere files.") if self.config.doReferenceCalibration: # Turn offset into simple catalog for persistence if necessary schema = afwTable.Schema() schema.addField('offset', type=np.float64, doc="Post-process calibration offset (mag)") offsetCat = afwTable.BaseCatalog(schema) offsetCat.resize(len(struct.offsets)) offsetCat['offset'][:] = struct.offsets butlerQC.put(offsetCat, outputRefs.fgcmOffsets) return
def __init__(self, *args, **kwargs): BatchPoolTask.__init__(self, *args, **kwargs) self.makeSubtask("sky") # Disposable schema suppresses warning from SourceDetectionTask.__init__ self.makeSubtask("detection", schema=afwTable.Schema())
def run(self, dataRef, selectDataList=[]): self.log.info("Processing %s" % (dataRef.dataId)) filters = self.config.filters.split("^") catalogs = dict(self.readCatalog(dataRef, f) for f in filters) coadds = dict(self.readCoadd(dataRef, f) for f in filters) wcs = coadds[filters[0]].getWcs() pixel_scale = wcs.pixelScale().asDegrees() * 3600.0 ref = catalogs[self.config.refFilterName] # create new table table mergedSchema = afwTable.Schema() fields = [] # define table fields fields.append(mergedSchema.addField("id", type="L", doc="Unique id")) fields.append(mergedSchema.addField("ra", type="F", doc="ra [deg]")) fields.append(mergedSchema.addField("dec", type="F", doc="dec [deg]")) fields.append( mergedSchema.addField( "countInputs", type="I", doc="Number of input single exposures for the reference filter" )) fields.append( mergedSchema.addField( "PSFDetRadius", type="F", doc= "Determinant radius for the PSF at the object position = sigma if gaussian [arcsec]" )) fields.append( mergedSchema.addField("EB_V", type="F", doc="Milky Way dust E(B-V) [mag]")) fields.append( mergedSchema.addField("isDuplicated", type="I", doc="1 if outside the inner tract or patch")) fields.append( mergedSchema.addField( "isEdge", type="I", doc="1 if offImage or in region masked EDGE or NO_DATA")) fields.append( mergedSchema.addField( "hasBadPhotometry", type="I", doc= "1 if interpolated, saturated, suspect, has CR at center or near bright object" )) fields.append( mergedSchema.addField("isClean", type="I", doc="1 if none of other flags is set")) if self.config.hasDepthInfo: fields.append( mergedSchema.addField( "isFullDepthColor", type="I", doc="1 if point located in full depth and color area")) # create table object merged = afwTable.BaseCatalog(mergedSchema) N = len(ref) for i in range(N): # for i in range(100,110): # create new record record = merged.addNew() coord = ref[i].get('coord') # record if any of the filter is flagged as bad photometry for f in filters: hasBadPhotometry = (catalogs[f][i].get('flags.pixel.interpolated.center')) \ | (catalogs[f][i].get('flags.pixel.saturated.center')) \ | (catalogs[f][i].get('flags.pixel.suspect.center')) \ | (catalogs[f][i].get('flags.pixel.cr.center')) \ | (catalogs[f][i].get('flags.pixel.bad')) \ | (catalogs[f][i].get('flags.pixel.bright.object.center')) if hasBadPhotometry: break isDuplicated = not ref[i].get('detect.is-primary') isEdge = (ref[i].get('flags.pixel.offimage')) | ( ref[i].get('flags.pixel.edge')) isClean = (not hasBadPhotometry) & (not isDuplicated) & ( not isEdge) # record common info from reference filter record.set(mergedSchema['id'].asKey(), ref[i].get('id')) record.set(mergedSchema['ra'].asKey(), coord.toFk5().getRa().asDegrees()) record.set(mergedSchema['dec'].asKey(), coord.toFk5().getDec().asDegrees()) record.set(mergedSchema['countInputs'].asKey(), ref[i].get('countInputs')) record.set( mergedSchema['PSFDetRadius'].asKey(), ref[i].get("shape.sdss.psf").getDeterminantRadius() * pixel_scale) record.set(mergedSchema['isDuplicated'].asKey(), int(isDuplicated)) record.set(mergedSchema['isEdge'].asKey(), int(isEdge)) record.set(mergedSchema['hasBadPhotometry'].asKey(), int(hasBadPhotometry)) record.set(mergedSchema['isClean'].asKey(), int(isClean)) if self.config.hasDepthInfo: record.set(mergedSchema['isFullDepthColor'].asKey(), int(ref[i].get('isFullDepthColor'))) # dust correction EB_V = self.getDustCorrection( self.dustMap, record.get(mergedSchema['ra'].asKey()), record.get(mergedSchema['dec'].asKey())) record.set(mergedSchema['EB_V'].asKey(), EB_V) # write catalog if self.config.fileOutName == "": # get output dir # TO DO: create new PAF # see /Users/coupon/local/source/hscPipe/install/DarwinX86/solvetansip/6.5.1p_hsc/python/hsc/meas/tansip/utils.py # and /Users/coupon/local/source/hscPipe/install/DarwinX86/pex_policy/HSC-4.0.0/tests/Policy_1.py if self.config.dirOutName == "": dirOutName = dataRef.getButler( ).mapper.root + "/" + self.config.coaddName + "Coadd-results" self.log.info( "WARNING: the output file will be written in {0:s}.". format(dirOutName)) else: dirOutName = self.config.dirOutName fileOutName = "{0}/{1}/{2}/{3}/multiRanCat-{2}-{3}.fits".format( dirOutName, "merged", dataRef.dataId["tract"], dataRef.dataId["patch"]) else: fileOutName = self.config.fileOutName self.log.info("Writing {0:s}".format(fileOutName)) self.mkdir_p(os.path.dirname(fileOutName)) merged.writeFits(fileOutName) # write catalog #self.log.info("Writing {0:s}".format(self.config.fileOutName)) #if self.config.fileOutName == "": # fileOutName = "{0}/{1}/{2}/{3}/MultiRanCat-{2}-{3}.fits".format(self.config.dirOutName,"merged",dataRef.dataId["tract"],dataRef.dataId["patch"]) # self.mkdir_p(os.path.dirname(fileOutName)) #else: # fileOutName = self.config.fileOutName #merged.writeFits(fileOutName) return
def setUp(self): self.schema = afwTable.Schema() self.schema.addField('test1', type='ArrayF', size=430000025) self.schema.addField('test2', type='ArrayF', size=430000025) self.cat = afwTable.BaseCatalog(self.schema)
def setUp(self): """Create a catalog to run through the ingestion process.""" # First, connect to the database. self.host = os.environ.get("TEST_MYSQL_HOST", "lsst-db.ncsa.illinois.edu") self.db = os.environ.get("TEST_MYSQL_DB", "test") self.port = int(os.environ.get("TEST_MYSQL_PORT", "3306")) self.conn = None try: self.conn = IngestCatalogTask.connect( host=self.host, port=self.port, db=self.db) except: pass # Create a schema containing one of every kind of afw field schema = afw_table.Schema() keys = ( # Scalar fields schema.addField("scalar.u", type="U"), schema.addField("scalar.i", type="I"), schema.addField("scalar.l", type="L"), schema.addField("scalar.f", type="F"), schema.addField("scalar.d", type="D"), schema.addField("scalar.flag", type="Flag"), schema.addField("scalar.angle", type="Angle"), # Fixed-length array types schema.addField("fix.string", type="String", size=4), schema.addField("fix.array.u", type="ArrayU", size=2), schema.addField("fix.array.i", type="ArrayI", size=2), schema.addField("fix.array.f", type="ArrayF", size=2), schema.addField("fix.array.d", type="ArrayD", size=2), # Variable-length array types schema.addField("var.array.u", type="ArrayU", size=0), schema.addField("var.array.i", type="ArrayI", size=0), schema.addField("var.array.f", type="ArrayF", size=0), schema.addField("var.array.d", type="ArrayD", size=0), ) # Setup schema aliases aliases = dict( S="scalar", F="fix", af="fix.array", V="var", av="var.array", vla="av", ) for source, target in aliases.iteritems(): schema.getAliasMap().set(source, target) # Create two rows of fake data... self.rows = ( ( 0, -2147483648, -9223372036854775808, 1.0, math.pi, False, Angle(1.0), "ab ", np.array([0, 65535], dtype=np.uint16), np.array([-2147483648, 2147483647], dtype=np.int32), np.array([1.0, 2.0], dtype=np.float32), np.array([math.pi, math.e], dtype=np.float64), np.array(range(0), dtype=np.uint16), np.array(range(1), dtype=np.int32), np.array(range(3), dtype=np.float32), np.array(range(4), dtype=np.float64), ), ( 65535, 2147483647, 9223372036854775807, 2.0, math.e, True, Angle(2.0), "", np.array(range(2), dtype=np.uint16), np.array(range(2), dtype=np.int32), np.array(range(2), dtype=np.float32), np.array(range(2), dtype=np.float64), np.array([], dtype=np.uint16), np.array([], dtype=np.int32), np.array([], dtype=np.float32), np.array([], dtype=np.float64), ) ) # and a corresponding catalog self.catalog = afw_table.BaseCatalog(schema) for row in self.rows: record = self.catalog.addNew() for i, k in enumerate(keys): record.set(k, row[i]) # Finally, choose table/view names that are unique with very high # probability. suffix = uuid.uuid4().hex self.table_name = "catalog_" + suffix self.view_name = "view_" + suffix
def _fgcmMakeVisitCatalog(self, butler, dataRefs): """ Make a visit catalog with all the key data from each visit Parameters ---------- butler: lsst.daf.persistence.Butler dataRefs: list of lsst.daf.persistence.ButlerDataRef Data references for the input visits If this is an empty list, all visits with src catalogs in the repository are used. Only one individual dataRef from a visit need be specified and the code will find the other source catalogs from each visit Returns ------- visitCat: afw.table.BaseCatalog """ startTime = time.time() camera = butler.get('camera') nCcd = len(camera) if len(dataRefs) == 0: # We did not specify any datarefs, so find all of them if not self.config.checkAllCcds: # Faster mode, scan through referenceCCD allVisits = butler.queryMetadata( 'src', format=[self.config.visitDataRefName, 'filter'], dataId={ self.config.ccdDataRefName: self.config.referenceCCD }) srcVisits = [] srcCcds = [] for dataset in allVisits: if (butler.datasetExists('src', dataId={ self.config.visitDataRefName: dataset[0], self.config.ccdDataRefName: self.config.referenceCCD })): srcVisits.append(dataset[0]) srcCcds.append(self.config.referenceCCD) else: # Slower mode, check all CCDs allVisits = butler.queryMetadata( 'src', format=[self.config.visitDataRefName, 'filter']) srcVisits = [] srcCcds = [] for dataset in allVisits: if dataset[0] in srcVisits: continue for ccd in xrange(nCcd): if (butler.datasetExists( 'src', dataId={ self.config.visitDataRefName: dataset[0], self.config.ccdDataRefName: ccd })): srcVisits.append(dataset[0]) srcCcds.append(ccd) # Once we find that a butler dataset exists, break out break else: # get the visits from the datarefs, only for referenceCCD srcVisits = [ d.dataId[self.config.visitDataRefName] for d in dataRefs if d.dataId[self.config.ccdDataRefName] == self.config.referenceCCD ] srcCcds = [self.config.referenceCCD] * len(srcVisits) # Sort the visits for searching/indexing srcVisits.sort() self.log.info("Found %d visits in %.2f s" % (len(srcVisits), time.time() - startTime)) schema = afwTable.Schema() schema.addField('visit', type=np.int32, doc="Visit number") schema.addField('filtername', type=str, size=2, doc="Filter name") schema.addField('telra', type=np.float64, doc="Pointing RA (deg)") schema.addField('teldec', type=np.float64, doc="Pointing Dec (deg)") schema.addField('telha', type=np.float64, doc="Pointing Hour Angle (deg)") schema.addField('mjd', type=np.float64, doc="MJD of visit") schema.addField('exptime', type=np.float32, doc="Exposure time") schema.addField('pmb', type=np.float32, doc="Pressure (millibar)") schema.addField('psfsigma', type=np.float32, doc="PSF sigma (reference CCD)") schema.addField('deltaaper', type=np.float32, doc="Delta-aperture") schema.addField('skybackground', type=np.float32, doc="Sky background (ADU) (reference CCD)") # the following field is not used yet schema.addField('deepflag', type=np.int32, doc="Deep observation") schema.addField('scaling', type='ArrayD', doc="Scaling applied due to flat adjustment", size=len(camera)) visitCat = afwTable.BaseCatalog(schema) visitCat.table.preallocate(len(srcVisits)) startTime = time.time() # reading in a small bbox is faster for non-gzipped images bbox = afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.PointI(1, 1)) # now loop over visits and get the information for i, srcVisit in enumerate(srcVisits): # We don't use the bypasses since we need the psf info which does # not have a bypass dataId = { self.config.visitDataRefName: srcVisit, self.config.ccdDataRefName: srcCcds[i] } exp = butler.get('calexp_sub', dataId=dataId, bbox=bbox, flags=afwTable.SOURCE_IO_NO_FOOTPRINTS) visitInfo = exp.getInfo().getVisitInfo() f = exp.getFilter() psf = exp.getPsf() rec = visitCat.addNew() rec['visit'] = srcVisit rec['filtername'] = f.getName() radec = visitInfo.getBoresightRaDec() rec['telra'] = radec.getRa().asDegrees() rec['teldec'] = radec.getDec().asDegrees() rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees() rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD) rec['exptime'] = visitInfo.getExposureTime() # convert from Pa to millibar # Note that I don't know if this unit will need to be per-camera config rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100 rec['deepflag'] = 0 rec['scaling'][:] = 1.0 rec['deltaaper'] = 0.0 rec['psfsigma'] = psf.computeShape().getDeterminantRadius() if butler.datasetExists('calexpBackground', dataId=dataId): # Get background for reference CCD # This approximation is good enough for now bgStats = ( bg[0].getStatsImage().getImage().array for bg in butler.get('calexpBackground', dataId=dataId)) rec['skybackground'] = sum( np.median(bg[np.isfinite(bg)]) for bg in bgStats) else: rec['skybackground'] = -1.0 # Compute flat scaling if desired... if self.config.renormalizeFlats: self.log.info("Reading flats for renormalizeFlats") scalingValues = self._computeFlatScaling(butler, visitCat) visitCat['scaling'] *= scalingValues self.log.info("Found all VisitInfo in %.2f s" % (time.time() - startTime)) # and now persist it butler.put(visitCat, 'fgcmVisitCatalog') return visitCat
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # Disposable schema suppresses warning from SourceDetectionTask.__init__ self.makeSubtask("detection", schema=afwTable.Schema()) self.makeSubtask("interpolate") self.makeSubtask("subtractBackground")
def __init__(self, butler=None, schema=None, refObjLoader=None, reuse=tuple(), **kwargs): """! @param[in] butler: the butler can be used to retrieve schema or passed to the refObjLoader constructor in case it is needed. @param[in] schema: the schema of the source detection catalog used as input. @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference catalog. May be None if the butler argument is provided or all steps requiring a reference catalog are disabled. """ BatchPoolTask.__init__(self, **kwargs) if schema is None: assert butler is not None, "Butler not provided" schema = butler.get(self.config.coaddName + "Coadd_det_schema", immediate=True).schema self.butler = butler self.reuse = tuple(reuse) self.makeSubtask("detectCoaddSources") self.makeSubtask("mergeCoaddDetections", schema=schema) if self.config.measureCoaddSources.inputCatalog.startswith( "deblended"): # Ensure that the output from deblendCoaddSources matches the input to measureCoaddSources self.measurementInput = self.config.measureCoaddSources.inputCatalog self.deblenderOutput = [] if self.config.deblendCoaddSources.simultaneous: self.deblenderOutput.append("deblendedModel") else: self.deblenderOutput.append("deblendedFlux") if self.measurementInput not in self.deblenderOutput: err = "Measurement input '{0}' is not in the list of deblender output catalogs '{1}'" raise ValueError( err.format(self.measurementInput, self.deblenderOutput)) self.makeSubtask( "deblendCoaddSources", schema=afwTable.Schema(self.mergeCoaddDetections.schema), peakSchema=afwTable.Schema( self.mergeCoaddDetections.merged.getPeakSchema()), butler=butler) measureInputSchema = afwTable.Schema( self.deblendCoaddSources.schema) else: measureInputSchema = afwTable.Schema( self.mergeCoaddDetections.schema) self.makeSubtask("measureCoaddSources", schema=measureInputSchema, peakSchema=afwTable.Schema( self.mergeCoaddDetections.merged.getPeakSchema()), refObjLoader=refObjLoader, butler=butler) self.makeSubtask("mergeCoaddMeasurements", schema=afwTable.Schema( self.measureCoaddSources.schema)) self.makeSubtask("forcedPhotCoadd", refSchema=afwTable.Schema( self.mergeCoaddMeasurements.schema)) if self.config.hasFakes: self.coaddType = "fakes_" + self.config.coaddName else: self.coaddType = self.config.coaddName
def makeZptSchema(superStarChebyshevSize, zptChebyshevSize): """ Make the zeropoint schema Parameters ---------- superStarChebyshevSize: `int` Length of the superstar chebyshev array zptChebyshevSize: `int` Length of the zeropoint chebyshev array Returns ------- zptSchema: `lsst.afw.table.schema` """ zptSchema = afwTable.Schema() zptSchema.addField('visit', type=np.int32, doc='Visit number') zptSchema.addField('ccd', type=np.int32, doc='CCD number') zptSchema.addField('fgcmFlag', type=np.int32, doc=('FGCM flag value: ' '1: Photometric, used in fit; ' '2: Photometric, not used in fit; ' '4: Non-photometric, on partly photometric night; ' '8: Non-photometric, on non-photometric night; ' '16: No zeropoint could be determined; ' '32: Too few stars for reliable gray computation')) zptSchema.addField('fgcmZpt', type=np.float64, doc='FGCM zeropoint (center of CCD)') zptSchema.addField('fgcmZptErr', type=np.float64, doc='Error on zeropoint, estimated from repeatability + number of obs') zptSchema.addField('fgcmfZptChebXyMax', type='ArrayD', size=2, doc='maximum x/maximum y to scale to apply chebyshev parameters') zptSchema.addField('fgcmfZptCheb', type='ArrayD', size=zptChebyshevSize, doc='Chebyshev parameters (flattened) for zeropoint') zptSchema.addField('fgcmfZptSstarCheb', type='ArrayD', size=superStarChebyshevSize, doc='Chebyshev parameters (flattened) for superStarFlat') zptSchema.addField('fgcmI0', type=np.float64, doc='Integral of the passband') zptSchema.addField('fgcmI10', type=np.float64, doc='Normalized chromatic integral') zptSchema.addField('fgcmR0', type=np.float64, doc='Retrieved i0 integral, estimated from stars (only for flag 1)') zptSchema.addField('fgcmR10', type=np.float64, doc='Retrieved i10 integral, estimated from stars (only for flag 1)') zptSchema.addField('fgcmGry', type=np.float64, doc='Estimated gray extinction relative to atmospheric solution; ' 'only for fgcmFlag <= 4 (see fgcmFlag) ') zptSchema.addField('fgcmDeltaChrom', type=np.float64, doc='Mean chromatic correction for stars in this ccd; ' 'only for fgcmFlag <= 4 (see fgcmFlag)') zptSchema.addField('fgcmZptVar', type=np.float64, doc='Variance of zeropoint over ccd') zptSchema.addField('fgcmTilings', type=np.float64, doc='Number of photometric tilings used for solution for ccd') zptSchema.addField('fgcmFpGry', type=np.float64, doc='Average gray extinction over the full focal plane ' '(same for all ccds in a visit)') zptSchema.addField('fgcmFpGryBlue', type=np.float64, doc='Average gray extinction over the full focal plane ' 'for 25% bluest stars') zptSchema.addField('fgcmFpGryBlueErr', type=np.float64, doc='Error on Average gray extinction over the full focal plane ' 'for 25% bluest stars') zptSchema.addField('fgcmFpGryRed', type=np.float64, doc='Average gray extinction over the full focal plane ' 'for 25% reddest stars') zptSchema.addField('fgcmFpGryRedErr', type=np.float64, doc='Error on Average gray extinction over the full focal plane ' 'for 25% reddest stars') zptSchema.addField('fgcmFpVar', type=np.float64, doc='Variance of gray extinction over the full focal plane ' '(same for all ccds in a visit)') zptSchema.addField('fgcmDust', type=np.float64, doc='Gray dust extinction from the primary/corrector' 'at the time of the exposure') zptSchema.addField('fgcmFlat', type=np.float64, doc='Superstarflat illumination correction') zptSchema.addField('fgcmAperCorr', type=np.float64, doc='Aperture correction estimated by fgcm') zptSchema.addField('fgcmDeltaMagBkg', type=np.float64, doc=('Local background correction from brightest percentile ' '(value set by deltaMagBkgOffsetPercentile) calibration ' 'stars.')) zptSchema.addField('exptime', type=np.float32, doc='Exposure time') zptSchema.addField('filtername', type=str, size=10, doc='Filter name') return zptSchema
def _fgcmMakeLut(self, butler): """ Make a FGCM Look-up Table Parameters ---------- butler: lsst.daf.persistence.Butler (used for mapper information) """ # need the camera for the detectors camera = butler.get('camera') # number of ccds from the length of the camera iterator nCcd = len(camera) self.log.info("Found %d ccds for look-up table" % (nCcd)) # Load in optics, etc. self._loadThroughputs(butler) # create the stub of the lutConfig lutConfig = {} lutConfig['logger'] = self.log lutConfig['filterNames'] = self.config.filterNames lutConfig['stdFilterNames'] = self.config.stdFilterNames lutConfig['nCCD'] = nCcd # atmosphereTable already validated if available if self.config.atmosphereTableName is not None: lutConfig['atmosphereTableName'] = self.config.atmosphereTableName else: # use the regular paramters (also validated if needed) lutConfig['elevation'] = self.config.parameters.elevation lutConfig['pmbRange'] = self.config.parameters.pmbRange lutConfig['pmbSteps'] = self.config.parameters.pmbSteps lutConfig['pwvRange'] = self.config.parameters.pwvRange lutConfig['pwvSteps'] = self.config.parameters.pwvSteps lutConfig['o3Range'] = self.config.parameters.o3Range lutConfig['o3Steps'] = self.config.parameters.o3Steps lutConfig['tauRange'] = self.config.parameters.tauRange lutConfig['tauSteps'] = self.config.parameters.tauSteps lutConfig['alphaRange'] = self.config.parameters.alphaRange lutConfig['alphaSteps'] = self.config.parameters.alphaSteps lutConfig['zenithRange'] = self.config.parameters.zenithRange lutConfig['zenithSteps'] = self.config.parameters.zenithSteps lutConfig['pmbStd'] = self.config.parameters.pmbStd lutConfig['pwvStd'] = self.config.parameters.pwvStd lutConfig['o3Std'] = self.config.parameters.o3Std lutConfig['tauStd'] = self.config.parameters.tauStd lutConfig['alphaStd'] = self.config.parameters.alphaStd lutConfig['airmassStd'] = self.config.parameters.airmassStd lutConfig['lambdaRange'] = self.config.parameters.lambdaRange lutConfig['lambdaStep'] = self.config.parameters.lambdaStep lutConfig['lambdaNorm'] = self.config.parameters.lambdaNorm # make the lut object self.log.info("Making the LUT maker object") self.fgcmLutMaker = fgcm.FgcmLUTMaker(lutConfig) # generate the throughput dictionary. Fun! # do this internally here at first. Later, break it into its own thing # these will be in Angstroms # note that lambdaStep is currently in nm, because dumb. convert to A throughputLambda = np.arange( self.fgcmLutMaker.lambdaRange[0], self.fgcmLutMaker.lambdaRange[1] + self.fgcmLutMaker.lambdaStep * 10, self.fgcmLutMaker.lambdaStep * 10.) self.log.info("Built throughput lambda, %.1f-%.1f, step %.2f" % (throughputLambda[0], throughputLambda[-1], throughputLambda[1] - throughputLambda[0])) throughputDict = {} for i, filterName in enumerate(self.config.filterNames): tDict = {} tDict['LAMBDA'] = throughputLambda for ccdIndex, detector in enumerate(camera): tDict[ccdIndex] = self._getThroughputDetector( detector, filterName, throughputLambda) throughputDict[filterName] = tDict # set the throughputs self.fgcmLutMaker.setThroughputs(throughputDict) # make the LUT self.log.info("Making LUT") self.fgcmLutMaker.makeLUT() # and save the LUT lutSchema = afwTable.Schema() # new version, which gets around the afwTable row length limitation # each LUT will be saved in a different row # there is overhead of the arrays that we only need one copy, but this # is going to be insignificant overall # build the index values comma = ',' filterNameString = comma.join(self.config.filterNames) stdFilterNameString = comma.join(self.config.stdFilterNames) atmosphereTableName = 'NoTableWasUsed' if self.config.atmosphereTableName is not None: atmosphereTableName = self.config.atmosphereTableName lutSchema.addField('tablename', type=str, doc='Atmosphere table name', size=len(atmosphereTableName)) lutSchema.addField('elevation', type=float, doc="Telescope elevation used for LUT") lutSchema.addField('filternames', type=str, doc='filterNames in LUT', size=len(filterNameString)) lutSchema.addField('stdfilternames', type=str, doc='Standard filterNames in LUT', size=len(stdFilterNameString)) lutSchema.addField('pmb', type='ArrayD', doc='Barometric Pressure', size=self.fgcmLutMaker.pmb.size) lutSchema.addField('pmbfactor', type='ArrayD', doc='PMB scaling factor', size=self.fgcmLutMaker.pmb.size) lutSchema.addField('pmbelevation', type=np.float64, doc='PMB Scaling at elevation') lutSchema.addField('pwv', type='ArrayD', doc='Preciptable Water Vapor', size=self.fgcmLutMaker.pwv.size) lutSchema.addField('o3', type='ArrayD', doc='Ozone', size=self.fgcmLutMaker.o3.size) lutSchema.addField('tau', type='ArrayD', doc='Aerosol optical depth', size=self.fgcmLutMaker.tau.size) lutSchema.addField('lambdanorm', type=np.float64, doc='AOD wavelength') lutSchema.addField('alpha', type='ArrayD', doc='Aerosol alpha', size=self.fgcmLutMaker.alpha.size) lutSchema.addField('zenith', type='ArrayD', doc='Zenith angle', size=self.fgcmLutMaker.zenith.size) lutSchema.addField('nccd', type=np.int32, doc='Number of CCDs') # and the standard values lutSchema.addField('pmbstd', type=np.float64, doc='PMB Standard') lutSchema.addField('pwvstd', type=np.float64, doc='PWV Standard') lutSchema.addField('o3std', type=np.float64, doc='O3 Standard') lutSchema.addField('taustd', type=np.float64, doc='Tau Standard') lutSchema.addField('alphastd', type=np.float64, doc='Alpha Standard') lutSchema.addField('zenithstd', type=np.float64, doc='Zenith angle Standard') lutSchema.addField('lambdarange', type='ArrayD', doc='Wavelength range', size=2) lutSchema.addField('lambdastep', type=np.float64, doc='Wavelength step') lutSchema.addField('lambdastd', type='ArrayD', doc='Standard Wavelength', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('lambdastdfilter', type='ArrayD', doc='Standard Wavelength (raw)', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i0std', type='ArrayD', doc='I0 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i1std', type='ArrayD', doc='I1 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('i10std', type='ArrayD', doc='I10 Standard', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('lambdab', type='ArrayD', doc='Wavelength for passband (no atm)', size=len(self.fgcmLutMaker.filterNames)) lutSchema.addField('atmlambda', type='ArrayD', doc='Atmosphere wavelengths (A)', size=self.fgcmLutMaker.atmLambda.size) lutSchema.addField('atmstdtrans', type='ArrayD', doc='Standard Atmosphere Throughput', size=self.fgcmLutMaker.atmStdTrans.size) # and the look-up-tables lutSchema.addField('luttype', type=str, size=20, doc='Look-up table type') lutSchema.addField('lut', type='ArrayF', doc='Look-up table for luttype', size=self.fgcmLutMaker.lut['I0'].size) lutCat = afwTable.BaseCatalog(lutSchema) lutCat.table.preallocate(14) # first fill the first index rec = lutCat.addNew() rec['tablename'] = atmosphereTableName rec['elevation'] = self.fgcmLutMaker.atmosphereTable.elevation rec['filternames'] = filterNameString rec['stdfilternames'] = stdFilterNameString rec['pmb'][:] = self.fgcmLutMaker.pmb rec['pmbfactor'][:] = self.fgcmLutMaker.pmbFactor rec['pmbelevation'] = self.fgcmLutMaker.pmbElevation rec['pwv'][:] = self.fgcmLutMaker.pwv rec['o3'][:] = self.fgcmLutMaker.o3 rec['tau'][:] = self.fgcmLutMaker.tau rec['lambdanorm'] = self.fgcmLutMaker.lambdaNorm rec['alpha'][:] = self.fgcmLutMaker.alpha rec['zenith'][:] = self.fgcmLutMaker.zenith rec['nccd'] = self.fgcmLutMaker.nCCD rec['pmbstd'] = self.fgcmLutMaker.pmbStd rec['pwvstd'] = self.fgcmLutMaker.pwvStd rec['o3std'] = self.fgcmLutMaker.o3Std rec['taustd'] = self.fgcmLutMaker.tauStd rec['alphastd'] = self.fgcmLutMaker.alphaStd rec['zenithstd'] = self.fgcmLutMaker.zenithStd rec['lambdarange'][:] = self.fgcmLutMaker.lambdaRange rec['lambdastep'] = self.fgcmLutMaker.lambdaStep rec['lambdastd'][:] = self.fgcmLutMaker.lambdaStd rec['lambdastdfilter'][:] = self.fgcmLutMaker.lambdaStdFilter rec['i0std'][:] = self.fgcmLutMaker.I0Std rec['i1std'][:] = self.fgcmLutMaker.I1Std rec['i10std'][:] = self.fgcmLutMaker.I10Std rec['lambdab'][:] = self.fgcmLutMaker.lambdaB rec['atmlambda'][:] = self.fgcmLutMaker.atmLambda rec['atmstdtrans'][:] = self.fgcmLutMaker.atmStdTrans rec['luttype'] = 'I0' rec['lut'][:] = self.fgcmLutMaker.lut['I0'].flatten() # and add the rest rec = lutCat.addNew() rec['luttype'] = 'I1' rec['lut'][:] = self.fgcmLutMaker.lut['I1'].flatten() derivTypes = [ 'D_PMB', 'D_PWV', 'D_O3', 'D_LNTAU', 'D_ALPHA', 'D_SECZENITH', 'D_PMB_I1', 'D_PWV_I1', 'D_O3_I1', 'D_LNTAU_I1', 'D_ALPHA_I1', 'D_SECZENITH_I1' ] for derivType in derivTypes: rec = lutCat.addNew() rec['luttype'] = derivType rec['lut'][:] = self.fgcmLutMaker.lutDeriv[derivType].flatten() butler.put(lutCat, 'fgcmLookUpTable')