class ParseConfig(Config): """Configuration for ParseTask""" translation = DictField(keytype=str, itemtype=str, default={}, doc="Translation table for property --> header") translators = DictField(keytype=str, itemtype=str, default={}, doc="Properties and name of translator method") defaults = DictField(keytype=str, itemtype=str, default={}, doc="Default values if header is not present") hdu = Field(dtype=int, default=0, doc="HDU to read for metadata") extnames = ListField(dtype=str, default=[], doc="Extension names to search for")
class RegisterConfig(Config): """Configuration for the RegisterTask""" table = Field(dtype=str, default="raw", doc="Name of table") columns = DictField( keytype=str, itemtype=str, doc="List of columns for raw table, with their types", itemCheck=lambda x: x in ("text", "int", "double"), default={ 'object': 'text', 'visit': 'int', 'ccd': 'int', 'filter': 'text', 'date': 'text', 'taiObs': 'text', 'expTime': 'double', }, ) unique = ListField( dtype=str, doc="List of columns to be declared unique for the table", default=["visit", "ccd"]) visit = ListField(dtype=str, default=["visit", "object", "date", "filter"], doc="List of columns for raw_visit table") ignore = Field(dtype=bool, default=False, doc="Ignore duplicates in the table?") permissions = Field(dtype=int, default=0o664, doc="Permissions mode for registry; 0o664 = rw-rw-r--")
class CalibrateCatalogConfig(Config): doApplyCalib = Field(dtype=bool, default=True, doc="Calibrate fluxes to magnitudes?") srcSchemaMap = DictField( doc="Mapping between different stack (e.g. HSC vs. LSST) schema names", keytype=str, itemtype=str, default=None, optional=True)
class PropagateVisitFlagsConfig(Config): """!Configuration for propagating flags to coadd""" flags = DictField(keytype=str, itemtype=float, default={"calib_psf_candidate": 0.2, "calib_psf_used": 0.2, "calib_psf_reserved": 0.2, "calib_astrometry_used": 0.2, "calib_photometry_used": 0.2, "calib_photometry_reserved": 0.2, }, doc=("Source catalog flags to propagate, with the threshold of relative occurrence " "(valid range: [0-1], default is 0.2). Coadd object will have flag set if the " "fraction of input visits in which it is flagged is greater than the threshold.")) matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)") ccdName = Field(dtype=str, default='ccd', doc="Name of ccd to give to butler")
class CombineDiaForcedConfig(Config): coaddName = Field(dtype=str, default="deep", doc="Name of coadd") ccdKey = Field(dtype=str, default='detector', doc="Name of ccd to give to butler") keepFields = DictField( keytype=str, itemtype=str, doc= 'Keep these fields from the diaSrc catalogs. Will be averaged over matches', default={ 'psf_flux': 'base_PsfFlux_instFlux', 'psf_flux_err': 'base_PsfFlux_instFluxErr' }) storage = Field(dtype=str, default="pickle", doc="pandas storage format")
class PropagateVisitFlagsConfig(Config): """!Configuration for propagating flags to coadd""" flags = DictField( keytype=str, itemtype=float, default={ "calib_psfCandidate": 0.2, "calib_psfUsed": 0.2, }, doc= "Source catalog flags to propagate, with the threshold of relative occurrence." ) matchRadius = Field(dtype=float, default=0.2, doc="Source matching radius (arcsec)")
class ProcessFocusConfig(Config): focus = ConfigField(dtype=FocusConfig, doc="Focus determination") zemax = DictField(keytype=str, itemtype=str, default={}, doc="Mapping from filter name to zemax configuration filename") isr = ConfigurableField(target=SubaruIsrTask, doc="Instrument Signature Removal") installPsf = ConfigurableField(target=InstallGaussianPsfTask, doc="Install a simple PSF model") background = ConfigField(dtype=measAlg.estimateBackground.ConfigClass, doc="Background removal") detectAndMeasure = ConfigurableField(target=DetectAndMeasureTask, doc="Source detection and measurement") starSelector = ConfigurableField(target=measAlg.ObjectSizeStarSelectorTask, doc="Star selection algorithm") doWrite = Field(dtype=bool, default=True, doc="Write processed image?") def setDefaults(self): """These defaults are suitable for HSC, but may be useful for other cameras if the focus code is employed elsewhere. """ Config.setDefaults(self) zemaxBase = os.path.join(os.environ["OBS_SUBARU_DIR"], "hsc", "zemax_config%d_0.0.dat") self.zemax = dict([(f, zemaxBase % n) for f,n in [ ('g', 9), ('r', 1), ('i', 3), ('z', 5), ('y', 7), ('N921', 5), ('N816', 3), ('N1010', 7), ('N387', 9), ('N515', 9), ]]) self.load(os.path.join(os.environ["OBS_SUBARU_DIR"], "config", "hsc", "isr.py")) self.installPsf.fwhm = 9 # pixels self.installPsf.width = 31 # pixels self.detectAndMeasure.detection.includeThresholdMultiplier = 3.0 self.detectAndMeasure.measurement.algorithms.names.add("base_GaussianCentroid") self.detectAndMeasure.measurement.slots.centroid = "base_GaussianCentroid" # set up simple shape, if available (because focus calibrations are for that) # If it's not available, we'll crash later; but we don't want to crash here (brings everything down)! if haveSimpleShape: self.detectAndMeasure.measurement.algorithms.names.add("ext_simpleShape_SimpleShape") self.detectAndMeasure.measurement.algorithms["ext_simpleShape_SimpleShape"].sigma = 5.0 # pixels # set up background estimate self.background.ignoredPixelMask = ['EDGE', 'NO_DATA', 'DETECTED', 'DETECTED_NEGATIVE', 'BAD'] self.detectAndMeasure.detection.background.algorithm='LINEAR' self.detectAndMeasure.doDeblend = False self.starSelector.badFlags = ["base_PixelFlags_flag_edge", "base_PixelFlags_flag_interpolatedCenter", "base_PixelFlags_flag_saturatedCenter", "base_PixelFlags_flag_bad", ] self.starSelector.sourceFluxField = "base_GaussianFlux_flux" self.starSelector.widthMax = 20.0 self.starSelector.widthStdAllowed = 5.0
class ConvertRepoConfig(Config): raws = ConfigurableField( "Configuration for subtask responsible for ingesting raws and adding " "visit and exposure dimension entries.", target=RawIngestTask, ) skyMaps = ConfigDictField( "Mapping from Gen3 skymap name to the parameters used to construct a " "BaseSkyMap instance. This will be used to associate names with " "existing skymaps found in the Gen2 repo.", keytype=str, itemtype=ConvertRepoSkyMapConfig, default={}) rootSkyMapName = Field( "Name of a Gen3 skymap (an entry in ``self.skyMaps``) to assume for " "datasets in the root repository when no SkyMap is found there. ", dtype=str, optional=True, default=None, ) collections = DictField( "Special collections (values) for certain dataset types (keys). " "These are used in addition to rerun collections for datasets in " "reruns. The 'raw' dataset must have an entry here if it is to be " "converted.", keytype=str, itemtype=str, default={ "deepCoadd_skyMap": "skymaps", "brightObjectMask": "masks", }) storageClasses = DictField( "Mapping from dataset type name or Gen2 policy entry (e.g. 'python' " "or 'persistable') to the Gen3 StorageClass name.", keytype=str, itemtype=str, default={ "BaseSkyMap": "SkyMap", "BaseCatalog": "Catalog", "BackgroundList": "Background", "raw": "Exposure", "MultilevelParquetTable": "DataFrame", "ParquetTable": "DataFrame", "SkyWcs": "Wcs", }) doRegisterInstrument = Field( "If True (default), add dimension records for the Instrument and its " "filters and detectors to the registry instead of assuming they are " "already present.", dtype=bool, default=True, ) doWriteCuratedCalibrations = Field( "If True (default), ingest human-curated calibrations directly via " "the Instrument interface. Note that these calibrations are never " "converted from Gen2 repositories.", dtype=bool, default=True, ) refCats = ListField( "The names of reference catalogs (subdirectories under ref_cats) to " "be converted", dtype=str, default=[]) fileIgnorePatterns = ListField( "Filename globs that should be ignored instead of being treated as " "datasets.", dtype=str, default=[ "README.txt", "*~?", "butler.yaml", "gen3.sqlite3", "registry.sqlite3", "calibRegistry.sqlite3", "_mapper", "_parent", "repositoryCfg.yaml" ]) datasetIncludePatterns = ListField( "Glob-style patterns for dataset type names that should be converted.", dtype=str, default=["*"]) datasetIgnorePatterns = ListField( "Glob-style patterns for dataset type names that should not be " "converted despite matching a pattern in datasetIncludePatterns.", dtype=str, default=[]) ccdKey = Field( "Key used for the Gen2 equivalent of 'detector' in data IDs.", dtype=str, default="ccd", ) relatedOnly = Field( "If True (default), only convert datasets that are related to the " "ingested visits. Ignored unless a list of visits is passed to " "run().", dtype=bool, default=False, ) @property def transfer(self): return self.raws.transfer @transfer.setter def transfer(self, value): self.raws.transfer = value @property def instrument(self): return self.raws.instrument @instrument.setter def instrument(self, value): self.raws.instrument = value def setDefaults(self): self.transfer = None
class ConvertRepoConfig(Config): raws = ConfigurableField( "Configuration for subtask responsible for ingesting raws and adding " "exposure dimension entries.", target=RawIngestTask, ) defineVisits = ConfigurableField( "Configuration for the subtask responsible for defining visits from " "exposures.", target=DefineVisitsTask, ) skyMaps = ConfigDictField( "Mapping from Gen3 skymap name to the parameters used to construct a " "BaseSkyMap instance. This will be used to associate names with " "existing skymaps found in the Gen2 repo.", keytype=str, itemtype=ConvertRepoSkyMapConfig, default={} ) rootSkyMapName = Field( "Name of a Gen3 skymap (an entry in ``self.skyMaps``) to assume for " "datasets in the root repository when no SkyMap is found there. ", dtype=str, optional=True, default=None, ) runs = DictField( "A mapping from dataset type name to the RUN collection they should " "be inserted into. This must include all datasets that can be found " "in the root repository; other repositories will use per-repository " "runs.", keytype=str, itemtype=str, default={}, ) runsForced = DictField( "Like ``runs``, but is used even when the dataset is present in a " "non-root repository (i.e. rerun), overriding the non-root " "repository's main collection.", keytype=str, itemtype=str, default={ "brightObjectMask": "masks", } ) storageClasses = DictField( "Mapping from dataset type name or Gen2 policy entry (e.g. 'python' " "or 'persistable') to the Gen3 StorageClass name.", keytype=str, itemtype=str, default={ "bias": "ExposureF", "dark": "ExposureF", "flat": "ExposureF", "defects": "Defects", "crosstalk": "CrosstalkCalib", "BaseSkyMap": "SkyMap", "BaseCatalog": "Catalog", "BackgroundList": "Background", "raw": "Exposure", "MultilevelParquetTable": "DataFrame", "ParquetTable": "DataFrame", "SkyWcs": "Wcs", } ) formatterClasses = DictField( "Mapping from dataset type name to formatter class. " "By default these are derived from the formatters listed in the" " Gen3 datastore configuration.", keytype=str, itemtype=str, default={} ) targetHandlerClasses = DictField( "Mapping from dataset type name to target handler class.", keytype=str, itemtype=str, default={} ) doRegisterInstrument = Field( "If True (default), add dimension records for the Instrument and its " "filters and detectors to the registry instead of assuming they are " "already present.", dtype=bool, default=True, ) refCats = ListField( "The names of reference catalogs (subdirectories under ref_cats) to " "be converted", dtype=str, default=[] ) fileIgnorePatterns = ListField( "Filename globs that should be ignored instead of being treated as " "datasets.", dtype=str, default=["README.txt", "*~?", "butler.yaml", "gen3.sqlite3", "registry.sqlite3", "calibRegistry.sqlite3", "_mapper", "_parent", "repositoryCfg.yaml"] ) rawDatasetType = Field( "Gen2 dataset type to use for raw data.", dtype=str, default="raw", ) datasetIncludePatterns = ListField( "Glob-style patterns for dataset type names that should be converted.", dtype=str, default=["*"] ) datasetIgnorePatterns = ListField( "Glob-style patterns for dataset type names that should not be " "converted despite matching a pattern in datasetIncludePatterns.", dtype=str, default=[] ) ccdKey = Field( "Key used for the Gen2 equivalent of 'detector' in data IDs.", dtype=str, default="ccd", ) relatedOnly = Field( "If True (default), only convert datasets that are related to the " "ingested visits. Ignored unless a list of visits is passed to " "run().", dtype=bool, default=False, ) doMakeUmbrellaCollection = Field( "If True (default), define an '<instrument>/defaults' CHAINED " "collection that includes everything found in the root repo as well " "as the default calibration collection.", dtype=bool, default=True, ) extraUmbrellaChildren = ListField( "Additional child collections to include in the umbrella collection. " "Ignored if doMakeUmbrellaCollection=False.", dtype=str, default=[] ) @property def transfer(self): return self.raws.transfer @transfer.setter def transfer(self, value): self.raws.transfer = value def setDefaults(self): self.transfer = None