class SedtermDict(Config):
"""A mapping of bands to Sedterms.
To construct a SedtermDict use keyword arguments::
SedtermDict(data=dataDict)
where dataDict is a Python dict of band to Sedterm
For example::
SedtermDict(data={
'g': Sedterm(primaryTerm='gr', secondaryTerm='ri', extrapolated=True, constant=0.25,
primaryBand='g', secondaryBand='r', tertiaryBand='i'),
'r': Sedterm(primaryTerm='gr', secondaryTerm='ri', extrapolated=False)
})
This is a subclass of Config. This follows the form of
`lsst.pipe.tasks.ColortermDict`.
"""
data = ConfigDictField(
doc="Mapping of band name to Sedterm",
keytype=str,
itemtype=Sedterm,
default={},
)
class ColortermDict(Config):
"""!A mapping of filterName to Colorterm
Different reference catalogs may need different ColortermDicts; see ColortermLibrary
To construct a ColortermDict use keyword arguments:
ColortermDict(data=dataDict)
where dataDict is a Python dict of filterName: Colorterm
For example:
ColortermDict(data={
'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883),
'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248),
'i': Colorterm(primary="i", secondary="z", c0= 0.00130204, c1=-0.16922042, c2=-0.01374245),
})
The constructor will likely be simplified at some point.
This is subclass of Config. That is a bit of a hack to make it easy to store the data
in an appropriate obs_* package as a config override file. In the long term some other
means of persistence will be used, at which point the constructor can be made saner.
"""
data = ConfigDictField(
doc="Mapping of filter name to Colorterm",
keytype=str,
itemtype=Colorterm,
default={},
)
class MultiXYTransformConfig(Config):
transformDict = ConfigDictField(
doc=
"Dict of index: OneXYTransformConfig (a transform wrapper); key order is transform order",
keytype=int,
itemtype=OneXYTransformConfig,
)
class MultiConfig(Config):
configs = ConfigDictField(
keytype=str,
itemtype=configClass,
optional=False,
default={},
doc="A collection of multiple configs to create multiple items "
"of the same type.")
class MultiTransformConfig(Config):
"""A Config representing a chain of consecutive ``Transforms``.
"""
transformDict = ConfigDictField(
doc="Dict of index: OneTransformConfig (a transform wrapper); "
"key order is transform order",
keytype=int,
itemtype=OneTransformConfig,
)
class SedboundarytermDict(Config):
"""A mapping of Sedboundaryterm name to Sedterm.
To construct a SedboundarytermDict use keyword arguments:
SedboundarytermDict(data=dataDict)
where dataDict is a Python dict of name: Sedterm
For example::
SedboundarytermDict(data={
'gr': Sedboundaryterm(primary="g", secondary="r"),
'ri': Sedboundaryterm(primary="r", secondary="i"),
})
This is a subclass of Config. This follows the form of
`lsst.pipe.tasks.ColortermDict`.
"""
data = ConfigDictField(
doc="Mapping of Sedboundary term name to Sedboundaryterm",
keytype=str,
itemtype=Sedboundaryterm,
default={},
)
class ColortermLibrary(Config):
"""!A mapping of photometric reference catalog name or glob to ColortermDict
This allows photometric calibration using a variety of reference catalogs.
To construct a ColortermLibrary, use keyword arguments:
ColortermLibrary(data=dataDict)
where dataDict is a Python dict of catalog_name_or_glob: ColortermDict
For example:
ColortermLibrary(data = {
"hsc*": ColortermDict(data={
'g': Colorterm(primary="g", secondary="g"),
'r': Colorterm(primary="r", secondary="r"),
...
}),
"sdss*": ColortermDict(data={
'g': Colorterm(primary="g", secondary="r", c0=-0.00816446, c1=-0.08366937, c2=-0.00726883),
'r': Colorterm(primary="r", secondary="i", c0= 0.00231810, c1= 0.01284177, c2=-0.03068248),
...
}),
})
This is subclass of Config. That is a bit of a hack to make it easy to store the data
in an appropriate obs_* package as a config override file. In the long term some other
means of persistence will be used, at which point the constructor can be made saner.
"""
data = ConfigDictField(
doc="Mapping of reference catalog name (or glob) to ColortermDict",
keytype=str,
itemtype=ColortermDict,
default={},
)
def getColorterm(self, filterName, photoCatName, doRaise=True):
"""!Get the appropriate Colorterm from the library
Use dict of color terms in the library that matches the photoCatName.
If the photoCatName exactly matches an entry in the library, that
dict is used; otherwise if the photoCatName matches a single glob (shell syntax,
e.g., "sdss-*" will match "sdss-dr8"), then that is used. If there is no
exact match and no unique match to the globs, raise an exception.
@param filterName name of filter
@param photoCatName name of photometric reference catalog from which to retrieve the data.
This argument is not glob-expanded (but the catalog names in the library are,
if no exact match is found).
@param[in] doRaise if True then raise ColortermNotFoundError if no suitable Colorterm found;
if False then return a null Colorterm with filterName as the primary and secondary filter
@return the appropriate Colorterm
@throw ColortermNotFoundError if no suitable Colorterm found and doRaise true;
other exceptions may be raised for unexpected errors, regardless of the value of doRaise
"""
try:
trueRefCatName = None
ctDictConfig = self.data.get(photoCatName)
if ctDictConfig is None:
# try glob expression
matchList = [
libRefNameGlob for libRefNameGlob in self.data
if fnmatch.fnmatch(photoCatName, libRefNameGlob)
]
if len(matchList) == 1:
trueRefCatName = matchList[0]
ctDictConfig = self.data[trueRefCatName]
elif len(matchList) > 1:
raise ColortermNotFoundError(
"Multiple library globs match photoCatName %r: %s" %
(photoCatName, matchList))
else:
raise ColortermNotFoundError(
"No colorterm dict found with photoCatName %r" %
photoCatName)
ctDict = ctDictConfig.data
if filterName not in ctDict:
# Perhaps it's an alias
try:
filterName = Filter(Filter(filterName).getId()).getName()
except pexExcept.NotFoundError:
pass # this will be handled shortly
if filterName not in ctDict:
errMsg = "No colorterm found for filter %r with photoCatName %r" % (
filterName, photoCatName)
if trueRefCatName is not None:
errMsg += " = catalog %r" % (trueRefCatName, )
raise ColortermNotFoundError(errMsg)
return ctDict[filterName]
except ColortermNotFoundError:
if doRaise:
raise
else:
return Colorterm(filterName, filterName)
class TestConfig(Config):
field = ConfigDictField(keytype=int, itemtype=ApdbConfig, doc="")
class ConvertRepoConfig(Config):
raws = ConfigurableField(
"Configuration for subtask responsible for ingesting raws and adding "
"visit and exposure dimension entries.",
target=RawIngestTask,
)
skyMaps = ConfigDictField(
"Mapping from Gen3 skymap name to the parameters used to construct a "
"BaseSkyMap instance. This will be used to associate names with "
"existing skymaps found in the Gen2 repo.",
keytype=str,
itemtype=ConvertRepoSkyMapConfig,
default={})
rootSkyMapName = Field(
"Name of a Gen3 skymap (an entry in ``self.skyMaps``) to assume for "
"datasets in the root repository when no SkyMap is found there. ",
dtype=str,
optional=True,
default=None,
)
collections = DictField(
"Special collections (values) for certain dataset types (keys). "
"These are used in addition to rerun collections for datasets in "
"reruns. The 'raw' dataset must have an entry here if it is to be "
"converted.",
keytype=str,
itemtype=str,
default={
"deepCoadd_skyMap": "skymaps",
"brightObjectMask": "masks",
})
storageClasses = DictField(
"Mapping from dataset type name or Gen2 policy entry (e.g. 'python' "
"or 'persistable') to the Gen3 StorageClass name.",
keytype=str,
itemtype=str,
default={
"BaseSkyMap": "SkyMap",
"BaseCatalog": "Catalog",
"BackgroundList": "Background",
"raw": "Exposure",
"MultilevelParquetTable": "DataFrame",
"ParquetTable": "DataFrame",
"SkyWcs": "Wcs",
})
doRegisterInstrument = Field(
"If True (default), add dimension records for the Instrument and its "
"filters and detectors to the registry instead of assuming they are "
"already present.",
dtype=bool,
default=True,
)
doWriteCuratedCalibrations = Field(
"If True (default), ingest human-curated calibrations directly via "
"the Instrument interface. Note that these calibrations are never "
"converted from Gen2 repositories.",
dtype=bool,
default=True,
)
refCats = ListField(
"The names of reference catalogs (subdirectories under ref_cats) to "
"be converted",
dtype=str,
default=[])
fileIgnorePatterns = ListField(
"Filename globs that should be ignored instead of being treated as "
"datasets.",
dtype=str,
default=[
"README.txt", "*~?", "butler.yaml", "gen3.sqlite3",
"registry.sqlite3", "calibRegistry.sqlite3", "_mapper", "_parent",
"repositoryCfg.yaml"
])
datasetIncludePatterns = ListField(
"Glob-style patterns for dataset type names that should be converted.",
dtype=str,
default=["*"])
datasetIgnorePatterns = ListField(
"Glob-style patterns for dataset type names that should not be "
"converted despite matching a pattern in datasetIncludePatterns.",
dtype=str,
default=[])
ccdKey = Field(
"Key used for the Gen2 equivalent of 'detector' in data IDs.",
dtype=str,
default="ccd",
)
relatedOnly = Field(
"If True (default), only convert datasets that are related to the "
"ingested visits. Ignored unless a list of visits is passed to "
"run().",
dtype=bool,
default=False,
)
@property
def transfer(self):
return self.raws.transfer
@transfer.setter
def transfer(self, value):
self.raws.transfer = value
@property
def instrument(self):
return self.raws.instrument
@instrument.setter
def instrument(self, value):
self.raws.instrument = value
def setDefaults(self):
self.transfer = None
class ConvertRepoConfig(Config):
raws = ConfigurableField(
"Configuration for subtask responsible for ingesting raws and adding "
"exposure dimension entries.",
target=RawIngestTask,
)
defineVisits = ConfigurableField(
"Configuration for the subtask responsible for defining visits from "
"exposures.",
target=DefineVisitsTask,
)
skyMaps = ConfigDictField(
"Mapping from Gen3 skymap name to the parameters used to construct a "
"BaseSkyMap instance. This will be used to associate names with "
"existing skymaps found in the Gen2 repo.",
keytype=str,
itemtype=ConvertRepoSkyMapConfig,
default={}
)
rootSkyMapName = Field(
"Name of a Gen3 skymap (an entry in ``self.skyMaps``) to assume for "
"datasets in the root repository when no SkyMap is found there. ",
dtype=str,
optional=True,
default=None,
)
runs = DictField(
"A mapping from dataset type name to the RUN collection they should "
"be inserted into. This must include all datasets that can be found "
"in the root repository; other repositories will use per-repository "
"runs.",
keytype=str,
itemtype=str,
default={},
)
runsForced = DictField(
"Like ``runs``, but is used even when the dataset is present in a "
"non-root repository (i.e. rerun), overriding the non-root "
"repository's main collection.",
keytype=str,
itemtype=str,
default={
"brightObjectMask": "masks",
}
)
storageClasses = DictField(
"Mapping from dataset type name or Gen2 policy entry (e.g. 'python' "
"or 'persistable') to the Gen3 StorageClass name.",
keytype=str,
itemtype=str,
default={
"bias": "ExposureF",
"dark": "ExposureF",
"flat": "ExposureF",
"defects": "Defects",
"crosstalk": "CrosstalkCalib",
"BaseSkyMap": "SkyMap",
"BaseCatalog": "Catalog",
"BackgroundList": "Background",
"raw": "Exposure",
"MultilevelParquetTable": "DataFrame",
"ParquetTable": "DataFrame",
"SkyWcs": "Wcs",
}
)
formatterClasses = DictField(
"Mapping from dataset type name to formatter class. "
"By default these are derived from the formatters listed in the"
" Gen3 datastore configuration.",
keytype=str,
itemtype=str,
default={}
)
targetHandlerClasses = DictField(
"Mapping from dataset type name to target handler class.",
keytype=str,
itemtype=str,
default={}
)
doRegisterInstrument = Field(
"If True (default), add dimension records for the Instrument and its "
"filters and detectors to the registry instead of assuming they are "
"already present.",
dtype=bool,
default=True,
)
refCats = ListField(
"The names of reference catalogs (subdirectories under ref_cats) to "
"be converted",
dtype=str,
default=[]
)
fileIgnorePatterns = ListField(
"Filename globs that should be ignored instead of being treated as "
"datasets.",
dtype=str,
default=["README.txt", "*~?", "butler.yaml", "gen3.sqlite3",
"registry.sqlite3", "calibRegistry.sqlite3", "_mapper",
"_parent", "repositoryCfg.yaml"]
)
rawDatasetType = Field(
"Gen2 dataset type to use for raw data.",
dtype=str,
default="raw",
)
datasetIncludePatterns = ListField(
"Glob-style patterns for dataset type names that should be converted.",
dtype=str,
default=["*"]
)
datasetIgnorePatterns = ListField(
"Glob-style patterns for dataset type names that should not be "
"converted despite matching a pattern in datasetIncludePatterns.",
dtype=str,
default=[]
)
ccdKey = Field(
"Key used for the Gen2 equivalent of 'detector' in data IDs.",
dtype=str,
default="ccd",
)
relatedOnly = Field(
"If True (default), only convert datasets that are related to the "
"ingested visits. Ignored unless a list of visits is passed to "
"run().",
dtype=bool,
default=False,
)
doMakeUmbrellaCollection = Field(
"If True (default), define an '<instrument>/defaults' CHAINED "
"collection that includes everything found in the root repo as well "
"as the default calibration collection.",
dtype=bool,
default=True,
)
extraUmbrellaChildren = ListField(
"Additional child collections to include in the umbrella collection. "
"Ignored if doMakeUmbrellaCollection=False.",
dtype=str,
default=[]
)
@property
def transfer(self):
return self.raws.transfer
@transfer.setter
def transfer(self, value):
self.raws.transfer = value
def setDefaults(self):
self.transfer = None