def prep(self):
# Docstring inherited from RepoConverter.
# Gather information about reference catalogs.
if self.task.isDatasetTypeIncluded("ref_cat") and len(self.task.config.refCats) != 0:
from lsst.meas.algorithms import DatasetConfig as RefCatDatasetConfig
for refCat in os.listdir(os.path.join(self.root, "ref_cats")):
path = os.path.join(self.root, "ref_cats", refCat)
configFile = os.path.join(path, "config.py")
if not os.path.exists(configFile):
continue
if refCat not in self.task.config.refCats:
continue
self.task.log.info(f"Preparing ref_cat {refCat} from root {self.root}.")
onDiskConfig = RefCatDatasetConfig()
onDiskConfig.load(configFile)
if onDiskConfig.indexer.name != "HTM":
raise ValueError(f"Reference catalog '{refCat}' uses unsupported "
f"pixelization '{onDiskConfig.indexer.name}'.")
level = onDiskConfig.indexer["HTM"].depth
try:
dimension = self.task.universe[f"htm{level}"]
except KeyError as err:
raise ValueError(f"Reference catalog {refCat} uses HTM level {level}, but no htm{level} "
f"skypix dimension is configured for this registry.") from err
self.task.useSkyPix(dimension)
self._refCats[refCat] = dimension
if self.task.isDatasetTypeIncluded("brightObjectMask") and self.task.config.rootSkyMapName:
self.task.useSkyMap(self._rootSkyMap, self.task.config.rootSkyMapName)
super().prep()
def setUp(self):
struct = lsst.jointcal.testUtils.createTwoFakeCcdImages(100, 100)
self.ccdImageList = struct.ccdImageList
# so that countStars() returns nonzero results
for ccdImage in self.ccdImageList:
ccdImage.resetCatalogForFit()
self.goodChi2 = lsst.jointcal.chi2.Chi2Statistic()
# chi2/ndof == 2.0 should be non-bad
self.goodChi2.chi2 = 200.0
self.goodChi2.ndof = 100
self.badChi2 = lsst.jointcal.chi2.Chi2Statistic()
self.badChi2.chi2 = 600.0
self.badChi2.ndof = 100
self.nanChi2 = lsst.jointcal.chi2.Chi2Statistic()
self.nanChi2.chi2 = np.nan
self.nanChi2.ndof = 100
self.maxSteps = 20
self.name = "testing"
self.dataName = "fake"
self.whatToFit = "" # unneeded, since we're mocking the fitter
# Mock a Butler so the refObjLoaders have something to call `get()` on.
self.butler = unittest.mock.Mock(spec=lsst.daf.persistence.Butler)
self.butler.get.return_value.indexer = DatasetConfig().indexer
# Mock the association manager and give it access to the ccd list above.
self.associations = mock.Mock(spec=lsst.jointcal.Associations)
self.associations.getCcdImageList.return_value = self.ccdImageList
# a default config to be modified by individual tests
self.config = lsst.jointcal.jointcal.JointcalConfig()
def prep(self):
# Docstring inherited from RepoConverter.
# Gather information about raws.
if self.task.raws is not None:
self.task.log.info(f"Preparing raws from root {self.root}.")
if self.subset is not None:
dataRefs = itertools.chain.from_iterable(
self.butler2.subset("raw", visit=visit)
for visit in self.subset.visits)
else:
dataRefs = self.butler2.subset("raw")
self._exposureData.extend(
self.task.raws.prep(dataRef.getUri() for dataRef in dataRefs))
# Gather information about reference catalogs.
if self.task.isDatasetTypeIncluded("ref_cat"):
from lsst.meas.algorithms import DatasetConfig as RefCatDatasetConfig
for refCat in os.listdir(os.path.join(self.root, "ref_cats")):
path = os.path.join(self.root, "ref_cats", refCat)
configFile = os.path.join(path, "config.py")
if not os.path.exists(configFile):
continue
if refCat not in self.task.config.refCats:
continue
self.task.log.info(
f"Preparing ref_cat {refCat} from root {self.root}.")
onDiskConfig = RefCatDatasetConfig()
onDiskConfig.load(configFile)
if onDiskConfig.indexer.name != "HTM":
raise ValueError(
f"Reference catalog '{refCat}' uses unsupported "
f"pixelization '{onDiskConfig.indexer.name}'.")
level = onDiskConfig.indexer["HTM"].depth
try:
dimension = self.task.universe[f"htm{level}"]
except KeyError as err:
raise ValueError(
f"Reference catalog {refCat} uses HTM level {level}, but no htm{level} "
f"skypix dimension is configured for this registry."
) from err
self.task.useSkyPix(dimension)
self._refCats.append((refCat, dimension))
if self.task.isDatasetTypeIncluded(
"brightObjectMask") and self.task.config.rootSkyMapName:
self.task.useSkyMap(self._rootSkyMap)
super().prep()
def mock_get(arg):
"""Return things that sub-tasks need from butler.get()."""
if arg == 'verify_job_filename':
return 'some/path/filename.job'
elif arg == 'ref_cat_config':
result = unittest.mock.Mock(DatasetConfig)
result.indexer = DatasetConfig().indexer
return result
else:
return None
def main():
import argparse
import sys
class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
pass
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=CustomFormatter)
parser.add_argument("path",
help="Directory (written by IngestIndexedReferenceTask) containing the"
" reference catalogs to overwrite."
" All files with a `.fits` extension in the directory will be processed,"
" including `master_schema.fits`, which must exist.")
parser.add_argument('-n', '--nprocesses', default=1, type=int,
help="Number of processes to use when reading and writing files.")
parser.add_argument('--write', action="store_true",
help="Write the corrected files (default just prints what would have changed).")
parser.add_argument('--quiet', action="store_true",
help="Be less verbose about what files and fields are being converted.")
args = parser.parse_args()
schema_file = os.path.join(args.path, "master_schema.fits")
if not os.path.isfile(schema_file):
print("Error: Cannot find master_schema.fits in supplied path:", args.path)
sys.exit(-1)
configPath = os.path.join(args.path, 'config.py')
config = DatasetConfig()
config.load(configPath)
if not is_old_schema(config, schema_file):
print("Catalog does not contain old-style fluxes; nothing to convert.")
sys.exit(0)
files = glob.glob(os.path.join(args.path, "*.fits"))
with concurrent.futures.ProcessPoolExecutor(max_workers=args.nprocesses) as executor:
futures = executor.map(process_one, files, itertools.repeat(args.write), itertools.repeat(args.quiet))
# we have to at least loop over the futures, otherwise exceptions will be lost
for future in futures:
pass
if args.write:
config.format_version = 1
# update the docstring to annotate the file.
msg = "\nUpdated refcat from version 0->1 to have nJy flux units via convert_refcat_to_nJy.py"
config._fields['format_version'].doc += msg
config.save(configPath)
if not args.quiet:
print("Added `format_version=1` to config.py")