def setUp(self):
self.butler = Butler(
os.path.join(weeklyRerun, "pipeline", self.configuration,
"pipeline"))
self.visits = dict(brn=getBrnVisits,
bmn=getBmnVisits)[self.configuration]()
self.design = PfsDesign.read(1, weeklyRaw)
def loadAllTables(filters, newpath, oldpath, dataPath, patch, tract):
logger.info("Loading new catalog")
newCats = loadCatalogs(filters, newpath, "template")
logger.info("Loading new flux conserved catalog")
newCats2 = loadCatalogs(filters, newpath)
logger.info("Loading old catalog")
oldCats = loadCatalogs(filters, oldpath)
logger.info("Building astropy tables")
newFlux, newSed = getAllFlux(newCats, filters)
newFlux2, newSed2 = getAllFlux(newCats2, filters)
newTable = buildAllTables(newCats, newFlux, newSed, filters)
newTable2 = buildAllTables(newCats2, newFlux2, newSed2, filters)
newTable = newTable[(newTable["parent"] != 0) & ~np.isnan(newTable["x"])]
newTable2 = newTable2[(newTable2["parent"] != 0)
& ~np.isnan(newTable2["x"])]
oldTables = OrderedDict([(f, buildTable(oldCats[f])) for f in filters])
logger.info("matching results")
oldTable, matches = matchAllCatalogs(newTable, oldTables, filters)
matchedNew = newTable[matches]
matchedNew2 = newTable2[matches]
logger.info("loading calexps")
butler = Butler(inputs=dataPath)
calexp = OrderedDict()
for f in filters:
calexp[f] = butler.get('deepCoadd_calexp',
patch=patch,
filter="HSC-" + f,
tract=tract)
return oldTable, matchedNew, matchedNew2, calexp, newCats
def makeCompatibleRepo(self, repoDir, calibRepoDir):
"""Set up a directory as a repository compatible with this dataset.
If the directory already exists, any files required by the dataset will
be added if absent; otherwise the directory will remain unchanged.
Parameters
----------
repoDir : `str`
The directory where the output repository will be created.
calibRepoDir : `str`
The directory where the output calibration repository will be created.
"""
mapperArgs = {'mapperArgs': {'calibRoot': calibRepoDir}}
if _isRepo(self.templateLocation):
# Stub repo is not a parent because can't mix v1 and v2 repositories in parents list
Butler(inputs=[{
"root": self.templateLocation,
"mode": "r"
}],
outputs=[{
"root": repoDir,
"mode": "rw",
**mapperArgs
}])
else:
Butler(inputs=[{
"root": self._stubInputRepo,
"mode": "r"
}],
outputs=[{
"root": repoDir,
"mode": "rw",
**mapperArgs
}])
def load_tract(repo, tract, patches=None, **kwargs):
"""Merge catalogs from forced-photometry coadds across available filters.
Parameters
--
tract: int
Tract of sky region to load
repo: str
File location of Butler repository+rerun to load.
patches: list of str
List of patches. If not specified, will default to '0,0'--'7,7'.
Returns
--
AstroPy Table of merged catalog
"""
butler = Butler(repo)
if patches is None:
# Extract the patches for this tract from the skymap
skymap = butler.get(datasetType='deepCoadd_skyMap')
patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]]
merged_patch_cats = []
for patch in patches:
this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs)
# Event if this_patch_merged_cat is an empty Table, it's still fine to append to the list here.
# They will get vstacked away below.
merged_patch_cats.append(this_patch_merged_cat)
merged_tract_cat = vstack(merged_patch_cats)
return merged_tract_cat
def load_tract(repo, tract, patches=None, **kwargs):
"""Merge catalogs from forced-photometry coadds across available filters.
Parameters
--
tract: int
Tract of sky region to load
repo: str
File location of Butler repository+rerun to load.
patches: list of str
List of patches. If not specified, will default to '0,0'--'7,7'.
Returns
--
Pandas DataFrame of merged catalog
"""
butler = Butler(repo)
if patches is None:
# Extract the patches for this tract from the skymap
skymap = butler.get(datasetType='deepCoadd_skyMap')
patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]]
merged_tract_cat = pd.DataFrame()
for patch in patches:
this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs)
merged_tract_cat.append(this_patch_merged_cat)
return merged_tract_cat
class MatplotlibStorageTestCase(lsst.utils.tests.TestCase):
def setUp(self):
inputDir = os.path.join(ROOT, "data", "input")
self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'),
prefix=type(self).__name__ + '-')
self.butler = Butler(inputs=inputDir,
outputs={
"root": self.testDir,
"mode": 'rw'
})
def tearDown(self):
del self.butler
if os.path.exists(self.testDir):
shutil.rmtree(self.testDir)
def testWriteFigure(self):
"""Test writing a matpotlib figure to a repository."""
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
fig = pyplot.figure()
pyplot.plot([0, 1], [0, 1], "k")
self.butler.put(fig, "test_plot", visit=1, filter="g")
self.assertTrue(
self.butler.datasetExists("test_plot", visit=1, filter="g"))
self.assertTrue(
os.path.exists(self.butler.getUri("test_plot", visit=1,
filter="g")))
def testCameraFromButler(self):
"""Test that the butler can return the camera
"""
butler = Butler(self.input)
camera = butler.get("camera", immediate=True)
self.assertEqual(camera.getName(), self.mapper.camera.getName())
self.assertEqual(len(camera), len(self.mapper.camera))
self.assertEqual(len(camera[0]), len(self.mapper.camera[0]))
def setUp(self):
inputDir = os.path.join(ROOT, "data", "input")
self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'),
prefix=type(self).__name__ + '-')
self.butler = Butler(inputs=inputDir,
outputs={
"root": self.testDir,
"mode": 'rw'
})
def makeLinearizerDecam(fromFile, force=False, verbose=False):
"""Convert the specified DECam linearity FITS table to standard LSST format
Details:
- Input format is one table per CCD, HDU is amplifier number,
the table has 3 columns: ADU, ADU_LINEAR_A, ADU_LINEAR_B.
The values of ADU contiguous (0, 1, 2...) but check and error out if not.
The values of the latter two are replacements (0+delta0, 1+delta1, 2+delta2...)
and this is converted to offsets for the LSST linearization tables (delta0, delta1, delta2...)
- Output is a set of LinearizeLookupTable instances, one per CCD, saved as dataset type "linearizer"
- The row indices for the linearization lookup table are (row index=amp name): 0=A, 1=B
@param[in] fromFile path to DECam linearity table (a FITS file with one HDU per amplifier)
"""
print("Making DECam linearizers from %r" % (fromFile, ))
butler = Butler(mapper=DecamMapper)
linearizerDir = DecamMapper.getLinearizerDir()
if os.path.exists(linearizerDir):
if not force:
print("Output directory %r exists; use --force to replace" %
(linearizerDir, ))
sys.exit(1)
print("Replacing data in linearizer directory %r" % (linearizerDir, ))
else:
print("Creating linearizer directory %r" % (linearizerDir, ))
os.makedirs(linearizerDir)
camera = DecamMapper().camera
fromHDUs = fits.open(fromFile)[1:] # HDU 0 has no data
assert len(fromHDUs) == len(camera)
for ccdind, (detector, hdu) in enumerate(zip(camera, fromHDUs)):
ccdnum = ccdind + 1
if verbose:
print("ccdnum=%s; detector=%s" % (ccdnum, detector.getName()))
fromData = hdu.data
assert len(fromData.dtype) == 3
lsstTable = np.zeros((2, len(fromData)), dtype=np.float32)
uncorr = fromData["ADU"]
if not np.allclose(uncorr, np.arange(len(fromData))):
raise RuntimeError(
"ADU data not a range of integers starting at 0")
for i, ampName in enumerate("AB"):
# convert DECam replacement table to LSST offset table
if verbose:
print("DECam table for %s=%s..." % (
ampName,
fromData["ADU_LINEAR_" + ampName][0:5],
))
lsstTable[i, :] = fromData["ADU_LINEAR_" + ampName] - uncorr
if verbose:
print("LSST table for %s=%s..." % (
ampName,
lsstTable[i, 0:5],
))
linearizer = LinearizeLookupTable(table=lsstTable, detector=detector)
butler.put(linearizer, "linearizer", dataId=dict(ccdnum=ccdnum))
print("Wrote %s linearizers" % (ccdind + 1, ))
def ingest(cls, root, camera, visit, filenames, sensors, metadata):
"""Add all images from an external visit (a full-focal-plane
exposure) to a data repository.
This both symlinks the external data files to the appropriate
location in the directory structure and adds the necessary
rows to the SQLite registry tables.
Parameters
----------
root : str
Directory of the data repository to add data to. Must have
an existing "registry.sqlite3" file present directly in the
root and a _mapper file pointing to HscAndExtMapper.
camera : str
Name of the camera used to produced the external observation.
Must have an entry in ExternalImage.CAMERA_INFO.
visit : int
Original integer visit ID for the observation, *before* adding
CAMERA_INFO[camera]["ID"]*CAMERA_ID_MULTIPLIER.
filenames : list
A list of file names containing the external data files, either
relative to the current directory or absolute.
sensors : list
A list of integer sensor IDs corresponding to the filenames list.
metadata : VisitMetadata
An object containing additional metadata for this visit to be
added to the registry. See VisitMetadata for a description of
what attributes are required.
"""
db = sqlite3.connect(os.path.join(root, "registry.sqlite3"))
butler = Butler(inputs=[root])
visit += cls.CAMERA_INFO[camera]["id"] * cls.CAMERA_ID_MULTIPLIER
ccdCols = [
"filter", "dateObs", "taiObs", "field", "expId", "pointing",
"dataType", "pa"
]
ccdSql = "INSERT INTO raw (visit, ccd, {}) VALUES (?, ?, {})".format(
", ".join(ccdCols), ", ".join(["?"] * len(ccdCols)))
ccdValues = tuple(getattr(metadata, col) for col in ccdCols)
visitCols = ["filter", "dateObs", "taiObs", "field"]
visitSql = "INSERT INTO raw_visit (visit, {}) VALUES (?, {})".format(
", ".join(visitCols), ", ".join(["?"] * len(visitCols)))
visitValues = tuple(getattr(metadata, col) for col in visitCols)
for filename, sensor in zip(filenames, sensors):
outputFileName = butler.get("external_filename",
visit=visit,
ccd=sensor)[0]
os.symlink(filename, outputFileName)
db.execute(ccdSql, (
visit,
sensor,
) + ccdValues)
db.execute(visitSql, (visit, ) + visitValues)
db.commit()
return visit
def testDetectors(self):
"""Test that the detector returned by the gen 2 butler is the same
as the expected one.
"""
for root, did, expected in zip(self.roots, self.ids, self.expecteds):
butler = Butler(root)
raw = butler.get("raw", dataId=did)
for amp1, amp2 in zip(expected['detector'], raw.getDetector()):
with self.subTest(amp=amp1.getName()):
self.assertEqual(amp1.getName(), amp2.getName())
self.assertAmpRawBBoxesEqual(amp1, amp2)
def testAssemble(self):
"""Test the assembly of E2V and ITL sensors
"""
task = AssembleCcdTask()
# exclude LATISS for this test since we don't have an expected output
for root, did, expected in zip(self.roots, self.ids, self.expecteds):
butler = Butler(root)
raw = butler.get("raw", dataId=did)
assembled = task.assembleCcd(raw)
count = numpy.sum(expected['expected'].read().array -
assembled.getImage().array)
self.assertEqual(count, 0)
class SimpleButlerInterface(BaseGenericCatalog):
"""
A simple butler interface.
Args:
-----
repo: str
path to repository containing the DM-processed data.
dataId: dict
"""
def _subclass_init(self, repo, datasetType, dataId=None, **kwargs):
if not _HAS_LSST_STACK:
raise RuntimeError('LSST Stack not available')
self._butler = Butler(repo)
self._datasetType = datasetType
self._dataId_cache = self._butler.subset(self._datasetType,
dataId=dataId).cache
self._columns = None
for dataId in self._dataId_cache:
data = self._get_data(dataId)
if data is not None:
self._columns = data.schema.getNames()
break
if not self._columns:
raise RuntimeError('No datasets or columns found!')
def _get_data(self, dataId, datasetType=None):
try:
data = self._butler.get(datasetType or self._datasetType,
dataId=dataId)
except NoResults:
return None
return data
def _generate_quantity_getter(self, dataId):
data = self._get_data(dataId)
if data is None:
return
return data.get
def _iter_native_dataset(self, native_filters=None):
for dataId in self._dataId_cache:
if native_filters is None or native_filters.check_scalar(dataId):
quantity_getter = self._generate_quantity_getter(dataId)
if quantity_getter is not None:
yield quantity_getter
def _generate_native_quantity_list(self):
return self._columns
def load_and_save_tract(repo,
tract,
filename,
key_prefix='coadd',
patches=None,
overwrite=True,
verbose=False,
**kwargs):
"""Save catalogs to HDF5 from forced-photometry coadds across available filters.
Iterates through patches, saving each in append mode to the save HDF5 file.
Parameters
--
repo: str
File location of Butler repository+rerun to load.
tract: int
Tract of sky region to load
filename: str
Filename for HDF file.
key_prefix: str
Base for the key in the HDF file.
Keys will be of the form "%s_%d_%s" % (keybase, tract, patch)
With the addition that the comma will be removed from the patch name
to provide a valid Python identifier: e.g., 'coadd_4849_11'
overwrite: bool
Overwrite an existing HDF file.
"""
butler = Butler(repo)
if patches is None:
# Extract the patches for this tract from the skymap
skymap = butler.get(datasetType='deepCoadd_skyMap')
patches = ['%d,%d' % patch.getIndex() for patch in skymap[tract]]
for patch in patches:
if verbose:
print("Processing tract %d, patch %s" % (tract, patch))
patch_merged_cat = load_patch(butler,
tract,
patch,
verbose=verbose,
**kwargs)
if len(patch_merged_cat) == 0:
if verbose:
print(" No good entries for tract %d, patch %s" %
(tract, patch))
continue
key = '%s_%d_%s' % (key_prefix, tract, patch)
key = valid_identifier_name(key)
patch_merged_cat.to_pandas().to_hdf(filename, key, format='fixed')
def makeLinearizerDecam(fromFile, force=False, verbose=False):
"""Convert the specified DECam linearity FITS table to standard LSST format
Details:
- Input format is one table per CCD, HDU is amplifier number,
the table has 3 columns: ADU, ADU_LINEAR_A, ADU_LINEAR_B.
The values of ADU contiguous (0, 1, 2...) but check and error out if not.
The values of the latter two are replacements (0+delta0, 1+delta1, 2+delta2...)
and this is converted to offsets for the LSST linearization tables (delta0, delta1, delta2...)
- Output is a set of LinearizeLookupTable instances, one per CCD, saved as dataset type "linearizer"
- The row indices for the linearization lookup table are (row index=amp name): 0=A, 1=B
@param[in] fromFile path to DECam linearity table (a FITS file with one HDU per amplifier)
"""
print("Making DECam linearizers from %r" % (fromFile,))
butler = Butler(mapper=DecamMapper)
linearizerDir = DecamMapper.getLinearizerDir()
if os.path.exists(linearizerDir):
if not force:
print("Output directory %r exists; use --force to replace" % (linearizerDir,))
sys.exit(1)
print("Replacing data in linearizer directory %r" % (linearizerDir,))
else:
print("Creating linearizer directory %r" % (linearizerDir,))
os.makedirs(linearizerDir)
camera = DecamMapper().camera
fromHDUs = fits.open(fromFile)[1:] # HDU 0 has no data
assert len(fromHDUs) == len(camera)
for ccdind, (detector, hdu) in enumerate(izip(camera, fromHDUs)):
ccdnum = ccdind + 1
if verbose:
print("ccdnum=%s; detector=%s" % (ccdnum, detector.getName()))
fromData = hdu.data
assert len(fromData.dtype) == 3
lsstTable = np.zeros((2, len(fromData)), dtype=np.float32)
uncorr = fromData["ADU"]
if not np.allclose(uncorr, np.arange(len(fromData))):
raise RuntimeError("ADU data not a range of integers starting at 0")
for i, ampName in enumerate("AB"):
# convert DECam replacement table to LSST offset table
if verbose:
print("DECam table for %s=%s..." % (ampName, fromData["ADU_LINEAR_" + ampName][0:5],))
lsstTable[i,:] = fromData["ADU_LINEAR_" + ampName] - uncorr
if verbose:
print("LSST table for %s=%s..." % (ampName, lsstTable[i,0:5],))
linearizer = LinearizeLookupTable(table=lsstTable, detector=detector)
butler.put(linearizer, "linearizer", dataId=dict(ccdnum=ccdnum))
print("Wrote %s linearizers" % (ccdind+1,))
def connect(self):
# search for metadata.yaml file
# 1. Look in path directory i.e. '/project/tmorton/tickets/DM-20015/RC2_w18/metadata.yaml'
# 2. Look for datafolder in current directory i.e. './RC2_w18/metadata.yaml'
# 3. Look for datafolder in dir specified in LSST_META env variable i.e. /user/name/lsst_meta/RC2_w18/metadata.yaml'
# when LSST_META='/user/name/lsst_meta'
print('-- read metadata file --')
# if Butler is available use it to connect. If not available we are reading from disk
if Butler:
try:
print('-- connect to butler --')
self.conn = Butler(str(self.path))
self.metadata = self.conn.get('qaDashboard_metadata')
self.failures = self.metadata.get('failures', {})
if not self.filters:
self.filters = list(self.metadata['visits'].keys())
if not self.tracts:
all_tracts = [list(self.metadata['visits'][filt].keys()) for filt in self.filters]
self.tracts = list(set([int(y) for x in all_tracts for y in x]))
except:
print(f'{self.path} is not available in Butler attempting to read parquet files instead')
else:
if self.path.joinpath(METADATA_FILENAME).exists():
self.metadata_path = self.path.joinpath(METADATA_FILENAME)
else:
self.metadata_path = Path(os.environ.get('LSST_META', os.curdir)).joinpath(self.path.name, METADATA_FILENAME)
with self.metadata_path.open('r') as f:
self.metadata = yaml.load(f, Loader=yaml.SafeLoader)
self.failures = self.metadata.get('failures', {})
if self.tracts is None:
self.tracts = list(set(x for v in self.metadata['visits'].values() for x in v.keys()))
print('-- read coadd table --')
self.fetch_coadd_table() # currently ignoring forced/unforced
# update metadata based on coadd table fields
print('-- generate other metadata fields --')
df = self.coadd['qaDashboardCoaddTable']
self.flags = df.columns[df.dtypes == bool].to_list()
if not Butler:
self.filters = list(self.metadata['visits'].keys())
self.metrics = set(df.columns.to_list()) - set(self.flags) - set(['patch', 'dec', 'label', 'psfMag',
'ra', 'filter', 'dataset', 'dir0', 'tract'])
print('-- read visit data --')
self.fetch_visits_by_metric()
print('-- done with reads --')
def testExposureIdInfo(self):
butler = Butler(self.input)
expIdBits = self.mapper.bypass_ccdExposureId_bits( # args are ignored
datasetType = None,
pythonType = int,
location = None,
dataId = dict(),
)
for visit in (1, 2, 3):
dataId = dict(visit=visit)
expIdInfo = butler.get("expIdInfo", dataId=dataId)
self.assertEqual(expIdInfo.expId, visit)
self.assertEqual(expIdInfo.expBits, expIdBits)
self.assertEqual(expIdInfo.maxBits, 64)
self.assertEqual(expIdInfo.unusedBits, expIdInfo.maxBits-expIdBits)
def testBasics(self):
"""Test detection and measurement on simple synthesized data
"""
bbox = Box2I(Point2I(256, 100), Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11 # kernel width
sky = 2000
# create an exposure without a Wcs; add the Wcs later
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=True)
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
task = DetectAndMeasureTask(config=config, schema=schema)
butler = Butler(root=InputDir)
dataRef = butler.dataRef("calexp", dataId=dict(visit=1))
wcs = dataRef.get("raw").getWcs()
exposure.setWcs(wcs)
exposureIdInfo = dataRef.get("expIdInfo")
taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
self.assertEqual(len(taskRes.sourceCat), numX * numY)
schema = taskRes.sourceCat.schema
centroidFlagKey = schema.find("slot_Centroid_flag").getKey()
parentKey = schema.find("parent").getKey()
psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey()
psfFluxKey = schema.find("slot_PsfFlux_flux").getKey()
for src in taskRes.sourceCat:
self.assertFalse(src.get(centroidFlagKey)) # centroid found
self.assertEqual(src.get(parentKey), 0) # not debelended
self.assertFalse(src.get(psfFluxFlagKey)) # flux measured
self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcCat = afwTable.SourceCatalog.readFits(
os.path.join(testDir, "data", "v695833-e0-c000.xy.fits"))
self.srcCat["slot_ApFlux_fluxSigma"] = 1
self.srcCat["slot_PsfFlux_fluxSigma"] = 1
# The .xy.fits file has sources in the range ~ [0,2000],[0,4500]
# which is bigger than the exposure
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(2048, 4612))
smallExposure = afwImage.ExposureF(
os.path.join(testDir, "data", "v695833-e0-c000-a00.sci.fits"))
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
self.exposure.setCalib(smallExposure.getCalib())
# Make a reference loader
butler = Butler(RefCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
logLevel = Log.TRACE
self.log = Log.getLogger('testPhotoCal')
self.log.setLevel(logLevel)
self.config = PhotoCalConfig()
# The test and associated data have been prepared on the basis that we
# use the PsfFlux to perform photometry.
self.config.fluxField = "base_PsfFlux_flux"
def load_tract(repo, tract, patches=None, **kwargs):
"""Merge catalogs from forced-photometry coadds across available filters.
Parameters
--
tract: int
Tract of sky region to load
repo: str
File location of Butler repository+rerun to load.
patches: list of str
List of patches. If not specified, will default to '0,0'--'7,7'.
Returns
--
AstroPy Table of merged catalog
"""
butler = Butler(repo)
if patches is None:
patches = ['%d,%d' % (i, j) for i in range(8) for j in range(8)]
merged_patch_cats = []
for patch in patches:
try:
this_patch_merged_cat = load_patch(butler, tract, patch, **kwargs)
except NoResults as e:
print(e)
continue
merged_patch_cats.append(this_patch_merged_cat)
merged_tract_cat = vstack(merged_patch_cats)
return merged_tract_cat
def get_butler_by_repo(repo, **kwargs):
"""Construct and return a Bulter for the requested repository
Parameters
----------
repo : `str`
Name of the repo, e.g., 'TS8' | 'BOT'
kwargs
Passed to the Bulter constructor
Returns
-------
butler : `Butler`
the requested Bulter
Raises
------
KeyError : If repo does not match any known repository
"""
try:
repo_path = BUTLER_REPO_DICT[repo]
except KeyError:
raise KeyError("Unknown Bulter repo key %s" % repo)
butler = Butler(repo_path, **kwargs)
return butler
def setUp(self):
refCatDir = os.path.join(os.path.dirname(__file__), "data",
"sdssrefcat")
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0),
afwGeom.Extent2I(3001, 3001))
self.ctrPix = afwGeom.Point2I(1500, 1500)
metadata = dafBase.PropertySet()
metadata.set("RADECSYS", "FK5")
metadata.set("EQUINOX", 2000.0)
metadata.set("CTYPE1", "RA---TAN")
metadata.set("CTYPE2", "DEC--TAN")
metadata.set("CUNIT1", "deg")
metadata.set("CUNIT2", "deg")
metadata.set("CRVAL1", 215.5)
metadata.set("CRVAL2", 53.0)
metadata.set("CRPIX1", self.ctrPix[0] + 1)
metadata.set("CRPIX2", self.ctrPix[1] + 1)
metadata.set("CD1_1", 5.1e-05)
metadata.set("CD1_2", 0.0)
metadata.set("CD2_2", -5.1e-05)
metadata.set("CD2_1", 0.0)
self.tanWcs = afwImage.makeWcs(metadata)
self.exposure = afwImage.ExposureF(self.bbox)
self.exposure.setWcs(self.tanWcs)
self.exposure.setFilter(afwImage.Filter("r", True))
butler = Butler(refCatDir)
self.refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
def __init__(
self, butlerpath, destination=None, dataset=None, engine="pyarrow", sample_frac=None, num_buckets=8,
):
self._butler = Butler(butlerpath)
if dataset is None:
dataset = self._default_dataset
self.dataset = dataset
if destination is None:
destination = f"{butlerpath}/ktk"
self.destination = destination
self.sample_frac = sample_frac
self.num_buckets = num_buckets
self.stats_path = f"{self.destination}/{self.dataset}_stats.parq"
self._store = None
self.engine = engine
self.metadata = self.butler.get("qaDashboard_info")
self.dataIds = [
dataId for dataId in self.iter_dataId() if self.butler.datasetExists(self.dataset, dataId)
]
self.filters = [filt for filt in self.metadata["visits"].keys()]
self.dataIds_by_filter = {
filt: [d for d in self.dataIds if d["filter"] == filt] for filt in self.filters
}
self._filenames = None
self._filenames_by_filter = None
def __main__(args):
repo_directory = "/lsstdata/offline/teststand/BOT/gen2repo"
butler = Butler(repo_directory)
runs = ['12672', '12673', '12844', '12845', '12853', '12855']
run = runs[0]
sensors = set(
butler.queryMetadata('raw', ['raftName', 'detectorName'],
dataId={
'run': run,
'imageType': 'BIAS'
}))
run_dark_regression_analysis_on_all_sensors(butler,
runs,
sensors,
show_plots=False)
def setUp(self):
# Load sample input from disk
testDir = os.path.dirname(__file__)
self.srcSet = SourceCatalog.readFits(os.path.join(testDir, "v695833-e0-c000.xy.fits"))
self.bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(2048, 4612)) # approximate
# create an exposure with the right metadata; the closest thing we have is
# apparently v695833-e0-c000-a00.sci.fits, which is much too small
smallExposure = ExposureF(os.path.join(testDir, "v695833-e0-c000-a00.sci.fits"))
self.exposure = ExposureF(self.bbox)
self.exposure.setWcs(smallExposure.getWcs())
self.exposure.setFilter(smallExposure.getFilter())
# copy the pixels we can, in case the user wants a debug display
mi = self.exposure.getMaskedImage()
mi.assign(smallExposure.getMaskedImage(), smallExposure.getBBox())
logLevel = Log.INFO
refCatDir = os.path.join(testDir, "data", "sdssrefcat")
butler = Butler(refCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=butler)
astrometryConfig = AstrometryTask.ConfigClass()
self.astrom = AstrometryTask(config=astrometryConfig, refObjLoader=refObjLoader)
self.astrom.log.setLevel(logLevel)
# Since our sourceSelector is a registry object we have to wait for it to be created
# before setting default values.
self.astrom.matcher.sourceSelector.config.minSnr = 0
class FitsStorageTestCase(lsst.utils.tests.TestCase):
def setUp(self):
inputDir = os.path.join(ROOT, "data", "input")
self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__+'-')
self.butler = Butler(inputs=inputDir, outputs={"root": self.testDir, "mode": 'rw'})
def tearDown(self):
del self.butler
if os.path.exists(self.testDir):
shutil.rmtree(self.testDir)
def testReadPropertyList(self):
"""Test that reading a FitsStorage dataset into a PropertyList
is the same as using the special _md Exposure hook.
"""
md1 = self.butler.get("raw_md", visit=1, filter="g")
md2 = self.butler.get("rawMetadataDirect", visit=1, filter="g")
self.assertEqual(md1, md2)
def get_diff_calexp_coadd_butler(diff_repo, calexp_repo, coadd_repo):
"""
Returns:
butler_dict = {
'diff': diff_butler,
'calexp_butler': calexp_butler,
'coadd_butler': coadd_butler
}
"""
diff_butler = Butler(diff_repo)
calexp_butler = Butler(calexp_repo)
coadd_butler = Butler(coadd_repo)
butler_dict = {
'diff': diff_butler,
'calexp': calexp_butler,
'coadd': coadd_butler
}
return butler_dict
def setUp(self):
np.random.seed(12345)
self.butler = Butler(RefCatDir)
refObjLoader = LoadIndexedReferenceObjectsTask(butler=self.butler)
center = lsst.afw.geom.SpherePoint(215.5, 53.0, lsst.afw.geom.degrees)
radius = 0.5 * lsst.afw.geom.degrees
self.filter = "r"
self.references = refObjLoader.loadSkyCircle(center, radius,
self.filter).refCat
def main(visit, detector, diarepo, outputdir='.'):
diabutler = Butler(diarepo)
diff_img_path = os.path.join(outputdir,
f'diff_exposure_v{visit}_d{detector}.fits')
science_img_path = os.path.join(
outputdir, f'science_exposure_v{visit}_d{detector}.fits')
imgD = diabutler.get('deepDiff_differenceExp',
visit=visit,
detector=detector)
imgD.writeFits(diff_img_path)
imgS = diabutler.get('calexp', visit=visit, detector=detector)
imgS.writeFits(science_img_path)
print(f'files copied to {outputdir}')
return
def testBasics(self):
"""Test detection and measurement on simple synthesized data
"""
bbox = Box2I(Point2I(256, 100), Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11 # kernel width
sky = 2000
# create an exposure without a Wcs; add the Wcs later
exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=True)
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
task = DetectAndMeasureTask(config=config, schema=schema)
butler = Butler(root=InputDir)
dataRef = butler.dataRef("calexp", dataId=dict(visit=1))
wcs = dataRef.get("raw").getWcs()
exposure.setWcs(wcs)
exposureIdInfo = dataRef.get("expIdInfo")
taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
self.assertEqual(len(taskRes.sourceCat), numX * numY)
schema = taskRes.sourceCat.schema
centroidFlagKey = schema.find("slot_Centroid_flag").getKey()
parentKey = schema.find("parent").getKey()
psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey()
psfFluxKey = schema.find("slot_PsfFlux_flux").getKey()
for src in taskRes.sourceCat:
self.assertFalse(src.get(centroidFlagKey)) # centroid found
self.assertEqual(src.get(parentKey), 0) # not debelended
self.assertFalse(src.get(psfFluxFlagKey)) # flux measured
self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def testBasics(self):
"""Test construction of a discrete sky map
"""
butler = Butler(inputs=self.inPath,
outputs={
'root': self.outPath,
'mode': 'rw'
})
coordList = [] # list of sky coords of all corners of all calexp
for dataId in (
dict(visit=1, filter="g"),
dict(visit=2, filter="g"),
dict(visit=3, filter="r"),
):
# TODO: pybind11 remove `immediate=True` once DM-9112 is resolved
rawImage = butler.get("raw", dataId, immediate=True)
# fake calexp by simply copying raw data; the task just cares about its bounding box
# (which is slightly larger for raw, but that doesn't matter for this test)
calexp = rawImage
butler.put(calexp, "calexp", dataId)
calexpWcs = calexp.getWcs()
calexpBoxD = Box2D(calexp.getBBox())
coordList += [
calexpWcs.pixelToSky(corner)
for corner in calexpBoxD.getCorners()
]
# use the calexp to make a sky map
retVal = MakeDiscreteSkyMapTask.parseAndRun(
args=[self.inPath, "--output", self.outPath, "--id", "filter=g^r"],
config=self.config,
doReturnResults=True,
)
self.assertEqual(len(retVal.resultList), 1)
skyMap = retVal.resultList[0].result.skyMap
self.assertEqual(type(skyMap), DiscreteSkyMap)
self.assertEqual(len(skyMap), 1)
tractInfo = skyMap[0]
self.assertEqual(tractInfo.getId(), 0)
self.assertEqual(tractInfo.getNumPatches(), Extent2I(3, 3))
tractWcs = tractInfo.getWcs()
tractBoxD = Box2D(tractInfo.getBBox())
for skyPoint in coordList:
self.assertTrue(tractBoxD.contains(tractWcs.skyToPixel(skyPoint)))
def _subclass_init(self, repo, datasetType, dataId=None, **kwargs):
if not _HAS_LSST_STACK:
raise RuntimeError('LSST Stack not available')
self._butler = Butler(repo)
self._datasetType = datasetType
self._dataId_cache = self._butler.subset(self._datasetType,
dataId=dataId).cache
self._columns = None
for dataId in self._dataId_cache:
data = self._get_data(dataId)
if data is not None:
self._columns = data.schema.getNames()
break
if not self._columns:
raise RuntimeError('No datasets or columns found!')
def extract_instrument_from_repo(repo):
"""Extract the last part of the mapper name from a Butler repo.
'lsst.obs.lsstSim.lsstSimMapper.LsstSimMapper' -> 'LSSTSIM'
'lsst.obs.cfht.megacamMapper.MegacamMapper' -> 'CFHT'
'lsst.obs.decam.decamMapper.DecamMapper' -> 'DECAM'
'lsst.obs.hsc.hscMapper.HscMapper' -> 'HSC'
"""
mapper_class = Butler.getMapperClass(repo)
instrument = mapper_class.getCameraName()
return instrument.upper()
def butler(self):
if not self._butler:
if self.gen3:
GEN3_REPO_ROOT = os.path.join(getPackageDir("ci_hsc_gen2"),
"DATAgen3")
self._butler = lsst.daf.butler.Butler(
GEN3_REPO_ROOT, collections=self.collection)
else:
self._butler = Butler(self.root)
return self._butler
def create(self):
"""Create a Butler.
@returns a new Butler.
"""
if hasattr(self.mapper, 'root'):
root = self.mapper.root
else:
root = None
return Butler(root=root, mapper=self.mapper)
class MatplotlibStorageTestCase(lsst.utils.tests.TestCase):
def setUp(self):
inputDir = os.path.join(ROOT, "data", "input")
self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__+'-')
self.butler = Butler(inputs=inputDir, outputs={"root": self.testDir, "mode": 'rw'})
def tearDown(self):
del self.butler
if os.path.exists(self.testDir):
shutil.rmtree(self.testDir)
def testWriteFigure(self):
"""Test writing a matpotlib figure to a repository."""
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot
fig = pyplot.figure()
pyplot.plot([0, 1], [0, 1], "k")
self.butler.put(fig, "test_plot", visit=1, filter="g")
self.assertTrue(self.butler.datasetExists("test_plot", visit=1, filter="g"))
self.assertTrue(os.path.exists(self.butler.getUri("test_plot", visit=1, filter="g")))
def testBasics(self):
"""Test construction of a discrete sky map
"""
butler = Butler(inputs=self.inPath, outputs={'root': self.outPath, 'mode': 'rw'})
coordList = [] # list of sky coords of all corners of all calexp
for dataId in (
dict(visit=1, filter="g"),
dict(visit=2, filter="g"),
dict(visit=3, filter="r"),
):
rawImage = butler.get("raw", dataId)
# fake calexp by simply copying raw data; the task just cares about its bounding box
# (which is slightly larger for raw, but that doesn't matter for this test)
calexp = rawImage
butler.put(calexp, "calexp", dataId)
calexpWcs = calexp.getWcs()
calexpBoxD = Box2D(calexp.getBBox())
coordList += [calexpWcs.pixelToSky(corner) for corner in calexpBoxD.getCorners()]
# use the calexp to make a sky map
retVal = MakeDiscreteSkyMapTask.parseAndRun(
args=[self.inPath, "--output", self.outPath, "--id", "filter=g^r"],
config=self.config,
doReturnResults=True,
)
self.assertEqual(len(retVal.resultList), 1)
skyMap = retVal.resultList[0].result.skyMap
self.assertEqual(type(skyMap), DiscreteSkyMap)
self.assertEqual(len(skyMap), 1)
tractInfo = skyMap[0]
self.assertEqual(tractInfo.getId(), 0)
self.assertEqual(tractInfo.getNumPatches(), Extent2I(3, 3))
tractWcs = tractInfo.getWcs()
tractBoxD = Box2D(tractInfo.getBBox())
for skyPoint in coordList:
self.assertTrue(tractBoxD.contains(tractWcs.skyToPixel(skyPoint)))
def __init__(self, inputs, outputs=None):
"""Initialize the butler wrapper class.
Parameters
----------
inputs : RepositoryArgs, dict, or str
Can be a single item or a list. Provides arguments to load an
existing repository (or repositories). String is assumed to be a
URI and is used as the cfgRoot (URI to the location of the cfg
file). (Local file system URI does not have to start with
'file://' and in this way can be a relative path). The
'RepositoryArgs' class can be used to provide more parameters with
which to initialize a repository (such as 'mapper', 'mapperArgs',
'tags', etc. See the 'RepositoryArgs' documentation for more
details). A dict may be used as shorthand for a 'RepositoryArgs'
class instance. The dict keys must match parameters to the
'RepositoryArgs.__init__' function.
outputs : RepositoryArgs, dict, or str, optional
Provides arguments to load one or more existing repositories or
create new ones. The different types are handled the same as for
'inputs'. (the default is None.)
"""
self._butler = Butler(inputs=inputs, outputs=outputs)
# Creates the data dictionary used in the ps example
from lsst.daf.persistence import Butler
import pickle
n_max=0
stats={}
# output dir of the stack processing
output_dir = ""
butler = Butler(output_dir)
for dataref in butler.subset(datasetType='src'):
if dataref.datasetExists(): # processrCcd did not fail
visit = dataref.dataId['visit']
src = dataref.get()
n = len(src)
print visit, n
if visit in stats:
stats[visit].append(n)
else:
stats[visit]=list()
stats[visit].append(n)
pickle.dump(stats, open( "../test/ps.pkl", "wb"))
class ButlerWrapper(object):
def __init__(self, inputs, outputs=None):
"""Initialize the butler wrapper class.
Parameters
----------
inputs : RepositoryArgs, dict, or str
Can be a single item or a list. Provides arguments to load an
existing repository (or repositories). String is assumed to be a
URI and is used as the cfgRoot (URI to the location of the cfg
file). (Local file system URI does not have to start with
'file://' and in this way can be a relative path). The
'RepositoryArgs' class can be used to provide more parameters with
which to initialize a repository (such as 'mapper', 'mapperArgs',
'tags', etc. See the 'RepositoryArgs' documentation for more
details). A dict may be used as shorthand for a 'RepositoryArgs'
class instance. The dict keys must match parameters to the
'RepositoryArgs.__init__' function.
outputs : RepositoryArgs, dict, or str, optional
Provides arguments to load one or more existing repositories or
create new ones. The different types are handled the same as for
'inputs'. (the default is None.)
"""
self._butler = Butler(inputs=inputs, outputs=outputs)
def setInputsAndOutputs(self, inputs, outputs=None):
"""Set the inputs and outputs of butler.
Parameters
----------
inputs : RepositoryArgs, dict, or str
Can be a single item or a list. Provides arguments to load an
existing repository (or repositories). String is assumed to be a
URI and is used as the cfgRoot (URI to the location of the cfg
file). (Local file system URI does not have to start with
'file://' and in this way can be a relative path). The
'RepositoryArgs' class can be used to provide more parameters with
which to initialize a repository (such as 'mapper', 'mapperArgs',
'tags', etc. See the 'RepositoryArgs' documentation for more
details). A dict may be used as shorthand for a 'RepositoryArgs'
class instance. The dict keys must match parameters to the
'RepositoryArgs.__init__' function.
outputs : RepositoryArgs, dict, or str, optional
Provides arguments to load one or more existing repositories or
create new ones. The different types are handled the same as for
'inputs'. (the default is None.)
"""
self._butler = Butler(inputs=inputs, outputs=outputs)
def getRawExp(self, visit, raft, sensor, snap=None):
"""Get the raw exposure.
Parameters
----------
visit : int
Visit Id.
raft : str
Abbreviated raft name (e.g. "R22").
sensor : str
Abbreviated sensor name (e.g. "S11").
snap : int, optional
Snap time (0 or 1) means first/ second exposure. (the default is
None.)
Returns
-------
lsst.afw.image.exposure.exposure.ExposureF
Raw exposure object.
"""
dataId = self._getDefaultDataId(visit, raft, sensor)
if (snap is not None) and isinstance(snap, (int, float)):
dataId["snap"] = int(snap)
return self._butler.get("raw", dataId=dataId)
def _getDefaultDataId(self, visit, raft, sensor):
"""Get the default data Id.
Parameters
----------
visit : int
Visit Id.
raft : str
Abbreviated raft name (e.g. "R22").
sensor : str
Abbreviated sensor name (e.g. "S11").
Returns
-------
dict
Default data Id.
"""
dataId = dict(visit=int(visit), raftName=raft, detectorName=sensor)
return dataId
def getPostIsrCcd(self, visit, raft, sensor, afilter=None):
"""Get the post-ISR CCD exposure.
ISR: Instrument signature removal.
CCD: Charge-coupled device.
Parameters
----------
visit : int
Visit Id.
raft : str
Abbreviated raft name (e.g. "R22").
sensor : str
Abbreviated sensor name (e.g. "S11").
afilter : str, optional
Active filter ("u", "g", "r", "i", "z", "y") (the default is None.)
Returns
-------
lsst.afw.image.exposure.exposure.ExposureF
Post-ISR CCD object.
"""
dataId = self._getDefaultDataId(visit, raft, sensor)
if (afilter is not None) and isinstance(afilter, str):
dataId["filter"] = afilter
return self._butler.get("postISRCCD", dataId=dataId)
@staticmethod
def getImageData(exposure):
"""Get the image data.
Parameters
----------
exposure : lsst.afw.image.exposure.exposure.ExposureF
Exposure object.
Returns
-------
numpy.ndarray
Image data.
"""
# Get the numpy array data based on the input object type
if isinstance(exposure, np.ndarray):
data = exposure
elif hasattr(exposure, "getMaskedImage"):
data = exposure.getMaskedImage().getImage().getArray()
elif hasattr(exposure, "getImage"):
data = exposure.getImage().getArray()
else:
data = exposure.getArray()
# Return the data in numpy array
return data
def setUp(self):
inputDir = os.path.join(ROOT, "data", "input")
self.testDir = tempfile.mkdtemp(dir=os.path.join(ROOT, 'tests'), prefix=type(self).__name__+'-')
self.butler = Butler(inputs=inputDir, outputs={"root": self.testDir, "mode": 'rw'})
