def run(root: str,
*,
tracts: List[int],
filters: List[str],
create: bool = False,
clobber: bool = False,
continue_: bool = False,
reruns: List[Rerun]):
if create:
if continue_:
raise ValueError("Cannot continue if create is True.")
if os.path.exists(root):
if clobber:
shutil.rmtree(root)
else:
raise ValueError("Repo exists and --clobber=False.")
Butler.makeRepo(root)
if reruns and set(filters) != set("grizy"):
raise ValueError(
"All filters must be included if reruns are converted.")
butler = Butler(root, run="HSC/raw/all")
task = makeTask(butler, continue_=continue_, reruns=reruns)
task.run(root=GEN2_RAW_ROOT,
reruns=reruns,
calibs=([CalibRepo(path="CALIB", labels=("gen2", "defaults"))]
if not continue_ else []),
visits=makeVisitList(tracts, filters))
if not continue_:
task.log.info("Ingesting y-band stray light data.")
task.instrument.ingestStrayLightData(Butler(root, writeable=True),
directory=os.path.join(
GEN2_RAW_ROOT, "CALIB",
"STRAY_LIGHT"),
transfer=task.config.transfer,
labels=("gen2", "defaults"))
def testWriteCuratedCalibrations(self):
"""Test that we can ingest the curated calibrations, and read them
with `loadCamera` both before and after.
"""
if self.curatedCalibrationDatasetTypes is None:
raise unittest.SkipTest(
"Class requests disabling of writeCuratedCalibrations test")
butler = Butler(self.root, writeable=False)
collection = self.instrumentClass.makeCalibrationCollectionName()
# Trying to load a camera with a data ID not known to the registry
# is an error, because we can't get any temporal information.
with self.assertRaises(LookupError):
lsst.obs.base.loadCamera(butler, {"exposure": 0},
collections=collection)
# Ingest raws in order to get some exposure records.
self._ingestRaws(transfer="auto")
# Load camera should returned an unversioned camera because there's
# nothing in the repo.
camera, isVersioned = lsst.obs.base.loadCamera(butler,
self.dataIds[0],
collections=collection)
self.assertFalse(isVersioned)
self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
self._writeCuratedCalibrations()
# Make a new butler instance to make sure we don't have any stale
# caches (e.g. of DatasetTypes). Note that we didn't give
# _writeCuratedCalibrations the butler instance we had, because it's
# trying to test the CLI interface anyway.
butler = Butler(self.root, writeable=False)
for datasetTypeName in self.curatedCalibrationDatasetTypes:
with self.subTest(dtype=datasetTypeName):
found = list(
butler.registry.queryDatasetAssociations(
datasetTypeName,
collections=collection,
))
self.assertGreater(len(found), 0,
f"Checking {datasetTypeName}")
# Load camera should returned the versioned camera from the repo.
camera, isVersioned = lsst.obs.base.loadCamera(butler,
self.dataIds[0],
collections=collection)
self.assertTrue(isVersioned)
self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
def testConstructor(self):
"""Independent test of constructor.
"""
butler = Butler(self.tmpConfigFile, run="ingest")
self.assertIsInstance(butler, Butler)
collections = butler.registry.getAllCollections()
self.assertEqual(collections, set())
butler2 = Butler(butler=butler, collection="other")
self.assertEqual(butler2.collection, "other")
self.assertIsNone(butler2.run)
self.assertIs(butler.registry, butler2.registry)
self.assertIs(butler.datastore, butler2.datastore)
def runPipeline(self, graph, taskFactory, args):
"""Execute complete QuantumGraph.
Parameters
----------
graph : `QuantumGraph`
Execution graph.
taskFactory : `~lsst.pipe.base.TaskFactory`
Task factory.
args : `argparse.Namespace`
Parsed command line
"""
# If default output collection is given then use it to override
# butler-configured one.
run = args.output.get("", None)
# make butler instance
butler = Butler(config=args.butler_config, run=run)
# at this point we require that output collection was defined
if not butler.run:
raise ValueError("no output collection defined in data butler")
preExecInit = PreExecInit(butler)
preExecInit.initialize(
graph,
taskFactory,
registerDatasetTypes=args.register_dataset_types,
saveInitOutputs=not args.skip_init_writes,
updateOutputCollection=True)
if not args.init_only:
executor = MPGraphExecutor(numProc=args.processes,
timeout=self.MP_TIMEOUT)
executor.execute(graph, butler, taskFactory)
def make_test_butler(root, data_ids):
"""Create an empty repository with default configuration.
Parameters
----------
root : `str`
The location of the root directory for the repository.
data_ids : `dict` [`str`, `iterable` [`dict`]]
A dictionary keyed by the dimensions used in the test. Each value
is a dictionary of fields and values for that dimension. See
:file:`daf/butler/config/dimensions.yaml` for required fields,
listed as "keys" and "requires" under each dimension's entry.
Returns
-------
butler : `lsst.daf.butler.Butler`
A Butler referring to the new repository.
"""
# TODO: takes 5 seconds to run; split up into class-level Butler
# with test-level runs after DM-21246
Butler.makeRepo(root)
butler = Butler(root, run="test")
for dimension, values in data_ids.items():
butler.registry.insertDimensionData(dimension, *values)
return butler
def testMatplotlibFormatter(self):
butler = Butler(self.root, run="testrun")
datasetType = DatasetType("test_plot", [],
"Plot",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Does not have to be a random image
pyplot.imshow([
self.rng.sample(range(50), 10),
self.rng.sample(range(50), 10),
self.rng.sample(range(50), 10),
])
ref = butler.put(pyplot.gcf(), datasetType)
uri = butler.getURI(ref)
# The test after this will not work if we don't have local file
self.assertEqual(uri.scheme, "file", "Testing returned URI: {uri}")
with tempfile.NamedTemporaryFile(suffix=".png") as file:
pyplot.gcf().savefig(file.name)
self.assertTrue(filecmp.cmp(uri.path, file.name, shallow=True))
self.assertTrue(butler.datasetExists(ref))
with self.assertRaises(ValueError):
butler.get(ref)
butler.pruneDatasets([ref], unstore=True, purge=True)
with self.assertRaises(LookupError):
butler.datasetExists(ref)
def put_values(repo, visit, detector, instrument, out_collection,
ra=None, dec=None, size=None, filename=None):
butler = Butler(repo, writeable=True, run=out_collection)
# This doesn't strictly need to be done every time,
# but doesn't seem to hurt if the
# dataset type already exists
position_dataset_type = DatasetType('cutout_positions', dimensions=['visit', 'detector', 'instrument'],
universe=butler.registry.dimensions,
storageClass='AstropyQTable')
butler.registry.registerDatasetType(position_dataset_type)
if filename:
poslist = numpy.genfromtxt(filename, dtype=None, delimiter=',')
else:
poslist = [(ra, dec, size), ]
ident = []
pos = []
size = []
for i, rec in enumerate(poslist):
pt = SkyCoord(rec[0], rec[1], frame='icrs', unit=u.deg)
pos.append(pt)
ident.append(i*u.dimensionless_unscaled)
size.append(float(rec[2])*u.dimensionless_unscaled)
out_table = QTable([ident, pos, size], names=['id', 'position', 'size'])
butler.put(out_table, 'cutout_positions', visit=visit, detector=detector, instrument=instrument)
def registerDcrSubfilters(repo, num_subfilters, band_names):
"""Construct a set of subfilters for chromatic modeling and add them to a
registry.
Parameters
----------
repo : `str`
URI to the location to read the repo.
num_subfilters : `int`
The number of subfilters to add.
band_names : `list` [`str`]
The filter band names to add.
Returns
-------
insertResults : ``InsertResults``
A class that contains the results of the subfilters that were inserted
or already exist in each filter band, that has a __str__ method so it
can be easily printed to the CLI output.
"""
butler = Butler(repo, writeable=True)
results = InsertResults()
for filterName in band_names:
try:
with butler.registry.transaction():
for sub in range(num_subfilters):
butler.registry.insertDimensionData("subfilter", {"band": filterName, "subfilter": sub})
results.add(filterName, sub, True)
except IntegrityError:
records = butler.registry.queryDimensionRecords("subfilter", dataId={"band": filterName})
for record in records:
results.add(filterName, record.id, False)
return results
def makeTestCollection(repo: Butler, uniqueId: Optional[str] = None) -> Butler:
"""Create a read/write Butler to a fresh collection.
Parameters
----------
repo : `lsst.daf.butler.Butler`
A previously existing Butler to a repository, such as that returned by
`~lsst.daf.butler.Butler.makeRepo` or `makeTestRepo`.
uniqueId : `str`, optional
A collection ID guaranteed by external code to be unique across all
calls to ``makeTestCollection`` for the same repository.
Returns
-------
butler : `lsst.daf.butler.Butler`
A Butler referring to a new collection in the repository at ``root``.
The collection is (almost) guaranteed to be new.
Notes
-----
This function creates a single run collection that does not necessarily
conform to any repository conventions. It is only suitable for creating an
isolated test area, and not for repositories intended for real data
processing or analysis.
"""
if not uniqueId:
# Create a "random" collection name
# Speed matters more than cryptographic guarantees
uniqueId = str(random.randrange(1_000_000_000))
collection = "test_" + uniqueId
return Butler(butler=repo, run=collection)
def testStrayLightIngest(self):
"""Ingested stray light files."""
butler = Butler(self.root, run=self.outputRun)
straylightDir = os.path.join(testDataDirectory, "hsc", "straylight")
instrument = self.instrumentClass()
# This will warn about lots of missing files
with self.assertLogs(level="WARNING") as cm:
instrument.ingestStrayLightData(butler,
straylightDir,
transfer="auto")
collection = self.instrumentClass.makeCalibrationCollectionName()
datasets = list(
butler.registry.queryDatasetAssociations("yBackground",
collections=collection))
# Should have at least one dataset and dataset+warnings = 112
self.assertGreaterEqual(len(datasets), 1)
self.assertEqual(
len(datasets) + len(cm.output), len(instrument.getCamera()))
# Ensure that we can read the first stray light file
strayLight = butler.getDirect(datasets[0].ref)
self.assertIsInstance(strayLight, SubaruStrayLightData)
def makeTestCollection(repo: Butler) -> Butler:
"""Create a read/write Butler to a fresh collection.
Parameters
----------
repo : `lsst.daf.butler.Butler`
A previously existing Butler to a repository, such as that returned by
`~lsst.daf.butler.Butler.makeRepo` or `makeTestRepo`.
Returns
-------
butler : `lsst.daf.butler.Butler`
A Butler referring to a new collection in the repository at ``root``.
The collection is (almost) guaranteed to be new.
Notes
-----
This function creates a single run collection that does not necessarily
conform to any repository conventions. It is only suitable for creating an
isolated test area, and not for repositories intended for real data
processing or analysis.
"""
# Create a "random" collection name
# Speed matters more than cryptographic guarantees
collection = "test" + str(random.randrange(1_000_000_000))
return Butler(butler=repo, run=collection)
def testMatplotlibFormatter(self):
butler = Butler(self.root, run="testrun")
datasetType = DatasetType("test_plot", [], "Plot",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
# Does not have to be a random image
pyplot.imshow([self.rng.sample(range(50), 10),
self.rng.sample(range(50), 10),
self.rng.sample(range(50), 10),
])
ref = butler.put(pyplot.gcf(), datasetType)
uri = butler.getURI(ref)
# Following test needs a local file
with uri.as_local() as local:
with tempfile.NamedTemporaryFile(suffix=".png") as file:
pyplot.gcf().savefig(file.name)
self.assertTrue(
filecmp.cmp(
local.ospath,
file.name,
shallow=True
)
)
self.assertTrue(butler.datasetExists(ref))
with self.assertRaises(ValueError):
butler.get(ref)
butler.pruneDatasets([ref], unstore=True, purge=True)
with self.assertRaises(LookupError):
butler.datasetExists(ref)
def writeCuratedCalibrations(repo, instrument, collection, labels):
"""Add an instrument's curated calibrations to the data repository.
Parameters
----------
repo : `str`
URI to the location to create the repo.
instrument : `str`
The name or the fully qualified class name of an instrument.
collection : `str` or `None`
The path to the collection that assocaites datasets with validity
ranges.
Can be `None` in which case the collection name will be determined
automatically.
labels : `Sequence` [ `str` ]
Extra strings to include in the names of collections that datasets are
inserted directly into, and if ``collection`` is `None`, the automatic
calibration collection name as well.
Raises
------
RuntimeError
Raised if the instrument can not be imported, instantiated, or obtained
from the registry.
TypeError
Raised if the instrument is not a subclass of
`lsst.obs.base.Instrument`.
"""
butler = Butler(repo, writeable=True)
instr = getInstrument(instrument, butler.registry)
instr.writeCuratedCalibrations(butler,
collection=collection,
labels=labels)
def registerSkymap(repo, config, config_file):
"""Make and register a SkyMap in a butler repository.
Parameters
----------
repo : `str`
URI to the location of the butler repository.
config : `dict` [`str`, `str`] or `None`
Key-value pairs to apply as overrides to the ingest config.
config_file : `str` or `None`
Path to a config overrides file.
Raises
------
RuntimeError
If a config overrides file is given and does not exist.
"""
skyMapConfig = MakeSkyMapConfig()
if config_file:
skyMapConfig.load(config_file)
if config:
skyMapConfig.update(**config)
butler = Butler(repo, writeable=True)
makeSkyMap(butler, skyMapConfig)
def makeSimpleButler(root: str,
run: str = "test",
inMemory: bool = True) -> Butler:
"""Create new data butler instance.
Parameters
----------
root : `str`
Path or URI to the root location of the new repository.
run : `str`, optional
Run collection name.
inMemory : `bool`, optional
If true make in-memory repository.
Returns
-------
butler : `~lsst.daf.butler.Butler`
Data butler instance.
"""
root_path = ResourcePath(root, forceDirectory=True)
if not root_path.isLocal:
raise ValueError(f"Only works with local root not {root_path}")
config = Config()
if not inMemory:
config["registry", "db"] = f"sqlite:///{root_path.ospath}/gen3.sqlite"
config[
"datastore",
"cls"] = "lsst.daf.butler.datastores.fileDatastore.FileDatastore"
repo = butlerTests.makeTestRepo(str(root_path), {}, config=config)
butler = Butler(butler=repo, run=run)
return butler
def testImportExport(self):
# Run put/get tests just to create and populate a repo.
storageClass = self.storageClassFactory.getStorageClass(
"StructuredDataNoComponents")
exportButler = self.runPutGetTest(storageClass, "test_metric")
# Test that the repo actually has at least one dataset.
datasets = list(
exportButler.registry.queryDatasets(..., collections=...))
self.assertGreater(len(datasets), 0)
# Export those datasets. We used TemporaryDirectory because there
# doesn't seem to be a way to get the filename (as opposed to the file
# object) from any of tempfile's temporary-file context managers.
with tempfile.TemporaryDirectory() as exportDir:
# TODO: When PosixDatastore supports transfer-on-exist, add tests
# for that.
exportFile = os.path.join(exportDir, "exports.yaml")
with exportButler.export(filename=exportFile) as export:
export.saveDatasets(datasets)
self.assertTrue(os.path.exists(exportFile))
with tempfile.TemporaryDirectory() as importDir:
Butler.makeRepo(importDir, config=Config(self.configFile))
importButler = Butler(importDir, run="ingest")
importButler.import_(filename=exportFile,
directory=exportButler.datastore.root,
transfer="symlink")
for ref in datasets:
with self.subTest(ref=ref):
# Test for existence by passing in the DatasetType and
# data ID separately, to avoid lookup by dataset_id.
self.assertTrue(
importButler.datasetExists(ref.datasetType,
ref.dataId))
def register_instrument(repo: str,
instrument: List[str],
update: bool = False) -> None:
"""Add an instrument to the data repository.
Parameters
----------
repo : `str`
URI to the butler repository to register this instrument.
instrument : `list` [`str`]
The fully-qualified name of an `~lsst.pipe.base.Instrument` subclass.
update : `bool`, optional
If `True` (`False` is default), update the existing instrument and
detector records if they differ from the new ones.
Raises
------
RuntimeError
Raised if the instrument can not be imported, instantiated, or obtained
from the registry.
TypeError
Raised iff the instrument is not a subclass of
`lsst.pipe.base.Instrument`.
"""
butler = Butler(repo, writeable=True)
for string in instrument:
instrument_instance = Instrument.from_string(string, butler.registry)
instrument_instance.register(butler.registry, update=update)
def testHealSparseMapFormatter(self):
butler = Butler(self.root, run="testrun")
datasetType = DatasetType("map", [],
"HealSparseMap",
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(datasetType)
ref = butler.put(self.hspMap, datasetType)
uri = butler.getURI(ref)
self.assertEqual(uri.getExtension(), '.hsp')
# Retrieve the full map.
hspMap = butler.get('map')
self.assertTrue(np.all(hspMap._sparse_map == self.hspMap._sparse_map))
# Retrieve the coverage map
coverage = butler.get('map.coverage')
self.assertTrue(
np.all(coverage.coverage_mask == self.hspMap.coverage_mask))
# Retrieve a partial map
pixels = [0, 6]
partialMap = butler.get('map', parameters={'pixels': pixels})
self.assertTrue(
np.all(np.where(partialMap.coverage_mask)[0] == np.array(pixels)))
self.assertTrue(np.all(partialMap[0:10000] == self.hspMap[0:10000]))
self.assertTrue(
np.all(partialMap[100000:110000] == self.hspMap[100000:110000]))
# Retrieve a degraded map
degradedMapRead = butler.get('map', parameters={'degrade_nside': 512})
degradedMap = self.hspMap.degrade(512)
self.assertTrue(
np.all(degradedMapRead._sparse_map == degradedMap._sparse_map))
def testBasicPutGet(self):
butler = Butler(self.configFile)
# Create and register a DatasetType
datasetTypeName = "test_metric"
dataUnits = ("Camera", "Visit")
storageClass = self.storageClassFactory.getStorageClass(
"StructuredData")
self.registerDatasetTypes(datasetTypeName, dataUnits, storageClass,
butler.registry)
# Create and store a dataset
metric = makeExampleMetrics()
dataId = {"camera": "DummyCam", "visit": 42}
ref = butler.put(metric, datasetTypeName, dataId)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(datasetTypeName, dataId)
self.assertEqual(metric, metricOut)
# Check we can get components
self.assertGetComponents(butler, datasetTypeName, dataId,
("summary", "data", "output"), metric)
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"), "DATA"),
instrument="HSC", skymap="discrete/ci_hsc",
writeable=False, collections=["HSC/runs/ci_hsc"])
self.skip_mock()
self._tract = 0
self._patch = 69
self._bands = ['r', 'i']
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_imsim"), "DATA"),
writeable=False,
collections=["LSSTCam-imSim/runs/ci_imsim"])
schemaFile = os.path.join(getPackageDir("sdm_schemas"), 'yml',
'imsim.yaml')
with open(schemaFile, "r") as f:
self.schema = yaml.safe_load(f)['tables']
def checkRepo(self, files=None):
# We ignore `files` because there's only one raw file in
# testdata_subaru, and we know it's a science frame.
# If we ever add more, this test will need to change.
butler = Butler(self.root, collections=[self.outputRun])
expanded = butler.registry.expandDataId(self.dataIds[0])
self.assertEqual(expanded.records["exposure"].observation_type,
"science")
def testConstructor(self):
"""Independent test of constructor.
"""
butler = Butler(self.tmpConfigFile)
self.assertIsInstance(butler, Butler)
collections = butler.registry.getAllCollections()
self.assertEqual(collections, set())
def ingestPhotodiode(repo,
instrument,
locations,
regex,
output_run,
config=None,
config_file=None,
transfer="copy",
track_file_attrs=True):
"""Ingests photodiode data into the butler registry.
Parameters
----------
repo : `str`
URI to the repository.
instrument : `str`
The name or fully-qualified class name of an instrument.
locations : `list` [`str`]
Files to ingest and directories to search for files that match
``regex`` to ingest.
regex : `str`
Regex string used to find files in directories listed in locations.
output_run : `str`
The path to the location, the run, where datasets should be put.
config : `dict` [`str`, `str`] or `None`
Key-value pairs to apply as overrides to the ingest config.
config_file : `str` or `None`
Path to a config file that contains overrides to the ingest config.
transfer : `str` or None
The external data transfer type, by default "copy".
track_file_attrs : `bool`, optional
Control whether file attributes such as the size or checksum should
be tracked by the datastore. Whether this parameter is honored
depends on the specific datastore implentation.
Raises
------
Exception
Raised if operations on configuration object fail.
"""
butler = Butler(repo, writeable=True)
instr = Instrument.from_string(instrument, butler.registry)
config = PhotodiodeIngestConfig()
config.transfer = transfer
configOverrides = ConfigOverrides()
if config_file is not None:
configOverrides.addFileOverride(config_file)
if config is not None:
for name, value in config.items():
configOverrides.addValueOverride(name, value)
configOverrides.applyTo(config)
task = PhotodiodeIngestTask(butler=butler, instrument=instr, config=config)
task.run(locations,
run=output_run,
file_filter=regex,
track_file_attrs=track_file_attrs)
def testTransaction(self):
butler = Butler(self.tmpConfigFile, run="ingest")
datasetTypeName = "test_metric"
dimensions = butler.registry.dimensions.extract(
["instrument", "visit"])
dimensionEntries = (("instrument", {
"instrument": "DummyCam"
}), ("physical_filter", {
"instrument": "DummyCam",
"name": "d-r",
"abstract_filter": "R"
}), ("visit", {
"instrument": "DummyCam",
"id": 42,
"name": "fortytwo",
"physical_filter": "d-r"
}))
storageClass = self.storageClassFactory.getStorageClass(
"StructuredData")
metric = makeExampleMetrics()
dataId = {"instrument": "DummyCam", "visit": 42}
with self.assertRaises(TransactionTestError):
with butler.transaction():
# Create and register a DatasetType
datasetType = self.addDatasetType(datasetTypeName, dimensions,
storageClass,
butler.registry)
# Add needed Dimensions
for args in dimensionEntries:
butler.registry.insertDimensionData(*args)
# Store a dataset
ref = butler.put(metric, datasetTypeName, dataId)
self.assertIsInstance(ref, DatasetRef)
# Test getDirect
metricOut = butler.getDirect(ref)
self.assertEqual(metric, metricOut)
# Test get
metricOut = butler.get(datasetTypeName, dataId)
self.assertEqual(metric, metricOut)
# Check we can get components
self.assertGetComponents(butler, ref,
("summary", "data", "output"), metric)
raise TransactionTestError(
"This should roll back the entire transaction")
with self.assertRaises(KeyError):
butler.registry.getDatasetType(datasetTypeName)
with self.assertRaises(LookupError):
butler.registry.expandDataId(dataId)
# Should raise KeyError for missing DatasetType
with self.assertRaises(KeyError):
butler.get(datasetTypeName, dataId)
# Also check explicitly if Dataset entry is missing
self.assertIsNone(
butler.registry.find(butler.collection, datasetType, dataId))
# Direct retrieval should not find the file in the Datastore
with self.assertRaises(FileNotFoundError):
butler.getDirect(ref)
def make_butler(self) -> Butler:
"""Return new Butler instance on each call."""
config = Config()
config["root"] = self.root
config["registry", "db"] = f"sqlite:///{self.root}/gen3.sqlite3"
butler = Butler(Butler.makeRepo(self.root, config=config),
writeable=True)
DatastoreMock.apply(butler)
return butler
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"),
"DATA"),
writeable=False,
collections=["HSC/runs/ci_hsc"])
self.skip_mock()
self.dataId = {"instrument": "HSC", "detector": 100, "visit": 903334}
self.calexp = self.butler.get("calexp", self.dataId)
self.src = self.butler.get("src", self.dataId)
def testPickle(self):
"""Test pickle support.
"""
butler = Butler(self.tmpConfigFile)
butlerOut = pickle.loads(pickle.dumps(butler))
self.assertIsInstance(butlerOut, Butler)
self.assertEqual(butlerOut.config, butler.config)
self.assertEqual(butlerOut.collection, butler.collection)
self.assertEqual(butlerOut.run, butler.run)
def setUp(self):
self.butler = Butler(os.path.join(getPackageDir("ci_hsc_gen3"),
"DATA"),
writeable=False,
collections=["HSC/runs/ci_hsc"])
schemaFile = os.path.join(getPackageDir("sdm_schemas"), 'yml',
'hsc.yaml')
with open(schemaFile, "r") as f:
self.schema = yaml.safe_load(f)['tables']
def verifyIngest(self, files=None, cli=False, fullCheck=False):
"""
Test that RawIngestTask ingested the expected files.
Parameters
----------
files : `list` [`str`], or None
List of files to be ingested, or None to use ``self.file``
fullCheck : `bool`, optional
If `True`, read the full raw dataset and check component
consistency. If `False` check that a component can be read
but do not read the entire raw exposure.
Notes
-----
Reading all the ingested test data can be expensive. The code paths
for reading the second raw are the same as reading the first so
we do not gain anything by doing full checks of everything.
Only read full pixel data for first dataset from file.
Don't even do that if we are requested not to by the caller.
This only really affects files that contain multiple datasets.
"""
butler = Butler(self.root, run=self.outputRun)
datasets = list(
butler.registry.queryDatasets("raw", collections=self.outputRun))
self.assertEqual(len(datasets), len(self.dataIds))
# Get the URI to the first dataset and check it is inside the
# datastore
datasetUri = butler.getURI(datasets[0])
self.assertIsNotNone(datasetUri.relative_to(butler.datastore.root))
for dataId in self.dataIds:
# Check that we can read metadata from a raw
metadata = butler.get("raw.metadata", dataId)
if not fullCheck:
continue
fullCheck = False
exposure = butler.get("raw", dataId)
self.assertEqual(metadata.toDict(),
exposure.getMetadata().toDict())
# Since components follow a different code path we check that
# WCS match and also we check that at least the shape
# of the image is the same (rather than doing per-pixel equality)
wcs = butler.get("raw.wcs", dataId)
self.assertEqual(wcs, exposure.getWcs())
rawImage = butler.get("raw.image", dataId)
self.assertEqual(rawImage.getBBox(), exposure.getBBox())
# check that the filter label got the correct band
filterLabel = butler.get("raw.filterLabel", dataId)
self.assertEqual(filterLabel, self.filterLabel)
self.checkRepo(files=files)
