def testPickle(self):
"""Test pickle support.
"""
storageClass = StorageClass("test_pickle")
datasetTypeName = "test"
dimensions = self.universe.extract(("instrument", "visit"))
# Un-pickling requires that storage class is registered with factory.
StorageClassFactory().registerStorageClass(storageClass)
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(datasetType.name, datasetTypeOut.name)
self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
self.assertIsNone(datasetTypeOut.parentStorageClass)
self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
self.assertFalse(datasetTypeOut.isCalibration())
datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration())
self.assertTrue(datasetTypeOut.isCalibration())
# And again with a composite
componentStorageClass = StorageClass("pickle_component")
StorageClassFactory().registerStorageClass(componentStorageClass)
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, componentStorageClass,
parentStorageClass=storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(componentDatasetType.name, datasetTypeOut.name)
self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass)
self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass)
self.assertEqual(datasetTypeOut.parentStorageClass.name,
storageClass.name)
self.assertEqual(datasetTypeOut, componentDatasetType)
# Now with a string and not a real storage class to test that
# pickling doesn't force the StorageClass to be resolved
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, "StrangeComponent",
parentStorageClass="UnknownParent")
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertEqual(datasetTypeOut, componentDatasetType)
self.assertEqual(datasetTypeOut._parentStorageClassName,
componentDatasetType._parentStorageClassName)
# Now with a storage class that is created by the factory
factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass")
factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass")
componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"),
dimensions, factoryComponentStorageClassClass(),
parentStorageClass=factoryStorageClassClass())
datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType))
self.assertEqual(datasetTypeOut, componentDatasetType)
self.assertEqual(datasetTypeOut._parentStorageClassName,
componentDatasetType._parentStorageClassName)
def setUpClass(cls):
if lsst.afw.table is None:
raise unittest.SkipTest("afw not available.")
cls.testDir = os.path.dirname(__file__)
cls.storageClassFactory = StorageClassFactory()
cls.configFile = os.path.join(cls.testDir, "config/basic/butler.yaml")
cls.storageClassFactory.addFromConfig(cls.configFile)
def setUpClass(cls):
"""Create a new butler once only."""
cls.storageClassFactory = StorageClassFactory()
cls.root = tempfile.mkdtemp(dir=TESTDIR)
data_ids = {
"instrument": [INSTRUMENT_NAME],
"detector": [0, 1, 2, 3, 4, 5],
"exposure": [11, 22],
}
configURI = ButlerURI("resource://spherex/configs",
forceDirectory=True)
butlerConfig = Config(configURI.join("butler.yaml"))
# in-memory db is being phased out
# butlerConfig["registry", "db"] = 'sqlite:///:memory:'
cls.creatorButler = makeTestRepo(
cls.root,
data_ids,
config=butlerConfig,
dimensionConfig=configURI.join("dimensions.yaml"))
for formatter in FORMATTERS:
datasetTypeName, storageClassName = (formatter["dataset_type"],
formatter["storage_class"])
storageClass = cls.storageClassFactory.getStorageClass(
storageClassName)
addDatasetType(cls.creatorButler, datasetTypeName, set(data_ids),
storageClass)
def __init__(self, config, registry, butlerRoot=None):
super().__init__(config, registry)
if "root" not in self.config:
raise ValueError("No root directory specified in configuration")
# Name ourselves either using an explicit name or a name
# derived from the (unexpanded) root
if "name" in self.config:
self.name = self.config["name"]
else:
self.name = "POSIXDatastore@{}".format(self.config["root"])
# Support repository relocation in config
self.root = replaceRoot(self.config["root"], butlerRoot)
if not os.path.isdir(self.root):
if "create" not in self.config or not self.config["create"]:
raise ValueError(f"No valid root at: {self.root}")
safeMakeDir(self.root)
self.locationFactory = LocationFactory(self.root)
self.formatterFactory = FormatterFactory()
self.storageClassFactory = StorageClassFactory()
# Now associate formatters with storage classes
self.formatterFactory.registerFormatters(
self.config["formatters"], universe=self.registry.dimensions)
# Read the file naming templates
self.templates = FileTemplates(self.config["templates"],
universe=self.registry.dimensions)
# And read the constraints list
constraintsConfig = self.config.get("constraints")
self.constraints = Constraints(constraintsConfig,
universe=self.registry.dimensions)
# Storage of paths and formatters, keyed by dataset_id
types = {
"path": str,
"formatter": str,
"storage_class": str,
"file_size": int,
"checksum": str,
"dataset_id": int
}
lengths = {
"path": 256,
"formatter": 128,
"storage_class": 64,
"checksum": 128
}
self.records = DatabaseDict.fromConfig(self.config["records"],
types=types,
value=self.RecordTuple,
key="dataset_id",
lengths=lengths,
registry=registry)
def __init__(self, config=None, root=REPO_ROOT):
self.root = root
if config is None:
config = self.root
self.butlerConfig = ButlerConfig(config, searchPaths=searchPaths)
StorageClassFactory().addFromConfig(self.butlerConfig)
# Force the configuration directory to refer to the ci_hsc root
self.butlerConfig.configDir = self.root
def testConstructor2(self):
"""Test construction from StorageClass name.
"""
datasetTypeName = "test"
storageClass = StorageClass("test_constructor2")
StorageClassFactory().registerStorageClass(storageClass)
dimensions = self.universe.extract(("instrument", "visit"))
datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2")
self.assertEqual(datasetType.name, datasetTypeName)
self.assertEqual(datasetType.storageClass, storageClass)
self.assertEqual(datasetType.dimensions, dimensions)
def _translateDatasetType(self, datasetType):
if "_" in datasetType:
if datasetType.endswith("_md"):
return f"{datasetType[:-3]}.metadata"
for component in StorageClassFactory().getStorageClass(
"Exposure").components:
suffix = f"_{component}"
if datasetType.endswith(suffix):
return "{}.{}".format(datasetType[:-len(suffix)],
component)
return datasetType
def setUpClass(cls):
# Storage Classes are fixed for all datastores in these tests
scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(scConfigFile)
# Read the Datastore config so we can get the class
# information (since we should not assume the constructor
# name here, but rely on the configuration file itself)
datastoreConfig = DatastoreConfig(cls.configFile)
cls.datastoreType = doImport(datastoreConfig["cls"])
cls.universe = DimensionUniverse.fromConfig()
def testPickle(self):
"""Test pickle support.
"""
storageClass = StorageClass("test_pickle")
datasetTypeName = "test"
dimensions = self.universe.extract(("instrument", "visit"))
# Un-pickling requires that storage class is registered with factory.
StorageClassFactory().registerStorageClass(storageClass)
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
datasetTypeOut = pickle.loads(pickle.dumps(datasetType))
self.assertIsInstance(datasetTypeOut, DatasetType)
self.assertEqual(datasetType.name, datasetTypeOut.name)
self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names)
self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
def setUpClass(cls):
"""Create a new butler once only."""
cls.storageClassFactory = StorageClassFactory()
cls.root = tempfile.mkdtemp(dir=TESTDIR)
dataIds = {
"instrument": ["DummyCam"],
"physical_filter": ["d-r"],
"visit": [42, 43, 44],
}
# Ensure that we test in a directory that will include some
# metacharacters
subdir = "sub?#dir"
butlerRoot = os.path.join(cls.root, subdir)
cls.creatorButler = makeTestRepo(butlerRoot,
dataIds,
config=Config.fromYaml(BUTLER_CONFIG))
# Create dataset types used by the tests
for datasetTypeName, storageClassName in (
("calexp", "ExposureF"),
("unknown", "ExposureCompositeF"),
("testCatalog", "SourceCatalog"),
("lossless", "ExposureF"),
("uncompressed", "ExposureF"),
("lossy", "ExposureF"),
):
storageClass = cls.storageClassFactory.getStorageClass(
storageClassName)
addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds),
storageClass)
# And some dataset types that have no dimensions for easy testing
for datasetTypeName, storageClassName in (
("ps", "PropertySet"),
("pl", "PropertyList"),
("pkg", "Packages"),
("config", "Config"),
):
storageClass = cls.storageClassFactory.getStorageClass(
storageClassName)
addDatasetType(cls.creatorButler, datasetTypeName, {},
storageClass)
def setUpClass(cls):
if lsst.afw is None:
raise unittest.SkipTest("afw not available.")
# Base classes need to know where the test directory is
cls.testDir = TESTDIR
# Storage Classes are fixed for all datastores in these tests
scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(scConfigFile)
# Read the Datastore config so we can get the class
# information (since we should not assume the constructor
# name here, but rely on the configuration file itself)
datastoreConfig = DatastoreConfig(cls.configFile)
cls.datastoreType = doImport(datastoreConfig["cls"])
def assertValidOutput(task, result):
"""Test that the output of a call to ``run`` conforms to its own
connections.
Parameters
----------
task : `lsst.pipe.base.PipelineTask`
The task whose connections need validation. This is a fully-configured
task object to support features such as optional outputs.
result : `lsst.pipe.base.Struct`
A result object produced by calling ``task.run``.
Raises
-------
AssertionError:
Raised if ``result`` does not match what's expected from ``task's``
connections.
"""
connections = task.config.ConnectionsClass(config=task.config)
recoveredOutputs = result.getDict()
for name in connections.outputs:
connection = connections.__getattribute__(name)
# name
try:
output = recoveredOutputs[name]
except KeyError:
raise AssertionError(f"No such output: {name}")
# multiple
if connection.multiple:
if not isinstance(output, collections.abc.Sequence):
raise AssertionError(
f"Expected {name} to be a sequence, got {output} instead.")
else:
# use lazy evaluation to not use StorageClassFactory unless
# necessary
if isinstance(output, collections.abc.Sequence) \
and not issubclass(
StorageClassFactory().getStorageClass(connection.storageClass).pytype,
collections.abc.Sequence):
raise AssertionError(
f"Expected {name} to be a single value, got {output} instead."
)
def __init__(self, config, registry=None, butlerRoot=None):
super().__init__(config, registry)
self.storageClassFactory = StorageClassFactory()
# Name ourselves with the timestamp the datastore
# was created.
self.name = "InMemoryDatastore@{}".format(time.time())
log.debug("Creating datastore %s", self.name)
# Storage of datasets, keyed by dataset_id
self.datasets = {}
# Records is distinct in order to track concrete composite components
# where we register multiple components for a single dataset.
self.records = {}
# And read the constraints list
constraintsConfig = self.config.get("constraints")
self.constraints = Constraints(constraintsConfig,
universe=self.registry.dimensions)
def _assertAttributeMatchesConnection(obj: Any, attrName: str,
connection: BaseConnection) -> None:
"""Test that an attribute on an object matches the specification given in
a connection.
Parameters
----------
obj
An object expected to contain the attribute ``attrName``.
attrName : `str`
The name of the attribute to be tested.
connection : `lsst.pipe.base.connectionTypes.BaseConnection`
The connection, usually some type of output, specifying ``attrName``.
Raises
------
AssertionError:
Raised if ``obj.attrName`` does not match what's expected
from ``connection``.
"""
# name
try:
attrValue = obj.__getattribute__(attrName)
except AttributeError:
raise AssertionError(f"No such attribute on {obj!r}: {attrName}")
# multiple
if connection.multiple:
if not isinstance(attrValue, collections.abc.Sequence):
raise AssertionError(
f"Expected {attrName} to be a sequence, got {attrValue!r} instead."
)
else:
# use lazy evaluation to not use StorageClassFactory unless
# necessary
if isinstance(attrValue, collections.abc.Sequence) and not issubclass(
StorageClassFactory().getStorageClass(
connection.storageClass).pytype, collections.abc.Sequence):
raise AssertionError(
f"Expected {attrName} to be a single value, got {attrValue!r} instead."
)
def testFactoryConfig(self):
factory = StorageClassFactory()
factory.addFromConfig(StorageClassConfig())
image = factory.getStorageClass("Image")
imageF = factory.getStorageClass("ImageF")
self.assertIsInstance(imageF, type(image))
self.assertNotEqual(imageF, image)
# Check component inheritance
exposure = factory.getStorageClass("Exposure")
exposureF = factory.getStorageClass("ExposureF")
self.assertIsInstance(exposureF, type(exposure))
self.assertIsInstance(exposure.components["image"], type(image))
self.assertNotIsInstance(exposure.components["image"], type(imageF))
self.assertIsInstance(exposureF.components["image"], type(image))
self.assertIsInstance(exposureF.components["image"], type(imageF))
self.assertIn("wcs", exposure.components)
self.assertIn("wcs", exposureF.components)
# Check parameters
factory.addFromConfig(
os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
thing1 = factory.getStorageClass("ThingOne")
thing2 = factory.getStorageClass("ThingTwo")
self.assertIsInstance(thing2, type(thing1))
param1 = thing1.parameters
param2 = thing2.parameters
self.assertIn("param3", thing2.parameters)
self.assertNotIn("param3", thing1.parameters)
param2.remove("param3")
self.assertEqual(param1, param2)
# Check that we can't have a new StorageClass that does not
# inherit from StorageClass
with self.assertRaises(ValueError):
factory.makeNewStorageClass("ClassName",
baseClass=StorageClassFactory)
sc = factory.makeNewStorageClass("ClassName")
self.assertIsInstance(sc(), StorageClass)
def testRegistry(self):
"""Check that storage classes can be created on the fly and stored
in a registry."""
className = "TestImage"
factory = StorageClassFactory()
newclass = StorageClass(className, pytype=PythonType)
factory.registerStorageClass(newclass)
sc = factory.getStorageClass(className)
self.assertIsInstance(sc, StorageClass)
self.assertEqual(sc.name, className)
self.assertFalse(sc.components)
self.assertEqual(sc.pytype, PythonType)
self.assertIn(sc, factory)
newclass2 = StorageClass("Temporary2", pytype=str)
self.assertNotIn(newclass2, factory)
factory.registerStorageClass(newclass2)
self.assertIn(newclass2, factory)
self.assertIn("Temporary2", factory)
self.assertNotIn("Temporary3", factory)
self.assertNotIn({}, factory)
# Make sure we can't register a storage class with the same name
# but different values
newclass3 = StorageClass("Temporary2", pytype=dict)
with self.assertRaises(ValueError):
factory.registerStorageClass(newclass3)
factory._unregisterStorageClass(newclass3.name)
self.assertNotIn(newclass3, factory)
self.assertNotIn(newclass3.name, factory)
factory.registerStorageClass(newclass3)
self.assertIn(newclass3, factory)
self.assertIn(newclass3.name, factory)
# Check you can silently insert something that is already there
factory.registerStorageClass(newclass3)
def getDatasetType(cls):
"""Return the DatasetType of the Datasets ingested by this Task.
"""
return DatasetType("raw", ("Instrument", "Detector", "Exposure"),
StorageClassFactory().getStorageClass("Exposure"))
class QueryDimensionRecordsTest(unittest.TestCase, ButlerTestHelper):
mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDimensionRecords"
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
storageClassFactory = StorageClassFactory()
expectedColumnNames = ("instrument", "id", "physical_filter",
"visit_system", "name", "day_obs", "exposure_time",
"target_name", "observation_reason",
"science_program", "zenith_angle", "region",
"timespan [2]")
def setUp(self):
self.root = makeTestTempDir(TESTDIR)
self.testRepo = MetricTestRepo(self.root,
configFile=os.path.join(
TESTDIR,
"config/basic/butler.yaml"))
self.runner = LogCliRunner()
def tearDown(self):
removeTestTempDir(self.root)
def testBasic(self):
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.root, "visit"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array(
(("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None",
"None", "None", "None", "None", "None", "None", "None .. None"),
("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None",
"None", "None", "None", "None", "None", "None", "None .. None")))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testWhere(self):
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--where",
"instrument='DummyCamComp' AND visit.name='fourtwentythree'"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "423", "d-r", "1", "fourtwentythree",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testCollection(self):
butler = Butler(self.root, run="foo")
# try replacing the testRepo's butler with the one with the "foo" run.
self.testRepo.butler = butler
self.testRepo.butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 425,
"name": "fourtwentyfive",
"physical_filter": "d-r",
"visit_system": 1
})
self.testRepo.addDataset(dataId={
"instrument": "DummyCamComp",
"visit": 425
},
run="foo")
# verify getting records from the "ingest/run" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"ingest/run", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array(
(("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None",
"None", "None", "None", "None", "None", "None", "None .. None"),
("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None",
"None", "None", "None", "None", "None", "None", "None .. None")))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
# verify getting records from the "foo" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"foo", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "425", "d-r", "1", "fourtwentyfive",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def setUpClass(cls):
if lsst.afw is None:
raise unittest.SkipTest("afw not available.")
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(cls.configFile)
def checkInstrumentWithRegistry(self, cls, testRaw):
Butler.makeRepo(self.root)
butler = Butler(self.root, run="tests")
instrument = cls()
scFactory = StorageClassFactory()
# Check instrument class and metadata translator agree on
# instrument name, using readRawFitsHeader to read the metadata.
filename = os.path.join(DATAROOT, testRaw)
md = readRawFitsHeader(filename, translator_class=cls.translatorClass)
obsInfo = ObservationInfo(md,
translator_class=cls.translatorClass,
filename=filename)
self.assertEqual(instrument.getName(), obsInfo.instrument)
# Add Instrument, Detector, and PhysicalFilter entries to the
# Butler Registry.
instrument.register(butler.registry)
# Define a DatasetType for the cameraGeom.Camera, which can be
# accessed just by identifying its Instrument.
# A real-world Camera DatasetType should be identified by a
# validity range as well.
cameraDatasetType = DatasetType(
"camera",
dimensions=["instrument"],
storageClass=scFactory.getStorageClass("Camera"),
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(cameraDatasetType)
# Define a DatasetType for cameraGeom.Detectors, which can be
# accessed by identifying its Instrument and (Butler) Detector.
# A real-world Detector DatasetType probably doesn't need to exist,
# as it would just duplicate information in the Camera, and
# reading a full Camera just to get a single Detector should be
# plenty efficient.
detectorDatasetType = DatasetType(
"detector",
dimensions=["instrument", "detector"],
storageClass=scFactory.getStorageClass("Detector"),
universe=butler.registry.dimensions)
butler.registry.registerDatasetType(detectorDatasetType)
# Put and get the Camera.
dataId = dict(instrument=instrument.instrument)
butler.put(instrument.getCamera(), "camera", dataId=dataId)
camera = butler.get("camera", dataId)
# Full camera comparisons are *slow*; just compare names.
self.assertEqual(instrument.getCamera().getName(), camera.getName())
# Put and get a random subset of the Detectors.
allDetectors = list(instrument.getCamera())
numDetectors = min(3, len(allDetectors))
someDetectors = [
allDetectors[i] for i in self.rng.choice(
len(allDetectors), size=numDetectors, replace=False)
]
for cameraGeomDetector in someDetectors:
# Right now we only support integer detector IDs in data IDs;
# support for detector names and groups (i.e. rafts) is
# definitely planned but not yet implemented.
dataId = dict(instrument=instrument.instrument,
detector=cameraGeomDetector.getId())
butler.put(cameraGeomDetector, "detector", dataId=dataId)
cameraGeomDetector2 = butler.get("detector", dataId=dataId)
# Full detector comparisons are *slow*; just compare names and
# serials.
self.assertEqual(cameraGeomDetector.getName(),
cameraGeomDetector2.getName())
self.assertEqual(cameraGeomDetector.getSerial(),
cameraGeomDetector2.getSerial())
def setUpClass(cls):
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(cls.configFile)
def setUpClass(cls):
cls.testDir = os.path.dirname(__file__)
cls.storageClassFactory = StorageClassFactory()
cls.configFile = os.path.join(cls.testDir, "config/basic/butler.yaml")
cls.storageClassFactory.addFromConfig(cls.configFile)
class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper):
mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDatasets"
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
storageClassFactory = StorageClassFactory()
@staticmethod
def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False):
return script.queryDatasets(repo, glob, collections, where, find_first, show_uri)
def setUp(self):
self.root = makeTestTempDir(TESTDIR)
self.testRepo = MetricTestRepo(self.root,
configFile=os.path.join(TESTDIR, "config/basic/butler.yaml"))
def tearDown(self):
removeTestTempDir(self.root)
def testShowURI(self):
"""Test for expected output with show_uri=True."""
tables = self._queryDatasets(repo=self.root, show_uri=True)
expectedTables = (
AstropyTable(array((
("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array((
("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array((
("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testNoShowURI(self):
"""Test for expected output without show_uri (default is False)."""
tables = self._queryDatasets(repo=self.root)
expectedTables = (
AstropyTable(array((
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testWhere(self):
"""Test using the where clause to reduce the number of rows returned.
"""
tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
expectedTables = (
AstropyTable(array(
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testGlobDatasetType(self):
"""Test specifying dataset type."""
# Create and register an additional DatasetType
self.testRepo.butler.registry.insertDimensionData("visit",
{"instrument": "DummyCamComp", "id": 425,
"name": "fourtwentyfive", "physical_filter": "d-r",
"visit_system": 1})
datasetType = addDatasetType(self.testRepo.butler,
"alt_test_metric_comp",
("instrument", "visit"),
"StructuredCompositeReadComp")
self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
# verify the new dataset type increases the number of tables found:
tables = self._queryDatasets(repo=self.root)
expectedTables = (
AstropyTable(array((
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
),
AstropyTable(array((
("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
)
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testFindFirstAndCollections(self):
"""Test the find-first option, and the collections option, since it
is required for find-first."""
# Add a new run, and add a dataset to shadow an existing dataset.
self.testRepo.addDataset(run="foo",
dataId={"instrument": "DummyCamComp", "visit": 424})
# Verify that without find-first, duplicate datasets are returned
tables = self._queryDatasets(repo=self.root,
collections=["foo", "ingest/run"],
show_uri=True)
expectedTables = (
AstropyTable(array(
(
("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
# Verify that with find first the duplicate dataset is eliminated and
# the more recent dataset is returned.
tables = self._queryDatasets(repo=self.root,
collections=["foo", "ingest/run"],
show_uri=True,
find_first=True)
expectedTables = (
AstropyTable(array(
(
("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def setUpClass(cls):
for name in ("SCA", "SCB", "SCC", "SCX", "SCY"):
StorageClassFactory().registerStorageClass(StorageClass(name))
class CliRetrieveArtifactsTest(unittest.TestCase, ButlerTestHelper):
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
storageClassFactory = StorageClassFactory()
def setUp(self):
self.root = makeTestTempDir(TESTDIR)
self.testRepo = MetricTestRepo(self.root, configFile=self.configFile)
def tearDown(self):
removeTestTempDir(self.root)
@staticmethod
def find_files(root: str) -> List[ButlerURI]:
return list(ButlerURI.findFileResources([root]))
def testRetrieveAll(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
# When preserving the path the run will be in the directory along
# with a . in the component name. When not preserving paths the
# filename will have an underscore rather than dot.
for counter, (preserve_path, prefix) in enumerate(
(("--preserve-path", "ingest/run/test_metric_comp."),
("--no-preserve-path", "test_metric_comp_"))):
destdir = f"tmp{counter}/"
result = runner.invoke(
cli,
["retrieve-artifacts", self.root, destdir, preserve_path])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 6\n"),
f"Expected 6 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 6,
f"Expected 6 artifacts: {artifacts}")
self.assertIn(f"{destdir}{prefix}", str(artifacts[1]))
def testRetrieveSubset(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp1/"
result = runner.invoke(cli, [
"retrieve-artifacts", self.root, destdir, "--where",
"instrument='DummyCamComp' AND visit=423"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 3\n"),
f"Expected 3 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 3,
f"Expected 3 artifacts: {artifacts}")
def testClobber(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp2/"
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Running again should fail
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
# But with clobber should pass
result = runner.invoke(
cli, ["retrieve-artifacts", self.root, destdir, "--clobber"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))