def testPickle(self): """Test pickle support. """ storageClass = StorageClass("test_pickle") datasetTypeName = "test" dimensions = self.universe.extract(("instrument", "visit")) # Un-pickling requires that storage class is registered with factory. StorageClassFactory().registerStorageClass(storageClass) datasetType = DatasetType(datasetTypeName, dimensions, storageClass) datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) self.assertIsInstance(datasetTypeOut, DatasetType) self.assertEqual(datasetType.name, datasetTypeOut.name) self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names) self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass) self.assertIsNone(datasetTypeOut.parentStorageClass) self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) self.assertFalse(datasetTypeOut.isCalibration()) datasetType = DatasetType(datasetTypeName, dimensions, storageClass, isCalibration=True) datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) self.assertIs(datasetType.isCalibration(), datasetTypeOut.isCalibration()) self.assertTrue(datasetTypeOut.isCalibration()) # And again with a composite componentStorageClass = StorageClass("pickle_component") StorageClassFactory().registerStorageClass(componentStorageClass) componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, componentStorageClass, parentStorageClass=storageClass) datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) self.assertIsInstance(datasetTypeOut, DatasetType) self.assertEqual(componentDatasetType.name, datasetTypeOut.name) self.assertEqual(componentDatasetType.dimensions.names, datasetTypeOut.dimensions.names) self.assertEqual(componentDatasetType.storageClass, datasetTypeOut.storageClass) self.assertEqual(componentDatasetType.parentStorageClass, datasetTypeOut.parentStorageClass) self.assertEqual(datasetTypeOut.parentStorageClass.name, storageClass.name) self.assertEqual(datasetTypeOut, componentDatasetType) # Now with a string and not a real storage class to test that # pickling doesn't force the StorageClass to be resolved componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, "StrangeComponent", parentStorageClass="UnknownParent") datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) self.assertEqual(datasetTypeOut, componentDatasetType) self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName) # Now with a storage class that is created by the factory factoryStorageClassClass = StorageClassFactory.makeNewStorageClass("ParentClass") factoryComponentStorageClassClass = StorageClassFactory.makeNewStorageClass("ComponentClass") componentDatasetType = DatasetType(DatasetType.nameWithComponent(datasetTypeName, "comp"), dimensions, factoryComponentStorageClassClass(), parentStorageClass=factoryStorageClassClass()) datasetTypeOut = pickle.loads(pickle.dumps(componentDatasetType)) self.assertEqual(datasetTypeOut, componentDatasetType) self.assertEqual(datasetTypeOut._parentStorageClassName, componentDatasetType._parentStorageClassName)
def setUpClass(cls): if lsst.afw.table is None: raise unittest.SkipTest("afw not available.") cls.testDir = os.path.dirname(__file__) cls.storageClassFactory = StorageClassFactory() cls.configFile = os.path.join(cls.testDir, "config/basic/butler.yaml") cls.storageClassFactory.addFromConfig(cls.configFile)
def setUpClass(cls): """Create a new butler once only.""" cls.storageClassFactory = StorageClassFactory() cls.root = tempfile.mkdtemp(dir=TESTDIR) data_ids = { "instrument": [INSTRUMENT_NAME], "detector": [0, 1, 2, 3, 4, 5], "exposure": [11, 22], } configURI = ButlerURI("resource://spherex/configs", forceDirectory=True) butlerConfig = Config(configURI.join("butler.yaml")) # in-memory db is being phased out # butlerConfig["registry", "db"] = 'sqlite:///:memory:' cls.creatorButler = makeTestRepo( cls.root, data_ids, config=butlerConfig, dimensionConfig=configURI.join("dimensions.yaml")) for formatter in FORMATTERS: datasetTypeName, storageClassName = (formatter["dataset_type"], formatter["storage_class"]) storageClass = cls.storageClassFactory.getStorageClass( storageClassName) addDatasetType(cls.creatorButler, datasetTypeName, set(data_ids), storageClass)
def __init__(self, config, registry, butlerRoot=None): super().__init__(config, registry) if "root" not in self.config: raise ValueError("No root directory specified in configuration") # Name ourselves either using an explicit name or a name # derived from the (unexpanded) root if "name" in self.config: self.name = self.config["name"] else: self.name = "POSIXDatastore@{}".format(self.config["root"]) # Support repository relocation in config self.root = replaceRoot(self.config["root"], butlerRoot) if not os.path.isdir(self.root): if "create" not in self.config or not self.config["create"]: raise ValueError(f"No valid root at: {self.root}") safeMakeDir(self.root) self.locationFactory = LocationFactory(self.root) self.formatterFactory = FormatterFactory() self.storageClassFactory = StorageClassFactory() # Now associate formatters with storage classes self.formatterFactory.registerFormatters( self.config["formatters"], universe=self.registry.dimensions) # Read the file naming templates self.templates = FileTemplates(self.config["templates"], universe=self.registry.dimensions) # And read the constraints list constraintsConfig = self.config.get("constraints") self.constraints = Constraints(constraintsConfig, universe=self.registry.dimensions) # Storage of paths and formatters, keyed by dataset_id types = { "path": str, "formatter": str, "storage_class": str, "file_size": int, "checksum": str, "dataset_id": int } lengths = { "path": 256, "formatter": 128, "storage_class": 64, "checksum": 128 } self.records = DatabaseDict.fromConfig(self.config["records"], types=types, value=self.RecordTuple, key="dataset_id", lengths=lengths, registry=registry)
def __init__(self, config=None, root=REPO_ROOT): self.root = root if config is None: config = self.root self.butlerConfig = ButlerConfig(config, searchPaths=searchPaths) StorageClassFactory().addFromConfig(self.butlerConfig) # Force the configuration directory to refer to the ci_hsc root self.butlerConfig.configDir = self.root
def testConstructor2(self): """Test construction from StorageClass name. """ datasetTypeName = "test" storageClass = StorageClass("test_constructor2") StorageClassFactory().registerStorageClass(storageClass) dimensions = self.universe.extract(("instrument", "visit")) datasetType = DatasetType(datasetTypeName, dimensions, "test_constructor2") self.assertEqual(datasetType.name, datasetTypeName) self.assertEqual(datasetType.storageClass, storageClass) self.assertEqual(datasetType.dimensions, dimensions)
def _translateDatasetType(self, datasetType): if "_" in datasetType: if datasetType.endswith("_md"): return f"{datasetType[:-3]}.metadata" for component in StorageClassFactory().getStorageClass( "Exposure").components: suffix = f"_{component}" if datasetType.endswith(suffix): return "{}.{}".format(datasetType[:-len(suffix)], component) return datasetType
def setUpClass(cls): # Storage Classes are fixed for all datastores in these tests scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(scConfigFile) # Read the Datastore config so we can get the class # information (since we should not assume the constructor # name here, but rely on the configuration file itself) datastoreConfig = DatastoreConfig(cls.configFile) cls.datastoreType = doImport(datastoreConfig["cls"]) cls.universe = DimensionUniverse.fromConfig()
def testPickle(self): """Test pickle support. """ storageClass = StorageClass("test_pickle") datasetTypeName = "test" dimensions = self.universe.extract(("instrument", "visit")) # Un-pickling requires that storage class is registered with factory. StorageClassFactory().registerStorageClass(storageClass) datasetType = DatasetType(datasetTypeName, dimensions, storageClass) datasetTypeOut = pickle.loads(pickle.dumps(datasetType)) self.assertIsInstance(datasetTypeOut, DatasetType) self.assertEqual(datasetType.name, datasetTypeOut.name) self.assertEqual(datasetType.dimensions.names, datasetTypeOut.dimensions.names) self.assertEqual(datasetType.storageClass, datasetTypeOut.storageClass)
def setUpClass(cls): """Create a new butler once only.""" cls.storageClassFactory = StorageClassFactory() cls.root = tempfile.mkdtemp(dir=TESTDIR) dataIds = { "instrument": ["DummyCam"], "physical_filter": ["d-r"], "visit": [42, 43, 44], } # Ensure that we test in a directory that will include some # metacharacters subdir = "sub?#dir" butlerRoot = os.path.join(cls.root, subdir) cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) # Create dataset types used by the tests for datasetTypeName, storageClassName in ( ("calexp", "ExposureF"), ("unknown", "ExposureCompositeF"), ("testCatalog", "SourceCatalog"), ("lossless", "ExposureF"), ("uncompressed", "ExposureF"), ("lossy", "ExposureF"), ): storageClass = cls.storageClassFactory.getStorageClass( storageClassName) addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) # And some dataset types that have no dimensions for easy testing for datasetTypeName, storageClassName in ( ("ps", "PropertySet"), ("pl", "PropertyList"), ("pkg", "Packages"), ("config", "Config"), ): storageClass = cls.storageClassFactory.getStorageClass( storageClassName) addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
def setUpClass(cls): if lsst.afw is None: raise unittest.SkipTest("afw not available.") # Base classes need to know where the test directory is cls.testDir = TESTDIR # Storage Classes are fixed for all datastores in these tests scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(scConfigFile) # Read the Datastore config so we can get the class # information (since we should not assume the constructor # name here, but rely on the configuration file itself) datastoreConfig = DatastoreConfig(cls.configFile) cls.datastoreType = doImport(datastoreConfig["cls"])
def assertValidOutput(task, result): """Test that the output of a call to ``run`` conforms to its own connections. Parameters ---------- task : `lsst.pipe.base.PipelineTask` The task whose connections need validation. This is a fully-configured task object to support features such as optional outputs. result : `lsst.pipe.base.Struct` A result object produced by calling ``task.run``. Raises ------- AssertionError: Raised if ``result`` does not match what's expected from ``task's`` connections. """ connections = task.config.ConnectionsClass(config=task.config) recoveredOutputs = result.getDict() for name in connections.outputs: connection = connections.__getattribute__(name) # name try: output = recoveredOutputs[name] except KeyError: raise AssertionError(f"No such output: {name}") # multiple if connection.multiple: if not isinstance(output, collections.abc.Sequence): raise AssertionError( f"Expected {name} to be a sequence, got {output} instead.") else: # use lazy evaluation to not use StorageClassFactory unless # necessary if isinstance(output, collections.abc.Sequence) \ and not issubclass( StorageClassFactory().getStorageClass(connection.storageClass).pytype, collections.abc.Sequence): raise AssertionError( f"Expected {name} to be a single value, got {output} instead." )
def __init__(self, config, registry=None, butlerRoot=None): super().__init__(config, registry) self.storageClassFactory = StorageClassFactory() # Name ourselves with the timestamp the datastore # was created. self.name = "InMemoryDatastore@{}".format(time.time()) log.debug("Creating datastore %s", self.name) # Storage of datasets, keyed by dataset_id self.datasets = {} # Records is distinct in order to track concrete composite components # where we register multiple components for a single dataset. self.records = {} # And read the constraints list constraintsConfig = self.config.get("constraints") self.constraints = Constraints(constraintsConfig, universe=self.registry.dimensions)
def _assertAttributeMatchesConnection(obj: Any, attrName: str, connection: BaseConnection) -> None: """Test that an attribute on an object matches the specification given in a connection. Parameters ---------- obj An object expected to contain the attribute ``attrName``. attrName : `str` The name of the attribute to be tested. connection : `lsst.pipe.base.connectionTypes.BaseConnection` The connection, usually some type of output, specifying ``attrName``. Raises ------ AssertionError: Raised if ``obj.attrName`` does not match what's expected from ``connection``. """ # name try: attrValue = obj.__getattribute__(attrName) except AttributeError: raise AssertionError(f"No such attribute on {obj!r}: {attrName}") # multiple if connection.multiple: if not isinstance(attrValue, collections.abc.Sequence): raise AssertionError( f"Expected {attrName} to be a sequence, got {attrValue!r} instead." ) else: # use lazy evaluation to not use StorageClassFactory unless # necessary if isinstance(attrValue, collections.abc.Sequence) and not issubclass( StorageClassFactory().getStorageClass( connection.storageClass).pytype, collections.abc.Sequence): raise AssertionError( f"Expected {attrName} to be a single value, got {attrValue!r} instead." )
def testFactoryConfig(self): factory = StorageClassFactory() factory.addFromConfig(StorageClassConfig()) image = factory.getStorageClass("Image") imageF = factory.getStorageClass("ImageF") self.assertIsInstance(imageF, type(image)) self.assertNotEqual(imageF, image) # Check component inheritance exposure = factory.getStorageClass("Exposure") exposureF = factory.getStorageClass("ExposureF") self.assertIsInstance(exposureF, type(exposure)) self.assertIsInstance(exposure.components["image"], type(image)) self.assertNotIsInstance(exposure.components["image"], type(imageF)) self.assertIsInstance(exposureF.components["image"], type(image)) self.assertIsInstance(exposureF.components["image"], type(imageF)) self.assertIn("wcs", exposure.components) self.assertIn("wcs", exposureF.components) # Check parameters factory.addFromConfig( os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) thing1 = factory.getStorageClass("ThingOne") thing2 = factory.getStorageClass("ThingTwo") self.assertIsInstance(thing2, type(thing1)) param1 = thing1.parameters param2 = thing2.parameters self.assertIn("param3", thing2.parameters) self.assertNotIn("param3", thing1.parameters) param2.remove("param3") self.assertEqual(param1, param2) # Check that we can't have a new StorageClass that does not # inherit from StorageClass with self.assertRaises(ValueError): factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) sc = factory.makeNewStorageClass("ClassName") self.assertIsInstance(sc(), StorageClass)
def testRegistry(self): """Check that storage classes can be created on the fly and stored in a registry.""" className = "TestImage" factory = StorageClassFactory() newclass = StorageClass(className, pytype=PythonType) factory.registerStorageClass(newclass) sc = factory.getStorageClass(className) self.assertIsInstance(sc, StorageClass) self.assertEqual(sc.name, className) self.assertFalse(sc.components) self.assertEqual(sc.pytype, PythonType) self.assertIn(sc, factory) newclass2 = StorageClass("Temporary2", pytype=str) self.assertNotIn(newclass2, factory) factory.registerStorageClass(newclass2) self.assertIn(newclass2, factory) self.assertIn("Temporary2", factory) self.assertNotIn("Temporary3", factory) self.assertNotIn({}, factory) # Make sure we can't register a storage class with the same name # but different values newclass3 = StorageClass("Temporary2", pytype=dict) with self.assertRaises(ValueError): factory.registerStorageClass(newclass3) factory._unregisterStorageClass(newclass3.name) self.assertNotIn(newclass3, factory) self.assertNotIn(newclass3.name, factory) factory.registerStorageClass(newclass3) self.assertIn(newclass3, factory) self.assertIn(newclass3.name, factory) # Check you can silently insert something that is already there factory.registerStorageClass(newclass3)
def getDatasetType(cls): """Return the DatasetType of the Datasets ingested by this Task. """ return DatasetType("raw", ("Instrument", "Detector", "Exposure"), StorageClassFactory().getStorageClass("Exposure"))
class QueryDimensionRecordsTest(unittest.TestCase, ButlerTestHelper): mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDimensionRecords" configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") storageClassFactory = StorageClassFactory() expectedColumnNames = ("instrument", "id", "physical_filter", "visit_system", "name", "day_obs", "exposure_time", "target_name", "observation_reason", "science_program", "zenith_angle", "region", "timespan [2]") def setUp(self): self.root = makeTestTempDir(TESTDIR) self.testRepo = MetricTestRepo(self.root, configFile=os.path.join( TESTDIR, "config/basic/butler.yaml")) self.runner = LogCliRunner() def tearDown(self): removeTestTempDir(self.root) def testBasic(self): result = self.runner.invoke( butlerCli, ["query-dimension-records", self.root, "visit"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array( (("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None", "None", "None", "None", "None", "None", "None", "None .. None"), ("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None", "None", "None", "None", "None", "None", "None", "None .. None"))) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected) def testWhere(self): result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--where", "instrument='DummyCamComp' AND visit.name='fourtwentythree'" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array((("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None", "None", "None", "None", "None", "None", "None", "None .. None"), )) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected) def testCollection(self): butler = Butler(self.root, run="foo") # try replacing the testRepo's butler with the one with the "foo" run. self.testRepo.butler = butler self.testRepo.butler.registry.insertDimensionData( "visit", { "instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r", "visit_system": 1 }) self.testRepo.addDataset(dataId={ "instrument": "DummyCamComp", "visit": 425 }, run="foo") # verify getting records from the "ingest/run" collection result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--collections", "ingest/run", "--datasets", "test_metric_comp" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array( (("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None", "None", "None", "None", "None", "None", "None", "None .. None"), ("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None", "None", "None", "None", "None", "None", "None", "None .. None"))) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected) # verify getting records from the "foo" collection result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--collections", "foo", "--datasets", "test_metric_comp" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array((("DummyCamComp", "425", "d-r", "1", "fourtwentyfive", "None", "None", "None", "None", "None", "None", "None", "None .. None"), )) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected)
def setUpClass(cls): if lsst.afw is None: raise unittest.SkipTest("afw not available.") cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(cls.configFile)
def checkInstrumentWithRegistry(self, cls, testRaw): Butler.makeRepo(self.root) butler = Butler(self.root, run="tests") instrument = cls() scFactory = StorageClassFactory() # Check instrument class and metadata translator agree on # instrument name, using readRawFitsHeader to read the metadata. filename = os.path.join(DATAROOT, testRaw) md = readRawFitsHeader(filename, translator_class=cls.translatorClass) obsInfo = ObservationInfo(md, translator_class=cls.translatorClass, filename=filename) self.assertEqual(instrument.getName(), obsInfo.instrument) # Add Instrument, Detector, and PhysicalFilter entries to the # Butler Registry. instrument.register(butler.registry) # Define a DatasetType for the cameraGeom.Camera, which can be # accessed just by identifying its Instrument. # A real-world Camera DatasetType should be identified by a # validity range as well. cameraDatasetType = DatasetType( "camera", dimensions=["instrument"], storageClass=scFactory.getStorageClass("Camera"), universe=butler.registry.dimensions) butler.registry.registerDatasetType(cameraDatasetType) # Define a DatasetType for cameraGeom.Detectors, which can be # accessed by identifying its Instrument and (Butler) Detector. # A real-world Detector DatasetType probably doesn't need to exist, # as it would just duplicate information in the Camera, and # reading a full Camera just to get a single Detector should be # plenty efficient. detectorDatasetType = DatasetType( "detector", dimensions=["instrument", "detector"], storageClass=scFactory.getStorageClass("Detector"), universe=butler.registry.dimensions) butler.registry.registerDatasetType(detectorDatasetType) # Put and get the Camera. dataId = dict(instrument=instrument.instrument) butler.put(instrument.getCamera(), "camera", dataId=dataId) camera = butler.get("camera", dataId) # Full camera comparisons are *slow*; just compare names. self.assertEqual(instrument.getCamera().getName(), camera.getName()) # Put and get a random subset of the Detectors. allDetectors = list(instrument.getCamera()) numDetectors = min(3, len(allDetectors)) someDetectors = [ allDetectors[i] for i in self.rng.choice( len(allDetectors), size=numDetectors, replace=False) ] for cameraGeomDetector in someDetectors: # Right now we only support integer detector IDs in data IDs; # support for detector names and groups (i.e. rafts) is # definitely planned but not yet implemented. dataId = dict(instrument=instrument.instrument, detector=cameraGeomDetector.getId()) butler.put(cameraGeomDetector, "detector", dataId=dataId) cameraGeomDetector2 = butler.get("detector", dataId=dataId) # Full detector comparisons are *slow*; just compare names and # serials. self.assertEqual(cameraGeomDetector.getName(), cameraGeomDetector2.getName()) self.assertEqual(cameraGeomDetector.getSerial(), cameraGeomDetector2.getSerial())
def setUpClass(cls): cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(cls.configFile)
def setUpClass(cls): cls.testDir = os.path.dirname(__file__) cls.storageClassFactory = StorageClassFactory() cls.configFile = os.path.join(cls.testDir, "config/basic/butler.yaml") cls.storageClassFactory.addFromConfig(cls.configFile)
class QueryDatasetsTest(unittest.TestCase, ButlerTestHelper): mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDatasets" configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") storageClassFactory = StorageClassFactory() @staticmethod def _queryDatasets(repo, glob=(), collections=(), where="", find_first=False, show_uri=False): return script.queryDatasets(repo, glob, collections, where, find_first, show_uri) def setUp(self): self.root = makeTestTempDir(TESTDIR) self.testRepo = MetricTestRepo(self.root, configFile=os.path.join(TESTDIR, "config/basic/butler.yaml")) def tearDown(self): removeTestTempDir(self.root) def testShowURI(self): """Test for expected output with show_uri=True.""" tables = self._queryDatasets(repo=self.root, show_uri=True) expectedTables = ( AstropyTable(array(( ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array(( ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array(( ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables) def testNoShowURI(self): """Test for expected output without show_uri (default is False).""" tables = self._queryDatasets(repo=self.root) expectedTables = ( AstropyTable(array(( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ), ) self.assertAstropyTablesEqual(tables, expectedTables) def testWhere(self): """Test using the where clause to reduce the number of rows returned. """ tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") expectedTables = ( AstropyTable(array( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), ), ) self.assertAstropyTablesEqual(tables, expectedTables) def testGlobDatasetType(self): """Test specifying dataset type.""" # Create and register an additional DatasetType self.testRepo.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r", "visit_system": 1}) datasetType = addDatasetType(self.testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp") self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) # verify the new dataset type increases the number of tables found: tables = self._queryDatasets(repo=self.root) expectedTables = ( AstropyTable(array(( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ), AstropyTable(array(( ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ) ) self.assertAstropyTablesEqual(tables, expectedTables) def testFindFirstAndCollections(self): """Test the find-first option, and the collections option, since it is required for find-first.""" # Add a new run, and add a dataset to shadow an existing dataset. self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) # Verify that without find-first, duplicate datasets are returned tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True) expectedTables = ( AstropyTable(array( ( ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables) # Verify that with find first the duplicate dataset is eliminated and # the more recent dataset is returned. tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True) expectedTables = ( AstropyTable(array( ( ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables)
def setUpClass(cls): for name in ("SCA", "SCB", "SCC", "SCX", "SCY"): StorageClassFactory().registerStorageClass(StorageClass(name))
class CliRetrieveArtifactsTest(unittest.TestCase, ButlerTestHelper): configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") storageClassFactory = StorageClassFactory() def setUp(self): self.root = makeTestTempDir(TESTDIR) self.testRepo = MetricTestRepo(self.root, configFile=self.configFile) def tearDown(self): removeTestTempDir(self.root) @staticmethod def find_files(root: str) -> List[ButlerURI]: return list(ButlerURI.findFileResources([root])) def testRetrieveAll(self): runner = LogCliRunner() with runner.isolated_filesystem(): # When preserving the path the run will be in the directory along # with a . in the component name. When not preserving paths the # filename will have an underscore rather than dot. for counter, (preserve_path, prefix) in enumerate( (("--preserve-path", "ingest/run/test_metric_comp."), ("--no-preserve-path", "test_metric_comp_"))): destdir = f"tmp{counter}/" result = runner.invoke( cli, ["retrieve-artifacts", self.root, destdir, preserve_path]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertTrue(result.stdout.endswith(": 6\n"), f"Expected 6 got: {result.stdout}") artifacts = self.find_files(destdir) self.assertEqual(len(artifacts), 6, f"Expected 6 artifacts: {artifacts}") self.assertIn(f"{destdir}{prefix}", str(artifacts[1])) def testRetrieveSubset(self): runner = LogCliRunner() with runner.isolated_filesystem(): destdir = "tmp1/" result = runner.invoke(cli, [ "retrieve-artifacts", self.root, destdir, "--where", "instrument='DummyCamComp' AND visit=423" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertTrue(result.stdout.endswith(": 3\n"), f"Expected 3 got: {result.stdout}") artifacts = self.find_files(destdir) self.assertEqual(len(artifacts), 3, f"Expected 3 artifacts: {artifacts}") def testClobber(self): runner = LogCliRunner() with runner.isolated_filesystem(): destdir = "tmp2/" result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) # Running again should fail result = runner.invoke(cli, ["retrieve-artifacts", self.root, destdir]) self.assertNotEqual(result.exit_code, 0, clickResultMsg(result)) # But with clobber should pass result = runner.invoke( cli, ["retrieve-artifacts", self.root, destdir, "--clobber"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result))