def testGlobDatasetType(self): """Test specifying dataset type.""" # Create and register an additional DatasetType self.testRepo.butler.registry.insertDimensionData("visit", {"instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r", "visit_system": 1}) datasetType = addDatasetType(self.testRepo.butler, "alt_test_metric_comp", ("instrument", "visit"), "StructuredCompositeReadComp") self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType) # verify the new dataset type increases the number of tables found: tables = self._queryDatasets(repo=self.root) expectedTables = ( AstropyTable(array(( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ), AstropyTable(array(( ("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ) ) self.assertAstropyTablesEqual(tables, expectedTables)
def testQueryDatasetTypes(self): self.maxDiff = None datasetName = "test" instrumentDimension = "instrument" visitDimension = "visit" storageClassName = "testDatasetType" expectedNotVerbose = AstropyTable((("test", ), ), names=("name", )) runner = LogCliRunner() with runner.isolated_filesystem(): butlerCfg = Butler.makeRepo("here") butler = Butler(butlerCfg, writeable=True) storageClass = StorageClass(storageClassName) butler.registry.storageClasses.registerStorageClass(storageClass) dimensions = butler.registry.dimensions.extract( (instrumentDimension, visitDimension)) datasetType = DatasetType(datasetName, dimensions, storageClass) butler.registry.registerDatasetType(datasetType) # check not-verbose output: result = runner.invoke(cli, ["query-dataset-types", "here"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertAstropyTablesEqual(readTable(result.output), expectedNotVerbose) # check glob output: result = runner.invoke(cli, ["query-dataset-types", "here", "t*"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertAstropyTablesEqual(readTable(result.output), expectedNotVerbose) # check verbose output: result = runner.invoke( cli, ["query-dataset-types", "here", "--verbose"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) expected = AstropyTable(array(( "test", "['band', 'instrument', 'physical_filter', 'visit_system', 'visit']", "testDatasetType")), names=("name", "dimensions", "storage class")) self.assertAstropyTablesEqual(readTable(result.output), expected) # Now remove and check that it was removed # First a non-existent one result = runner.invoke(cli, ["remove-dataset-type", "here", "unreal"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) # Now one we now has been registered result = runner.invoke( cli, ["remove-dataset-type", "here", datasetName]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) # and check that it has gone result = runner.invoke(cli, ["query-dataset-types", "here"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertIn("No results", result.output)
def testShowURI(self): """Test for expected output with show_uri=True.""" tables = self._queryDatasets(repo=self.root, show_uri=True) expectedTables = ( AstropyTable(array( (("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( (("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")) )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array(( ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml") ))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables)
def testRegisterFilters(self): """Register a few filters and verify they are added to the repo.""" with self.runner.isolated_filesystem(): result = self.runner.invoke(butlerCli, ["create", self.repo]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) result = self.runner.invoke( butlerCli, ["register-dcr-subfilters", self.repo, "3", "foo"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertIn( registerDcrSubfilters.registeredMsg.format( band="foo", subfilters="[0, 1, 2]"), result.output) result = self.runner.invoke( butlerCli, ["query-dimension-records", self.repo, "subfilter"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertAstropyTablesEqual( AstropyTable((("foo", "foo", "foo"), (0, 1, 2)), names=("band", "id")), readTable(result.output)) # Verify expected output message for registering subfilters in a # band that already has subfilters result = self.runner.invoke( butlerCli, ["register-dcr-subfilters", self.repo, "5", "foo"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertIn( registerDcrSubfilters.notRegisteredMsg.format( band="foo", subfilters="[0, 1, 2]"), result.output) # Add subfilters for two filters, one new filter and one existing. # Verify expected result messages and registry values. result = self.runner.invoke( butlerCli, ["register-dcr-subfilters", self.repo, "3", "foo", "bar"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) self.assertIn( registerDcrSubfilters.notRegisteredMsg.format( band="foo", subfilters="[0, 1, 2]"), result.output) self.assertIn( registerDcrSubfilters.registeredMsg.format( band="bar", subfilters="[0, 1, 2]"), result.output) result = self.runner.invoke( butlerCli, ["query-dimension-records", self.repo, "subfilter"]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) resultTable = readTable(result.output) resultTable.sort(["band", "id"]) self.assertAstropyTablesEqual( AstropyTable((("bar", "bar", "bar", "foo", "foo", "foo"), (0, 1, 2, 0, 1, 2)), names=("band", "id")), resultTable)
def getAstropyTable(self): """Get the table as an astropy table. Returns ------- table : `astropy.table.Table` The dataIds, sorted by spatial and temporal columns first, and then the rest of the columns, with duplicate dataIds removed. """ # Should never happen; adding a dataset should be the action that # causes a _Table to be created. if not self.dataIds: raise RuntimeError("No DataIds were provided.") dataId = next(iter(self.dataIds)) dimensions = list(dataId.full.keys()) columnNames = [str(item) for item in dimensions] # Need to hint the column types for numbers since the per-row # constructor of Table does not work this out on its own and sorting # will not work properly without. typeMap = {float: np.float64, int: np.int64} columnTypes = [ typeMap.get(type(value)) for value in dataId.full.values() ] rows = [[value for value in dataId.full.values()] for dataId in self.dataIds] table = AstropyTable(np.array(rows), names=columnNames, dtype=columnTypes) return sortAstropyTable(table, dimensions)
def testCollection(self): butler = Butler(self.root, run="foo") # try replacing the testRepo's butler with the one with the "foo" run. self.testRepo.butler = butler self.testRepo.butler.registry.insertDimensionData( "visit", { "instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r", "visit_system": 1 }) self.testRepo.addDataset(dataId={ "instrument": "DummyCamComp", "visit": 425 }, run="foo") # verify getting records from the "ingest/run" collection result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--collections", "ingest/run", "--datasets", "test_metric_comp" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array( (("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None", "None", "None", "None", "None", "None", "None", "None .. None"), ("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None", "None", "None", "None", "None", "None", "None", "None .. None"))) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected) # verify getting records from the "foo" collection result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--collections", "foo", "--datasets", "test_metric_comp" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array((("DummyCamComp", "425", "d-r", "1", "fourtwentyfive", "None", "None", "None", "None", "None", "None", "None", "None .. None"), )) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected)
def testDatasets(self): """Test getting datasets.""" res = self._queryDataIds(self.root, datasets="test_metric_comp") expected = AstropyTable(array((("R", "DummyCamComp", "d-r", 1, 423), ("R", "DummyCamComp", "d-r", 1, 424))), names=("band", "instrument", "physical_filter", "visit_system", "visit")) self.assertAstropyTablesEqual(res, expected)
def testWhere(self): result = self.runner.invoke(butlerCli, [ "query-dimension-records", self.root, "visit", "--where", "instrument='DummyCamComp' AND visit.name='fourtwentythree'" ]) self.assertEqual(result.exit_code, 0, clickResultMsg(result)) rows = array((("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None", "None", "None", "None", "None", "None", "None", "None .. None"), )) expected = AstropyTable(rows, names=self.expectedColumnNames) self.assertAstropyTablesEqual(readTable(result.output), expected)
def testWhere(self): """Test getting datasets.""" res = self._queryDataIds( self.root, dimensions=("visit", ), where="instrument='DummyCamComp' AND visit=423") expected = AstropyTable(array( (("R", "DummyCamComp", "d-r", 1, 423), )), names=("band", "instrument", "physical_filter", "visit_system", "visit")) self.assertAstropyTablesEqual(res, expected)
def getAstropyTable(self, datasetTypeName): """Get the table as an astropy table. Parameters ---------- datasetTypeName : `str` The dataset type name to show in the ``type`` column of the table. Returns ------- table : `astropy.table._Table` The table with the provided column names and rows. """ # Should never happen; adding a dataset should be the action that # causes a _Table to be created. if not self.datasetRefs: raise RuntimeError( f"No DatasetRefs were provided for dataset type {datasetTypeName}" ) refInfo = next(iter(self.datasetRefs)) dimensions = list(refInfo.datasetRef.dataId.full.keys()) columnNames = [ "type", "run", "id", *[str(item) for item in dimensions] ] # Need to hint the column types for numbers since the per-row # constructor of Table does not work this out on its own and sorting # will not work properly without. typeMap = {float: np.float64, int: np.int64} columnTypes = [ None, None, np.int64, *[ typeMap.get(type(value)) for value in refInfo.datasetRef.dataId.full.values() ] ] if refInfo.uri: columnNames.append("URI") columnTypes.append(None) rows = [] for refInfo in self.datasetRefs: row = [ datasetTypeName, refInfo.datasetRef.run, refInfo.datasetRef.id, *[value for value in refInfo.datasetRef.dataId.full.values()] ] if refInfo.uri: row.append(refInfo.uri) rows.append(row) dataset_table = AstropyTable(np.array(rows), names=columnNames, dtype=columnTypes) return sortAstropyTable(dataset_table, dimensions, ["type", "run"])
def testCollections(self): """Test getting datasets using the collections option.""" # Add a dataset in a different collection self.butler = Butler(self.root, run="foo") self.repo.butler.registry.insertDimensionData( "visit", { "instrument": "DummyCamComp", "id": 425, "name": "fourtwentyfive", "physical_filter": "d-r", "visit_system": 1 }) self.repo.addDataset(dataId={ "instrument": "DummyCamComp", "visit": 425 }, run="foo") # Verify the new dataset is not found in the "ingest/run" collection. res = self._queryDataIds(repo=self.root, dimensions=("visit", ), collections=("ingest/run", ), datasets="test_metric_comp") expected = AstropyTable(array((("R", "DummyCamComp", "d-r", 1, 423), ("R", "DummyCamComp", "d-r", 1, 424))), names=("band", "instrument", "physical_filter", "visit_system", "visit")) self.assertAstropyTablesEqual(res, expected) # Verify the new dataset is found in the "foo" collection. res = self._queryDataIds(repo=self.root, dimensions=("visit", ), collections=("foo", ), datasets="test_metric_comp") expected = AstropyTable(array( (("R", "DummyCamComp", "d-r", 1, 425), )), names=("band", "instrument", "physical_filter", "visit_system", "visit")) self.assertAstropyTablesEqual(res, expected)
def testNoShowURI(self): """Test for expected output without show_uri (default is False).""" tables = self._queryDatasets(repo=self.root) expectedTables = ( AstropyTable(array(( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"), ("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit") ), ) self.assertAstropyTablesEqual(tables, expectedTables)
def testWhere(self): """Test using the where clause to reduce the number of rows returned. """ tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423") expectedTables = ( AstropyTable(array( ("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"), ), ) self.assertAstropyTablesEqual(tables, expectedTables)
def testFindFirstAndCollections(self): """Test the find-first option, and the collections option, since it is required for find-first.""" # Add a new run, and add a dataset to shadow an existing dataset. self.testRepo.addDataset(run="foo", dataId={"instrument": "DummyCamComp", "visit": 424}) # Verify that without find-first, duplicate datasets are returned tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True) expectedTables = ( AstropyTable(array( ( ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000424_fDummyCamComp_data.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables) # Verify that with find first the duplicate dataset is eliminated and # the more recent dataset is returned. tables = self._queryDatasets(repo=self.root, collections=["foo", "ingest/run"], show_uri=True, find_first=True) expectedTables = ( AstropyTable(array( ( ("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")), ("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.data/" "test_metric_comp_v00000423_fDummyCamComp_data.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.output/" "test_metric_comp_v00000424_fDummyCamComp_output.yaml")), ("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.output/" "test_metric_comp_v00000423_fDummyCamComp_output.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), AstropyTable(array( ( ("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424", self.testRepo.butler.datastore.root.join( "foo/test_metric_comp.summary/" "test_metric_comp_v00000424_fDummyCamComp_summary.yaml")), ("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423", self.testRepo.butler.datastore.root.join( "ingest/run/test_metric_comp.summary/" "test_metric_comp_v00000423_fDummyCamComp_summary.yaml")), )), names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit", "URI")), ) self.assertAstropyTablesEqual(tables, expectedTables)