def testGlobDatasetType(self):
"""Test specifying dataset type."""
# Create and register an additional DatasetType
self.testRepo.butler.registry.insertDimensionData("visit",
{"instrument": "DummyCamComp", "id": 425,
"name": "fourtwentyfive", "physical_filter": "d-r",
"visit_system": 1})
datasetType = addDatasetType(self.testRepo.butler,
"alt_test_metric_comp",
("instrument", "visit"),
"StructuredCompositeReadComp")
self.testRepo.addDataset(dataId={"instrument": "DummyCamComp", "visit": 425}, datasetType=datasetType)
# verify the new dataset type increases the number of tables found:
tables = self._queryDatasets(repo=self.root)
expectedTables = (
AstropyTable(array((
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
),
AstropyTable(array((
("alt_test_metric_comp", "ingest/run", "3", "R", "DummyCamComp", "d-r", "1", "425"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
)
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = AstropyTable((("test", ), ), names=("name", ))
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = AstropyTable(array((
"test",
"['band', 'instrument', 'physical_filter', 'visit_system', 'visit']",
"testDatasetType")),
names=("name", "dimensions",
"storage class"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Now remove and check that it was removed
# First a non-existent one
result = runner.invoke(cli,
["remove-dataset-type", "here", "unreal"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Now one we now has been registered
result = runner.invoke(
cli, ["remove-dataset-type", "here", datasetName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# and check that it has gone
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn("No results", result.output)
def testShowURI(self):
"""Test for expected output with show_uri=True."""
tables = self._queryDatasets(repo=self.root, show_uri=True)
expectedTables = (
AstropyTable(array(
(("test_metric_comp.data", "ingest/run", "1", "R",
"DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "2", "R",
"DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")))),
names=("type", "run", "id", "band", "instrument",
"physical_filter", "visit_system", "visit",
"URI")),
AstropyTable(array(
(("test_metric_comp.output", "ingest/run", "1", "R",
"DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "2", "R",
"DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml"))
)),
names=("type", "run", "id", "band", "instrument",
"physical_filter", "visit_system", "visit",
"URI")),
AstropyTable(array((
("test_metric_comp.summary",
"ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "2", "R",
"DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")
))),
names=("type", "run", "id", "band", "instrument",
"physical_filter", "visit_system", "visit",
"URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testRegisterFilters(self):
"""Register a few filters and verify they are added to the repo."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", self.repo])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "3", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(
AstropyTable((("foo", "foo", "foo"), (0, 1, 2)),
names=("band", "id")), readTable(result.output))
# Verify expected output message for registering subfilters in a
# band that already has subfilters
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "5", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
# Add subfilters for two filters, one new filter and one existing.
# Verify expected result messages and registry values.
result = self.runner.invoke(
butlerCli,
["register-dcr-subfilters", self.repo, "3", "foo", "bar"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="bar", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
resultTable = readTable(result.output)
resultTable.sort(["band", "id"])
self.assertAstropyTablesEqual(
AstropyTable((("bar", "bar", "bar", "foo", "foo", "foo"),
(0, 1, 2, 0, 1, 2)),
names=("band", "id")), resultTable)
def getAstropyTable(self):
"""Get the table as an astropy table.
Returns
-------
table : `astropy.table.Table`
The dataIds, sorted by spatial and temporal columns first, and then
the rest of the columns, with duplicate dataIds removed.
"""
# Should never happen; adding a dataset should be the action that
# causes a _Table to be created.
if not self.dataIds:
raise RuntimeError("No DataIds were provided.")
dataId = next(iter(self.dataIds))
dimensions = list(dataId.full.keys())
columnNames = [str(item) for item in dimensions]
# Need to hint the column types for numbers since the per-row
# constructor of Table does not work this out on its own and sorting
# will not work properly without.
typeMap = {float: np.float64, int: np.int64}
columnTypes = [
typeMap.get(type(value)) for value in dataId.full.values()
]
rows = [[value for value in dataId.full.values()]
for dataId in self.dataIds]
table = AstropyTable(np.array(rows),
names=columnNames,
dtype=columnTypes)
return sortAstropyTable(table, dimensions)
def testCollection(self):
butler = Butler(self.root, run="foo")
# try replacing the testRepo's butler with the one with the "foo" run.
self.testRepo.butler = butler
self.testRepo.butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 425,
"name": "fourtwentyfive",
"physical_filter": "d-r",
"visit_system": 1
})
self.testRepo.addDataset(dataId={
"instrument": "DummyCamComp",
"visit": 425
},
run="foo")
# verify getting records from the "ingest/run" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"ingest/run", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array(
(("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None",
"None", "None", "None", "None", "None", "None", "None .. None"),
("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None",
"None", "None", "None", "None", "None", "None", "None .. None")))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
# verify getting records from the "foo" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"foo", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "425", "d-r", "1", "fourtwentyfive",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testDatasets(self):
"""Test getting datasets."""
res = self._queryDataIds(self.root, datasets="test_metric_comp")
expected = AstropyTable(array((("R", "DummyCamComp", "d-r", 1, 423),
("R", "DummyCamComp", "d-r", 1, 424))),
names=("band", "instrument", "physical_filter",
"visit_system", "visit"))
self.assertAstropyTablesEqual(res, expected)
def testWhere(self):
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--where",
"instrument='DummyCamComp' AND visit.name='fourtwentythree'"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "423", "d-r", "1", "fourtwentythree",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testWhere(self):
"""Test getting datasets."""
res = self._queryDataIds(
self.root,
dimensions=("visit", ),
where="instrument='DummyCamComp' AND visit=423")
expected = AstropyTable(array(
(("R", "DummyCamComp", "d-r", 1, 423), )),
names=("band", "instrument", "physical_filter",
"visit_system", "visit"))
self.assertAstropyTablesEqual(res, expected)
def getAstropyTable(self, datasetTypeName):
"""Get the table as an astropy table.
Parameters
----------
datasetTypeName : `str`
The dataset type name to show in the ``type`` column of the table.
Returns
-------
table : `astropy.table._Table`
The table with the provided column names and rows.
"""
# Should never happen; adding a dataset should be the action that
# causes a _Table to be created.
if not self.datasetRefs:
raise RuntimeError(
f"No DatasetRefs were provided for dataset type {datasetTypeName}"
)
refInfo = next(iter(self.datasetRefs))
dimensions = list(refInfo.datasetRef.dataId.full.keys())
columnNames = [
"type", "run", "id", *[str(item) for item in dimensions]
]
# Need to hint the column types for numbers since the per-row
# constructor of Table does not work this out on its own and sorting
# will not work properly without.
typeMap = {float: np.float64, int: np.int64}
columnTypes = [
None, None, np.int64, *[
typeMap.get(type(value))
for value in refInfo.datasetRef.dataId.full.values()
]
]
if refInfo.uri:
columnNames.append("URI")
columnTypes.append(None)
rows = []
for refInfo in self.datasetRefs:
row = [
datasetTypeName, refInfo.datasetRef.run, refInfo.datasetRef.id,
*[value for value in refInfo.datasetRef.dataId.full.values()]
]
if refInfo.uri:
row.append(refInfo.uri)
rows.append(row)
dataset_table = AstropyTable(np.array(rows),
names=columnNames,
dtype=columnTypes)
return sortAstropyTable(dataset_table, dimensions, ["type", "run"])
def testCollections(self):
"""Test getting datasets using the collections option."""
# Add a dataset in a different collection
self.butler = Butler(self.root, run="foo")
self.repo.butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 425,
"name": "fourtwentyfive",
"physical_filter": "d-r",
"visit_system": 1
})
self.repo.addDataset(dataId={
"instrument": "DummyCamComp",
"visit": 425
},
run="foo")
# Verify the new dataset is not found in the "ingest/run" collection.
res = self._queryDataIds(repo=self.root,
dimensions=("visit", ),
collections=("ingest/run", ),
datasets="test_metric_comp")
expected = AstropyTable(array((("R", "DummyCamComp", "d-r", 1, 423),
("R", "DummyCamComp", "d-r", 1, 424))),
names=("band", "instrument", "physical_filter",
"visit_system", "visit"))
self.assertAstropyTablesEqual(res, expected)
# Verify the new dataset is found in the "foo" collection.
res = self._queryDataIds(repo=self.root,
dimensions=("visit", ),
collections=("foo", ),
datasets="test_metric_comp")
expected = AstropyTable(array(
(("R", "DummyCamComp", "d-r", 1, 425), )),
names=("band", "instrument", "physical_filter",
"visit_system", "visit"))
self.assertAstropyTablesEqual(res, expected)
def testNoShowURI(self):
"""Test for expected output without show_uri (default is False)."""
tables = self._queryDatasets(repo=self.root)
expectedTables = (
AstropyTable(array((
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423"),
("test_metric_comp", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424"))),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit")
),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testWhere(self):
"""Test using the where clause to reduce the number of rows returned.
"""
tables = self._queryDatasets(repo=self.root, where="instrument='DummyCamComp' AND visit=423")
expectedTables = (
AstropyTable(array(
("test_metric_comp", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423")),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system", "visit"),
),
)
self.assertAstropyTablesEqual(tables, expectedTables)
def testFindFirstAndCollections(self):
"""Test the find-first option, and the collections option, since it
is required for find-first."""
# Add a new run, and add a dataset to shadow an existing dataset.
self.testRepo.addDataset(run="foo",
dataId={"instrument": "DummyCamComp", "visit": 424})
# Verify that without find-first, duplicate datasets are returned
tables = self._queryDatasets(repo=self.root,
collections=["foo", "ingest/run"],
show_uri=True)
expectedTables = (
AstropyTable(array(
(
("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "2", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
# Verify that with find first the duplicate dataset is eliminated and
# the more recent dataset is returned.
tables = self._queryDatasets(repo=self.root,
collections=["foo", "ingest/run"],
show_uri=True,
find_first=True)
expectedTables = (
AstropyTable(array(
(
("test_metric_comp.data", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.data/test_metric_comp_v00000424_fDummyCamComp_data.yaml")),
("test_metric_comp.data", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.data/"
"test_metric_comp_v00000423_fDummyCamComp_data.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.output", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.output/"
"test_metric_comp_v00000424_fDummyCamComp_output.yaml")),
("test_metric_comp.output", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.output/"
"test_metric_comp_v00000423_fDummyCamComp_output.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
AstropyTable(array(
(
("test_metric_comp.summary", "foo", "3", "R", "DummyCamComp", "d-r", "1", "424",
self.testRepo.butler.datastore.root.join(
"foo/test_metric_comp.summary/"
"test_metric_comp_v00000424_fDummyCamComp_summary.yaml")),
("test_metric_comp.summary", "ingest/run", "1", "R", "DummyCamComp", "d-r", "1", "423",
self.testRepo.butler.datastore.root.join(
"ingest/run/test_metric_comp.summary/"
"test_metric_comp_v00000423_fDummyCamComp_summary.yaml")),
)),
names=("type", "run", "id", "band", "instrument", "physical_filter", "visit_system",
"visit", "URI")),
)
self.assertAstropyTablesEqual(tables, expectedTables)
