class ConfigValidateUseTest(unittest.TestCase):
"""Test executing the command."""
def setUp(self):
self.runner = LogCliRunner()
def testConfigValidate(self):
"""Test validating a valid config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
# verify the just-created repo validates without error
result = self.runner.invoke(butler.cli,
["config-validate", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
self.assertEqual(result.stdout,
"No problems encountered with configuration.\n")
def testConfigValidate_ignore(self):
"""Test the ignore flag"""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
# verify the just-created repo validates without error
result = self.runner.invoke(butler.cli, [
"config-validate", "here", "--ignore",
"storageClasses,repoTransferFormats", "-i", "dimensions"
])
self.assertEqual(result.exit_code, 0, result.stdout)
self.assertEqual(result.stdout,
"No problems encountered with configuration.\n")
class RegisterSkymapConfigTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testNoConfigOverride(self):
"""Verify expected arguments are passed to makeSkyMap with no config
overrides."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch(
"lsst.pipe.tasks.script.registerSkymap.makeSkyMap"
) as mock:
# call without any config overrides
result = self.runner.invoke(butlerCli,
["register-skymap", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedConfig = registerSkymap.MakeSkyMapConfig()
mock.assert_called_once()
# assert that the first argument to the call to makeSkyMap was a butler
self.assertIsInstance(mock.call_args[0][0], Butler)
# assert that the second argument matches the expected config
self.assertEqual(mock.call_args[0][1], expectedConfig)
def testConfigOverride(self):
"""Verify expected arguments are passed to makeSkyMap with config
overrides."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch(
"lsst.pipe.tasks.script.registerSkymap.makeSkyMap"
) as mock:
# call and override the name parameter of the config
result = self.runner.invoke(
butlerCli,
["register-skymap", "repo", "--config", "name=bar"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedConfig = registerSkymap.MakeSkyMapConfig()
expectedConfig.update(name="bar")
mock.assert_called_once()
# assert that the first argument to the makeSkyMap call was a butler
self.assertIsInstance(mock.call_args[0][0], Butler)
# assert that the second argument matches the expected config
self.assertEqual(mock.call_args[0][1], expectedConfig)
def testNonExistantConfigFile(self):
"""Verify an expected error when a given config override file does not
exist. """
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butlerCli,
["register-skymap", "repo", "--config-file", "foo.py"])
# foo.py does not exist; exit could should be non-zero.
self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
class RawIngestMockTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test(self):
"""Verify config gets applied properly."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch("lsst.obs.base.RawIngestTask",
new=PatchRawIngestTask) as mock:
# call and override the name parameter of the config
result = self.runner.invoke(butlerCli, [
"ingest-raws", "repo", "resources", "--config",
"transfer=hardlink"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Verify the mock class was initialized exactly once:
self.assertEqual(len(mock.init_args), 1)
# Verify that the task was initialized with a 'butler' kwarg
# that received a butler instance:
self.assertIsInstance(mock.init_args[0][1]["butler"], Butler)
# Verify that the task was initialized with a 'config' kwarg
# that received an expected config:
expectedConfig = RawIngestConfig()
expectedConfig.update(transfer="hardlink")
self.assertEqual(mock.init_args[0][1]["config"],
expectedConfig)
class RegisterDcrSubfiltersTest(unittest.TestCase, ButlerTestHelper):
def setUp(self):
self.runner = LogCliRunner()
self.repo = "here"
def testRegisterFilters(self):
"""Register a few filters and verify they are added to the repo."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", self.repo])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "3", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(
AstropyTable((("foo", "foo", "foo"), (0, 1, 2)),
names=("band", "id")), readTable(result.output))
# Verify expected output message for registering subfilters in a
# band that already has subfilters
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "5", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
# Add subfilters for two filters, one new filter and one existing.
# Verify expected result messages and registry values.
result = self.runner.invoke(
butlerCli,
["register-dcr-subfilters", self.repo, "3", "foo", "bar"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="bar", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
resultTable = readTable(result.output)
resultTable.sort(["band", "id"])
self.assertAstropyTablesEqual(
AstropyTable((("bar", "bar", "bar", "foo", "foo", "foo"),
(0, 1, 2, 0, 1, 2)),
names=("band", "id")), resultTable)
def testGetCollections(self):
run = "ingest/run"
tag = "ingest"
expected = {"collections": [run, tag]}
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
# the purpose of this call is to create some collections
_ = Butler(butlerCfg, run=run, tags=[tag], collections=[tag])
result = runner.invoke(cli, ["query-collections", "here"])
self.assertEqual(expected, yaml.safe_load(result.output))
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = AstropyTable((("test", ), ), names=("name", ))
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = AstropyTable(array((
"test",
"['band', 'instrument', 'physical_filter', 'visit_system', 'visit']",
"testDatasetType")),
names=("name", "dimensions",
"storage class"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Now remove and check that it was removed
# First a non-existent one
result = runner.invoke(cli,
["remove-dataset-type", "here", "unreal"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Now one we now has been registered
result = runner.invoke(
cli, ["remove-dataset-type", "here", datasetName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# and check that it has gone
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn("No results", result.output)
def testRetrieveSubset(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp1/"
result = runner.invoke(cli, [
"retrieve-artifacts", self.root, destdir, "--where",
"instrument='DummyCamComp' AND visit=423"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 3\n"),
f"Expected 3 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 3,
f"Expected 3 artifacts: {artifacts}")
class PruneCollectionsTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testPruneCollections(self):
"""Test removing a collection and run from a repository using the
butler prune-collection subcommand."""
with self.runner.isolated_filesystem():
repoName = "myRepo"
runName = "myRun"
taggedName = "taggedCollection"
# Add the run and the tagged collection to the repo:
result = self.runner.invoke(butlerCli, ["create", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Use the butler initalizer to create the run, then create a tagged
# collection.
butler = Butler(repoName, run=runName)
butler.registry.registerCollection(taggedName,
CollectionType.TAGGED)
# Verify the run and tag show up in query-collections:
result = self.runner.invoke(butlerCli,
["query-collections", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(runName, result.output)
self.assertIn(taggedName, result.output)
# Verify the tagged collection can be removed:
result = self.runner.invoke(
butlerCli,
["prune-collection", repoName, taggedName, "--unstore"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(butlerCli,
["query-collections", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(runName, result.output)
self.assertNotIn(taggedName, result.output)
# Verify the run can be removed:
result = self.runner.invoke(butlerCli, [
"prune-collection", repoName, runName, "--purge", "--unstore"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertNotIn(runName, result.output)
self.assertNotIn(taggedName, result.output)
def testClobber(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp2/"
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Running again should fail
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
# But with clobber should pass
result = runner.invoke(
cli, ["retrieve-artifacts", self.root, destdir, "--clobber"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = {"datasetTypes": [datasetName]}
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertEqual(expectedNotVerbose, yaml.safe_load(result.output))
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertEqual(expectedNotVerbose, yaml.safe_load(result.output))
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
response = yaml.safe_load(result.output)
# output dimension names contain all required dimensions, more than
# the registered dimensions, so verify the expected components
# individually.
self.assertEqual(response["datasetTypes"][0]["name"], datasetName)
self.assertEqual(response["datasetTypes"][0]["storageClass"],
storageClassName)
self.assertIn(instrumentDimension,
response["datasetTypes"][0]["dimensions"])
self.assertIn(visitDimension,
response["datasetTypes"][0]["dimensions"])
class QueryCollectionsScriptTest(ButlerTestHelper, unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testGetCollections(self):
run = "ingest/run"
tag = "tag"
with self.runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
# the purpose of this call is to create some collections
butler = Butler(butlerCfg,
run=run,
collections=[tag],
writeable=True)
butler.registry.registerCollection(tag, CollectionType.TAGGED)
# Verify collections that were created are found by
# query-collections.
result = self.runner.invoke(cli, ["query-collections", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("ingest/run", "tag"), ("RUN", "TAGGED")),
names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Verify that with a glob argument, that only collections whose
# name matches with the specified pattern are returned.
result = self.runner.invoke(cli,
["query-collections", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("tag", ), ("TAGGED", )), names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Verify that with a collection type argument, only collections of
# that type are returned.
result = self.runner.invoke(
cli, ["query-collections", "here", "--collection-type", "RUN"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("ingest/run", ), ("RUN", )),
names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testRetrieveAll(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
# When preserving the path the run will be in the directory along
# with a . in the component name. When not preserving paths the
# filename will have an underscore rather than dot.
for counter, (preserve_path, prefix) in enumerate(
(("--preserve-path", "ingest/run/test_metric_comp."),
("--no-preserve-path", "test_metric_comp_"))):
destdir = f"tmp{counter}/"
result = runner.invoke(
cli,
["retrieve-artifacts", self.root, destdir, preserve_path])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 6\n"),
f"Expected 6 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 6,
f"Expected 6 artifacts: {artifacts}")
self.assertIn(f"{destdir}{prefix}", str(artifacts[1]))
def test_cli(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
result = runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
registerInstrumentArgs = [
"register-instrument", "here", self.instrumentClassName
]
if self.secondInstrumentClassName is not None:
registerInstrumentArgs.append(self.secondInstrumentClassName)
result = runner.invoke(butler.cli, registerInstrumentArgs)
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
result = runner.invoke(butler.cli, [
"write-curated-calibrations", "here", self.instrumentName,
"--collection", "collection"
])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
class CliLogTestBase():
"""Tests log initialization, reset, and setting log levels."""
lsstLogHandlerId = None
def setUp(self):
self.runner = LogCliRunner()
def tearDown(self):
self.lsstLogHandlerId = None
class PythonLogger:
"""Keeps track of log level of a component and number of handlers
attached to it at the time this object was initialized."""
def __init__(self, component):
self.logger = logging.getLogger(component)
self.initialLevel = self.logger.level
class LsstLogger:
"""Keeps track of log level for a component at the time this object was
initialized."""
def __init__(self, component):
self.logger = lsstLog.getLogger(component) if lsstLog else None
self.initialLevel = self.logger.getLevel() if lsstLog else None
def runTest(self, cmd):
"""Test that the log context manager works with the butler cli to
initialize the logging system according to cli inputs for the duration
of the command execution and resets the logging system to its previous
state or expected state when command execution finishes."""
pyRoot = self.PythonLogger(None)
pyButler = self.PythonLogger("lsst.daf.butler")
lsstRoot = self.LsstLogger("")
lsstButler = self.LsstLogger("lsst.daf.butler")
with command_test_env(self.runner, "lsst.daf.butler.tests.cliLogTestBase",
"command-log-settings-test"):
result = cmd()
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
if lsstLog is not None:
self.assertFalse(hasLsstLogHandler(logging.getLogger()),
msg="CliLog should remove the lsst.log handler it added to the root logger.")
self.assertEqual(pyRoot.logger.level, logging.INFO)
self.assertEqual(pyButler.logger.level, pyButler.initialLevel)
if lsstLog is not None:
self.assertEqual(lsstRoot.logger.getLevel(), lsstLog.INFO)
# lsstLogLevel can either be the inital level, or uninitialized or
# the defined default value.
expectedLsstLogLevel = ((lsstButler.initialLevel, ) if lsstButler.initialLevel != -1
else(-1, CliLog.defaultLsstLogLevel))
self.assertIn(lsstButler.logger.getLevel(), expectedLsstLogLevel)
def test_butlerCliLog(self):
"""Test that the log context manager works with the butler cli to
initialize the logging system according to cli inputs for the duration
of the command execution and resets the logging system to its previous
state or expected state when command execution finishes."""
self.runTest(partial(self.runner.invoke,
butlerCli,
["--log-level", "WARNING",
"--log-level", "lsst.daf.butler=DEBUG",
"command-log-settings-test",
"--expected-pyroot-level", logging.WARNING,
"--expected-pybutler-level", logging.DEBUG,
"--expected-lsstroot-level", lsstLog.WARN if lsstLog else 0,
"--expected-lsstbutler-level", lsstLog.DEBUG if lsstLog else 0]))
def test_helpLogReset(self):
"""Verify that when a command does not execute, like when the help menu
is printed instead, that CliLog is still reset."""
self.runTest(partial(self.runner.invoke, butlerCli, ["command-log-settings-test", "--help"]))
def testLongLog(self):
"""Verify the timestamp is in the log messages when the --long-log
flag is set."""
# When longlog=True, loglines start with the log level and a
# timestamp with the following format:
# "year-month-day T hour-minute-second.millisecond-zoneoffset"
# If lsst.log is importable then the timestamp will have
# milliseconds, as described above. If lsst.log is NOT
# importable then milliseconds (and the preceding ".") are
# omitted (the python `time` module does not support
# milliseconds in its format string). Examples of expected
# strings follow:
# lsst.log: "DEBUG 2020-10-29T10:20:31.518-0700 ..."
# pure python "DEBUG 2020-10-28T10:20:31-0700 ...""
# The log level name can change, we verify there is an all
# caps word there but do not verify the word. We do not verify
# the rest of the log string, assume that if the timestamp is
# in the string that the rest of the string will appear as
# expected.
# N.B. this test is defined in daf_butler which does not depend
# on lsst.log. However, CliLog may be used in packages that do
# depend on lsst.log and so both forms of timestamps must be
# supported. These packages should have a test (the file is
# usually called test_cliLog.py) that subclasses CliLogTestBase
# and unittest.TestCase so that these tests are run in that
# package.
timestampRegex = re.compile(
r".*[A-Z]+ [0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(.[0-9]{3})?([-,+][0-9]{4}|Z) .*")
# When longlog=False, log lines start with the module name and
# log level, for example:
# lsst.daf.butler.core.config DEBUG: ...
modulesRegex = re.compile(
r".* ([a-z]+\.)+[a-z]+ [A-Z]+: .*")
with self.runner.isolated_filesystem():
for longlog in (True, False):
# The click test does not capture logging emitted from lsst.log
# so use subprocess to run the test instead.
if longlog:
args = ("butler", "--log-level", "DEBUG", "--long-log", "create", "here")
else:
args = ("butler", "--log-level", "DEBUG", "create", "here")
result = subprocess.run(args, capture_output=True)
# There are cases where the newlines are stripped from the log
# output (like in Jenkins), since we can't depend on newlines
# in log output they are removed here from test output.
output = StringIO((result.stderr.decode().replace("\n", " ")))
startedWithTimestamp = any([timestampRegex.match(line) for line in output.readlines()])
output.seek(0)
startedWithModule = any(modulesRegex.match(line) for line in output.readlines())
if longlog:
self.assertTrue(startedWithTimestamp,
msg=f"did not find timestamp in: \n{output.getvalue()}")
self.assertFalse(startedWithModule,
msg=f"found lines starting with module in: \n{output.getvalue()}")
else:
self.assertFalse(startedWithTimestamp,
msg=f"found timestamp in: \n{output.getvalue()}")
self.assertTrue(startedWithModule,
msg=f"did not find lines starting with module in: \n{output.getvalue()}")
def run_test(self,
mockPruneDatasets,
mockQueryDatasets_init,
mockQueryDatasets_getDatasets,
mockQueryDatasets_getTables,
cliArgs,
exMsgs,
exPruneDatasetsCallArgs,
exGetTablesCalled,
exQueryDatasetsCallArgs,
invokeInput=None,
exPruneDatasetsExitCode=0):
"""Execute the test.
Makes a temporary repo, invokes ``prune-datasets``. Verifies expected
output, exit codes, and mock calls.
Parameters
----------
mockPruneDatasets : `MagicMock`
The MagicMock for the ``Butler.pruneDatasets`` function.
mockQueryDatasets_init : `MagicMock`
The MagicMock for the ``QueryDatasets.__init__`` function.
mockQueryDatasets_getDatasets : `MagicMock`
The MagicMock for the ``QueryDatasets.getDatasets`` function.
mockQueryDatasets_getTables : `MagicMock`
The MagicMock for the ``QueryDatasets.getTables`` function.
cliArgs : `list` [`str`]
The arguments to pass to the command line. Do not include the
subcommand name or the repo.
exMsgs : `list` [`str`] or None
A list of text fragments that should appear in the text output
after calling the CLI command, or None if no output should be
produced.
exPruneDatasetsCallArgs : `dict` [`str`, `Any`]
The arguments that ``Butler.pruneDatasets`` should have been called
with, or None if that function should not have been called.
exGetTablesCalled : bool
`True` if ``QueryDatasets.getTables`` should have been called, else
`False`.
exQueryDatasetsCallArgs : `dict` [`str`, `Any`]
The arguments that ``QueryDatasets.__init__`` should have bene
called with, or `None` if the function should not have been called.
invokeInput : `str`, optional.
As string to pass to the ``CliRunner.invoke`` `input` argument. By
default None.
exPruneDatasetsExitCode : `int`
The expected exit code returned from invoking ``prune-datasets``.
"""
runner = LogCliRunner()
with runner.isolated_filesystem():
# Make a repo so a butler can be created
result = runner.invoke(butlerCli, ["create", self.repo])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Run the prune-datasets CLI command, this will call all of our
# mocks:
cliArgs = ["prune-datasets", self.repo] + cliArgs
result = runner.invoke(butlerCli, cliArgs, input=invokeInput)
self.assertEqual(result.exit_code, exPruneDatasetsExitCode, clickResultMsg(result))
# Verify the Butler.pruneDatasets was called exactly once with
# expected arguments. The datasets argument is the value returned
# by QueryDatasets, which we've mocked with side effect
# ``getDatasets()``.
if exPruneDatasetsCallArgs:
mockPruneDatasets.assert_called_once_with(**exPruneDatasetsCallArgs)
else:
mockPruneDatasets.assert_not_called()
# Less critical, but do a quick verification that the QueryDataset
# member function mocks were called, in this case we expect one
# time each.
if exQueryDatasetsCallArgs:
mockQueryDatasets_init.assert_called_once_with(**exQueryDatasetsCallArgs)
else:
mockQueryDatasets_init.assert_not_called()
# If Butler.pruneDatasets was not called, then
# QueryDatasets.getDatasets also does not get called.
if exPruneDatasetsCallArgs:
mockQueryDatasets_getDatasets.assert_called_once()
else:
mockQueryDatasets_getDatasets.assert_not_called()
if exGetTablesCalled:
mockQueryDatasets_getTables.assert_called_once()
else:
mockQueryDatasets_getTables.assert_not_called()
if exMsgs is None:
self.assertEqual("", result.output)
else:
for expectedMsg in exMsgs:
self.assertIn(expectedMsg, result.output)
class ConfigDumpUseTest(unittest.TestCase):
"""Test executing the command."""
def setUp(self):
self.runner = LogCliRunner()
def test_stdout(self):
"""Test dumping the config to stdout."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# test dumping to stdout:
result = self.runner.invoke(butler.cli, ["config-dump", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# check for some expected keywords:
cfg = yaml.safe_load(result.stdout)
self.assertIn("datastore", cfg)
self.assertIn("composites", cfg["datastore"])
self.assertIn("storageClasses", cfg)
def test_file(self):
"""test dumping the config to a file."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--file=there"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# check for some expected keywords:
with open("there", "r") as f:
cfg = yaml.safe_load(f)
self.assertIn("datastore", cfg)
self.assertIn("composites", cfg["datastore"])
self.assertIn("storageClasses", cfg)
def test_subset(self):
"""Test selecting a subset of the config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--subset", "datastore"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
cfg = yaml.safe_load(result.stdout)
# count the keys in the datastore config
self.assertIs(len(cfg), 7)
self.assertIn("cls", cfg)
self.assertIn("create", cfg)
self.assertIn("formatters", cfg)
self.assertIn("records", cfg)
self.assertIn("root", cfg)
self.assertIn("templates", cfg)
def test_invalidSubset(self):
"""Test selecting a subset key that does not exist in the config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# test dumping to stdout:
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--subset", "foo"])
self.assertEqual(result.exit_code, 1)
self.assertIn("Error: 'foo not found in config at here'",
result.output)
class ConfigDumpUseTest(unittest.TestCase):
"""Test executing the command."""
def setUp(self):
self.runner = LogCliRunner()
def test_stdout(self):
"""Test dumping the config to stdout."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# test dumping to stdout:
result = self.runner.invoke(butler.cli, ["config-dump", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# check for some expected keywords:
cfg = yaml.safe_load(result.stdout)
self.assertIn("datastore", cfg)
self.assertIn("composites", cfg["datastore"])
self.assertIn("storageClasses", cfg)
def test_file(self):
"""test dumping the config to a file."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--file=there"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# check for some expected keywords:
with open("there", "r") as f:
cfg = yaml.safe_load(f)
self.assertIn("datastore", cfg)
self.assertIn("composites", cfg["datastore"])
self.assertIn("storageClasses", cfg)
def test_subset(self):
"""Test selecting a subset of the config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--subset", "datastore"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
cfg = yaml.safe_load(result.stdout)
# count the keys in the datastore config
self.assertIs(len(cfg), 7)
self.assertIn("cls", cfg)
self.assertIn("create", cfg)
self.assertIn("formatters", cfg)
self.assertIn("records", cfg)
self.assertIn("root", cfg)
self.assertIn("templates", cfg)
def test_invalidSubset(self):
"""Test selecting a subset key that does not exist in the config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# test dumping to stdout:
result = self.runner.invoke(
butler.cli, ["config-dump", "here", "--subset", "foo"])
self.assertEqual(result.exit_code, 1)
self.assertEqual(result.exception.args,
KeyError('foo not found in config at here').args)
def test_presets(self):
"""Test that file overrides can set command line options in bulk.
"""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
overrides_path = os.path.join(TESTDIR, "data",
"config-overrides.yaml")
# Run with a presets file
result = self.runner.invoke(
butler.cli,
["config-dump", "here", "--options-file", overrides_path])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
cfg = yaml.safe_load(result.stdout)
# Look for datastore information
self.assertIn("formatters", cfg)
self.assertIn("root", cfg)
# Now run with an explicit subset and presets
result = self.runner.invoke(butler.cli, [
"config-dump", "here", f"-@{overrides_path}", "--subset",
".registry"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
cfg = yaml.safe_load(result.stdout)
# Look for datastore information
self.assertNotIn("formatters", cfg)
self.assertIn("managers", cfg)
# Now with subset before presets -- explicit always trumps
# presets.
result = self.runner.invoke(butler.cli, [
"config-dump", "here", "--subset", ".registry",
"--options-file", overrides_path
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
cfg = yaml.safe_load(result.stdout)
# Look for datastore information
self.assertNotIn("formatters", cfg)
self.assertIn("managers", cfg)
class ChainedCollectionsTest(ButlerTestHelper, unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testChained(self):
with self.runner.isolated_filesystem():
# Create a butler and add some chained collections:
butlerCfg = Butler.makeRepo("here")
butler1 = Butler(butlerCfg, writeable=True)
# Replace datastore functions with mocks:
DatastoreMock.apply(butler1)
butler1.import_(filename=os.path.join(TESTDIR, "data", "registry",
"base.yaml"))
butler1.import_(filename=os.path.join(TESTDIR, "data", "registry",
"datasets.yaml"))
registry1 = butler1.registry
registry1.registerRun("run1")
registry1.registerCollection("tag1", CollectionType.TAGGED)
registry1.registerCollection("calibration1",
CollectionType.CALIBRATION)
registry1.registerCollection("chain1", CollectionType.CHAINED)
registry1.registerCollection("chain2", CollectionType.CHAINED)
registry1.setCollectionChain("chain1", ["tag1", "run1", "chain2"])
registry1.setCollectionChain("chain2", ["calibration1", "run1"])
# Use the script function to test the query-collections TREE
# option, because the astropy.table.Table.read method, which we are
# using for verification elsewhere in this file, seems to strip
# leading whitespace from columns. This makes it impossible to test
# the nested TREE output of the query-collections subcommand from
# the command line interface.
table = queryCollections("here",
glob=(),
collection_type=CollectionType.all(),
chains="TREE")
# self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table(array(
(("imported_g", "RUN"), ("imported_r", "RUN"), ("run1", "RUN"),
("tag1", "TAGGED"), ("calibration1", "CALIBRATION"),
("chain1", "CHAINED"), (" tag1", "TAGGED"),
(" run1", "RUN"), (" chain2", "CHAINED"),
(" calibration1", "CALIBRATION"), (" run1", "RUN"),
("chain2", "CHAINED"), (" calibration1",
"CALIBRATION"), (" run1", "RUN"))),
names=("Name", "Type"))
self.assertAstropyTablesEqual(table, expected)
result = self.runner.invoke(cli, ["query-collections", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table(array(
(("imported_g", "RUN", ""), ("imported_r", "RUN", ""),
("run1", "RUN", ""), ("tag1", "TAGGED", ""),
("calibration1", "CALIBRATION", ""), ("chain1", "CHAINED",
"[tag1, run1, chain2]"),
("chain2", "CHAINED", "[calibration1, run1]"))),
names=("Name", "Type", "Definition"))
table = readTable(result.output)
self.assertAstropyTablesEqual(readTable(result.output), expected)
result = self.runner.invoke(
cli, ["query-collections", "here", "--chains", "FLATTEN"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table(array(
(("imported_g", "RUN"), ("imported_r", "RUN"), ("run1", "RUN"),
("tag1", "TAGGED"), ("calibration1", "CALIBRATION"),
("tag1", "TAGGED"), ("run1", "RUN"), ("calibration1",
"CALIBRATION"),
("calibration1", "CALIBRATION"), ("run1", "RUN"))),
names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
class PluginLoaderTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test_loadAndExecutePluginCommand(self):
"""Test that a plugin command can be loaded and executed."""
with command_test_env(self.runner, "test_cliPluginLoader",
"command-test"):
result = self.runner.invoke(butler.cli, "command-test")
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
self.assertEqual(result.stdout, "test command\n")
def test_loadAndExecuteLocalCommand(self):
"""Test that a command in daf_butler can be loaded and executed."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "test_repo"])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
self.assertTrue(os.path.exists("test_repo"))
def test_loadTopHelp(self):
"""Test that an expected command is produced by 'butler --help'"""
with command_test_env(self.runner, "test_cliPluginLoader",
"command-test"):
result = self.runner.invoke(butler.cli, "--help")
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
self.assertIn("command-test", result.stdout)
def test_getLocalCommands(self):
"""Test getting the daf_butler CLI commands."""
localCommands = butler.ButlerCLI().getLocalCommands()
# the number of local commands should equal the number of functions
# in cmd.__all__
self.assertEqual(len(localCommands), len(cmd.__all__))
def test_mergeCommandLists(self):
"""Verify dicts of command to list-of-source-package get merged
properly."""
first = defaultdict(list, {"a": [1]})
second = defaultdict(list, {"b": [2]})
self.assertEqual(butler.LoaderCLI._mergeCommandLists(first, second), {
"a": [1],
"b": [2]
})
first = defaultdict(list, {"a": [1]})
second = defaultdict(list, {"a": [2]})
self.assertEqual(butler.LoaderCLI._mergeCommandLists(first, second),
{"a": [1, 2]})
def test_listCommands_duplicate(self):
"""Test executing a command in a situation where duplicate commands are
present and verify it fails to run.
"""
self.maxDiff = None
with duplicate_command_test_env(self.runner):
result = self.runner.invoke(butler.cli, ["create", "test_repo"])
self.assertEqual(
result.exit_code, 1,
f"output: {result.output} exception: {result.exception}")
self.assertEqual(
result.output, "Error: Command 'create' "
"exists in packages lsst.daf.butler.cli.cmd, test_cliPluginLoader. "
"Duplicate commands are not supported, aborting.\n")