class FailedLoadTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test_unimportablePlugin(self):
with command_test_env(self.runner, "test_cliPluginLoader",
"non-existant-command-function"):
with self.assertLogs() as cm:
result = self.runner.invoke(butler.cli, "--help")
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
expectedErrMsg = "Could not import plugin from " \
"test_cliPluginLoader.non_existant_command_function, skipping."
self.assertIn(expectedErrMsg, " ".join(cm.output))
def test_unimportableLocalPackage(self):
class FailCLI(butler.LoaderCLI):
localCmdPkg = "lsst.daf.butler.cli.cmds" # should not be an importable location
@click.command(cls=FailCLI)
def cli():
pass
with self.assertLogs() as cm:
result = self.runner.invoke(cli)
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
expectedErrMsg = f"Could not import plugin from {FailCLI.localCmdPkg}, skipping."
self.assertIn(expectedErrMsg, " ".join(cm.output))
class ToUpperTestCase(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test_isolated(self):
"""test the to_upper callback by itself"""
ctx = "unused"
param = "unused"
self.assertEqual(to_upper(ctx, param, "debug"), "DEBUG")
def test_lowerToUpper(self):
"""test the to_upper callback in an option with a lowercase value"""
result = self.runner.invoke(cli, ["--value", "debug"])
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.stdout, "DEBUG\n")
def test_upperToUpper(self):
"""test the to_upper callback in an option with a uppercase value"""
result = self.runner.invoke(cli, ["--value", "DEBUG"])
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.stdout, "DEBUG\n")
def test_mixedToUpper(self):
"""test the to_upper callback in an option with a mixed-case value"""
result = self.runner.invoke(cli, ["--value", "DeBuG"])
self.assertEqual(result.exit_code, 0)
self.assertEqual(result.stdout, "DEBUG\n")
class ConfigValidateUseTest(unittest.TestCase):
"""Test executing the command."""
def setUp(self):
self.runner = LogCliRunner()
def testConfigValidate(self):
"""Test validating a valid config."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
# verify the just-created repo validates without error
result = self.runner.invoke(butler.cli,
["config-validate", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
self.assertEqual(result.stdout,
"No problems encountered with configuration.\n")
def testConfigValidate_ignore(self):
"""Test the ignore flag"""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(result.exit_code, 0, result.stdout)
# verify the just-created repo validates without error
result = self.runner.invoke(butler.cli, [
"config-validate", "here", "--ignore",
"storageClasses,repoTransferFormats", "-i", "dimensions"
])
self.assertEqual(result.exit_code, 0, result.stdout)
self.assertEqual(result.stdout,
"No problems encountered with configuration.\n")
class RawIngestMockTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test(self):
"""Verify config gets applied properly."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch("lsst.obs.base.RawIngestTask",
new=PatchRawIngestTask) as mock:
# call and override the name parameter of the config
result = self.runner.invoke(butlerCli, [
"ingest-raws", "repo", "resources", "--config",
"transfer=hardlink"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Verify the mock class was initialized exactly once:
self.assertEqual(len(mock.init_args), 1)
# Verify that the task was initialized with a 'butler' kwarg
# that received a butler instance:
self.assertIsInstance(mock.init_args[0][1]["butler"], Butler)
# Verify that the task was initialized with a 'config' kwarg
# that received an expected config:
expectedConfig = RawIngestConfig()
expectedConfig.update(transfer="hardlink")
self.assertEqual(mock.init_args[0][1]["config"],
expectedConfig)
class MWArgumentDecoratorTest(unittest.TestCase):
"""Tests for the MWArgumentDecorator class."""
things_argument = MWArgumentDecorator("things")
otherHelpText = "Help text for OTHER."
other_argument = MWArgumentDecorator("other", help=otherHelpText)
def setUp(self):
self.runner = LogCliRunner()
def test_help(self):
"""Verify expected help text output.
Verify argument help gets inserted after the usage, in the order
arguments are declared.
Verify that MWArgument adds " ..." after the option metavar when
`nargs` != 1. The default behavior of click is to add elipsis when
nargs does not equal 1, but it does not put a space before the elipsis
and we prefer a space between the metavar and the elipsis."""
# nargs can be -1 for any number of args, or >= 1 for a specified
# number of arguments.
helpText = "Things help text."
for numberOfArgs in (-1, 1, 2):
for required in (True, False):
@click.command()
@self.things_argument(required=required, nargs=numberOfArgs, help=helpText)
@self.other_argument()
def cmd(things, other):
"""Cmd help text."""
pass
result = self.runner.invoke(cmd, ["--help"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedOutut = (f"""Usage: cmd [OPTIONS] {'THINGS' if required else '[THINGS]'} {'... ' if numberOfArgs != 1 else ''}OTHER
Cmd help text.
{helpText}
{self.otherHelpText}
""")
self.assertIn(expectedOutut, result.output)
def testUse(self):
"""Test using the MWArgumentDecorator with a command."""
mock = MagicMock()
@click.command()
@self.things_argument()
def cli(things):
mock(things)
self.runner = click.testing.CliRunner()
result = self.runner.invoke(cli, ("foo"))
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
mock.assert_called_with("foo")
def _createRepo(cls):
"""Use the Click `testing` module to call the butler command line api
to create a repository."""
runner = LogCliRunner()
result = runner.invoke(butlerCli, ["create", cls.root])
# Classmethod so assertEqual does not work
assert result.exit_code == 0, f"output: {result.output} exception: {result.exception}"
class SplitCommasTestCase(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test_separate(self):
"""test the split_commas callback by itself"""
ctx = "unused"
param = "unused"
self.assertEqual(
split_commas(ctx, param, ("one,two", "three,four")), # noqa E231
("one", "two", "three", "four"))
self.assertEqual(split_commas(ctx, param, None), None)
def test_single(self):
"""test the split_commas callback in an option with one value"""
result = self.runner.invoke(cli, ["-l", "one"])
self.assertEqual(result.exit_code, 0, msg=clickResultMsg(result))
mock.assert_called_with(("one", ))
def test_multiple(self):
"""test the split_commas callback in an option with two single
values"""
result = self.runner.invoke(cli, ["-l", "one", "-l", "two"])
self.assertEqual(result.exit_code, 0, msg=clickResultMsg(result))
mock.assert_called_with(("one", "two"))
def test_singlePair(self):
"""test the split_commas callback in an option with one pair of
values"""
result = self.runner.invoke(cli, ["-l", "one,two"])
self.assertEqual(result.exit_code, 0, msg=clickResultMsg(result))
mock.assert_called_with(("one", "two"))
def test_multiplePair(self):
"""test the split_commas callback in an option with two pairs of
values"""
result = self.runner.invoke(cli, ["-l", "one,two", "-l", "three,four"])
self.assertEqual(result.exit_code, 0, msg=clickResultMsg(result))
mock.assert_called_with(("one", "two", "three", "four"))
def test_none(self):
"""test that passing None does not fail and returns None, producing an
empty tuple in the command function call."""
result = self.runner.invoke(cli, [])
self.assertEqual(result.exit_code, 0, msg=clickResultMsg(result))
mock.assert_called_with(())
def test_callMock(self):
"""Test that a mocked subcommand calls the Mocker and can be verified.
"""
runner = LogCliRunner(env=mockEnvVar)
result = runner.invoke(butler.cli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
Mocker.mock.assert_called_with(repo="repo", seed_config=None, standalone=False, override=False,
outfile=None)
class AssociateTestCase(unittest.TestCase):
"""Tests the ``associate`` ``butler`` subcommand.
``script.associate`` contains no logic, so instead of mocking the
internals, just mock the call to that function to test for expected inputs
and input types.
"""
def setUp(self):
self.runner = LogCliRunner()
@patch("lsst.daf.butler.script.associate")
def test_defaults(self, mockAssociate):
"""Test the expected default values & types for optional options.
"""
result = self.runner.invoke(
butlerCli, ["associate", "myRepo", "myCollection"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
mockAssociate.assert_called_once_with(
repo="myRepo",
collection="myCollection",
dataset_type=tuple(),
collections=tuple(),
where=None,
find_first=False
)
@patch("lsst.daf.butler.script.associate")
def test_values(self, mockAssociate):
"""Test expected values & types when passing in options.
"""
result = self.runner.invoke(
butlerCli, ["associate", "myRepo", "myCollection",
"--dataset-type", "myDatasetType",
"--collections", "myCollection,otherCollection",
"--where", "'a=b'",
"--find-first"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
mockAssociate.assert_called_once_with(
repo="myRepo",
collection="myCollection",
dataset_type=("myDatasetType",),
collections=("myCollection", "otherCollection"),
where="'a=b'",
find_first=True
)
def testClobber(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp2/"
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Running again should fail
result = runner.invoke(cli,
["retrieve-artifacts", self.root, destdir])
self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
# But with clobber should pass
result = runner.invoke(
cli, ["retrieve-artifacts", self.root, destdir, "--clobber"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
class PruneCollectionsTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testPruneCollections(self):
"""Test removing a collection and run from a repository using the
butler prune-collection subcommand."""
with self.runner.isolated_filesystem():
repoName = "myRepo"
runName = "myRun"
taggedName = "taggedCollection"
# Add the run and the tagged collection to the repo:
result = self.runner.invoke(butlerCli, ["create", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Use the butler initalizer to create the run, then create a tagged
# collection.
butler = Butler(repoName, run=runName)
butler.registry.registerCollection(taggedName,
CollectionType.TAGGED)
# Verify the run and tag show up in query-collections:
result = self.runner.invoke(butlerCli,
["query-collections", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(runName, result.output)
self.assertIn(taggedName, result.output)
# Verify the tagged collection can be removed:
result = self.runner.invoke(
butlerCli,
["prune-collection", repoName, taggedName, "--unstore"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(butlerCli,
["query-collections", repoName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(runName, result.output)
self.assertNotIn(taggedName, result.output)
# Verify the run can be removed:
result = self.runner.invoke(butlerCli, [
"prune-collection", repoName, runName, "--purge", "--unstore"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertNotIn(runName, result.output)
self.assertNotIn(taggedName, result.output)
def _registerInstrument(cls):
"""Use the Click `testing` module to call the butler command line api
to register the instrument."""
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
["register-instrument", cls.root, cls.instrumentClassName])
# Classmethod so assertEqual does not work
assert result.exit_code == 0, f"output: {result.output} exception: {result.exception}"
def _writeCuratedCalibrations(self):
"""Use the Click `testing` module to call the butler command line api
to write curated calibrations."""
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
["write-curated-calibrations", self.root, self.instrumentName])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
def testPhotodiode(self):
"""Test ingest to a repo with the exposure information will not raise.
"""
# Ingest raw to provide exposure information.
outputRun = "raw_ingest_" + self.id()
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
[
"ingest-raws",
self.root,
self.file,
"--output-run",
outputRun,
"--ingest-task",
self.rawIngestTask,
],
)
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
# Ingest photodiode matching this exposure.
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
[
"ingest-photodiode",
self.root,
self.instrumentClassName,
self.pdPath,
],
)
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
# Confirm that we can retrieve the ingested photodiode, and
# that it has the correct type.
butler = Butler(self.root, run="LSSTCam/calib/photodiode")
getResult = butler.get('photodiode', dataId=self.dataIds[0])
self.assertIsInstance(getResult, PhotodiodeCalib)
def testGetCollections(self):
run = "ingest/run"
tag = "ingest"
expected = {"collections": [run, tag]}
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
# the purpose of this call is to create some collections
_ = Butler(butlerCfg, run=run, tags=[tag], collections=[tag])
result = runner.invoke(cli, ["query-collections", "here"])
self.assertEqual(expected, yaml.safe_load(result.output))
class MWOptionTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def test_addElipsisToMultiple(self):
"""Verify that MWOption adds elipsis to the option metavar when
`multiple=True`
The default behavior of click is to not add elipsis to options that
have `multiple=True`."""
@click.command()
@click.option("--things", cls=MWOption, multiple=True)
def cmd(things):
pass
result = self.runner.invoke(cmd, ["--help"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedOutut = """Options:
--things TEXT ..."""
self.assertIn(expectedOutut, result.output)
def test_addElipsisToNargs(self):
"""Verify that MWOption adds " ..." after the option metavar when
`nargs` is set to more than 1 and less than 1.
The default behavior of click is to add elipsis when nargs does not
equal 1, but it does not put a space before the elipsis and we prefer
a space between the metavar and the elipsis."""
for numberOfArgs in (0, 1, 2): # nargs must be >= 0 for an option
@click.command()
@click.option("--things", cls=MWOption, nargs=numberOfArgs)
def cmd(things):
pass
result = self.runner.invoke(cmd, ["--help"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedOutut = f"""Options:
--things TEXT{' ...' if numberOfArgs != 1 else ''}"""
self.assertIn(expectedOutut, result.output)
class QueryCollectionsScriptTest(ButlerTestHelper, unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testGetCollections(self):
run = "ingest/run"
tag = "tag"
with self.runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
# the purpose of this call is to create some collections
butler = Butler(butlerCfg,
run=run,
collections=[tag],
writeable=True)
butler.registry.registerCollection(tag, CollectionType.TAGGED)
# Verify collections that were created are found by
# query-collections.
result = self.runner.invoke(cli, ["query-collections", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("ingest/run", "tag"), ("RUN", "TAGGED")),
names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Verify that with a glob argument, that only collections whose
# name matches with the specified pattern are returned.
result = self.runner.invoke(cli,
["query-collections", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("tag", ), ("TAGGED", )), names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Verify that with a collection type argument, only collections of
# that type are returned.
result = self.runner.invoke(
cli, ["query-collections", "here", "--collection-type", "RUN"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = Table((("ingest/run", ), ("RUN", )),
names=("Name", "Type"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = {"datasetTypes": [datasetName]}
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertEqual(expectedNotVerbose, yaml.safe_load(result.output))
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertEqual(expectedNotVerbose, yaml.safe_load(result.output))
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
response = yaml.safe_load(result.output)
# output dimension names contain all required dimensions, more than
# the registered dimensions, so verify the expected components
# individually.
self.assertEqual(response["datasetTypes"][0]["name"], datasetName)
self.assertEqual(response["datasetTypes"][0]["storageClass"],
storageClassName)
self.assertIn(instrumentDimension,
response["datasetTypes"][0]["dimensions"])
self.assertIn(visitDimension,
response["datasetTypes"][0]["dimensions"])
def setUp(self):
"""Setup for lightweight photodiode ingest task.
This will create the repo and register the instrument.
"""
self.root = tempfile.mkdtemp(dir=self.ingestDir)
# Create Repo
runner = LogCliRunner()
result = runner.invoke(butlerCli, ["create", self.root])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
# Register Instrument
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
["register-instrument", self.root, self.instrumentClassName])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
def testRetrieveSubset(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
destdir = "tmp1/"
result = runner.invoke(cli, [
"retrieve-artifacts", self.root, destdir, "--where",
"instrument='DummyCamComp' AND visit=423"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 3\n"),
f"Expected 3 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 3,
f"Expected 3 artifacts: {artifacts}")
def test_cli(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
result = runner.invoke(butler.cli, ["create", "here"])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
registerInstrumentArgs = [
"register-instrument", "here", self.instrumentClassName
]
if self.secondInstrumentClassName is not None:
registerInstrumentArgs.append(self.secondInstrumentClassName)
result = runner.invoke(butler.cli, registerInstrumentArgs)
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
result = runner.invoke(butler.cli, [
"write-curated-calibrations", "here", self.instrumentName,
"--collection", "collection"
])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
class RegisterSkymapConfigTest(unittest.TestCase):
def setUp(self):
self.runner = LogCliRunner()
def testNoConfigOverride(self):
"""Verify expected arguments are passed to makeSkyMap with no config
overrides."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch(
"lsst.pipe.tasks.script.registerSkymap.makeSkyMap"
) as mock:
# call without any config overrides
result = self.runner.invoke(butlerCli,
["register-skymap", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedConfig = registerSkymap.MakeSkyMapConfig()
mock.assert_called_once()
# assert that the first argument to the call to makeSkyMap was a butler
self.assertIsInstance(mock.call_args[0][0], Butler)
# assert that the second argument matches the expected config
self.assertEqual(mock.call_args[0][1], expectedConfig)
def testConfigOverride(self):
"""Verify expected arguments are passed to makeSkyMap with config
overrides."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
with unittest.mock.patch(
"lsst.pipe.tasks.script.registerSkymap.makeSkyMap"
) as mock:
# call and override the name parameter of the config
result = self.runner.invoke(
butlerCli,
["register-skymap", "repo", "--config", "name=bar"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expectedConfig = registerSkymap.MakeSkyMapConfig()
expectedConfig.update(name="bar")
mock.assert_called_once()
# assert that the first argument to the makeSkyMap call was a butler
self.assertIsInstance(mock.call_args[0][0], Butler)
# assert that the second argument matches the expected config
self.assertEqual(mock.call_args[0][1], expectedConfig)
def testNonExistantConfigFile(self):
"""Verify an expected error when a given config override file does not
exist. """
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", "repo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butlerCli,
["register-skymap", "repo", "--config-file", "foo.py"])
# foo.py does not exist; exit could should be non-zero.
self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
class RegisterDcrSubfiltersTest(unittest.TestCase, ButlerTestHelper):
def setUp(self):
self.runner = LogCliRunner()
self.repo = "here"
def testRegisterFilters(self):
"""Register a few filters and verify they are added to the repo."""
with self.runner.isolated_filesystem():
result = self.runner.invoke(butlerCli, ["create", self.repo])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "3", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(
AstropyTable((("foo", "foo", "foo"), (0, 1, 2)),
names=("band", "id")), readTable(result.output))
# Verify expected output message for registering subfilters in a
# band that already has subfilters
result = self.runner.invoke(
butlerCli, ["register-dcr-subfilters", self.repo, "5", "foo"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
# Add subfilters for two filters, one new filter and one existing.
# Verify expected result messages and registry values.
result = self.runner.invoke(
butlerCli,
["register-dcr-subfilters", self.repo, "3", "foo", "bar"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn(
registerDcrSubfilters.notRegisteredMsg.format(
band="foo", subfilters="[0, 1, 2]"), result.output)
self.assertIn(
registerDcrSubfilters.registeredMsg.format(
band="bar", subfilters="[0, 1, 2]"), result.output)
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.repo, "subfilter"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
resultTable = readTable(result.output)
resultTable.sort(["band", "id"])
self.assertAstropyTablesEqual(
AstropyTable((("bar", "bar", "bar", "foo", "foo", "foo"),
(0, 1, 2, 0, 1, 2)),
names=("band", "id")), resultTable)
def testPhotodiodeFailure(self):
"""Test ingest to a repo missing exposure information will raise.
"""
runner = LogCliRunner()
result = runner.invoke(
butlerCli,
[
"ingest-photodiode",
self.root,
self.instrumentClassName,
self.pdPath,
],
)
self.assertEqual(
result.exit_code, 1,
f"output: {result.output} exception: {result.exception}")
def testQueryDatasetTypes(self):
self.maxDiff = None
datasetName = "test"
instrumentDimension = "instrument"
visitDimension = "visit"
storageClassName = "testDatasetType"
expectedNotVerbose = AstropyTable((("test", ), ), names=("name", ))
runner = LogCliRunner()
with runner.isolated_filesystem():
butlerCfg = Butler.makeRepo("here")
butler = Butler(butlerCfg, writeable=True)
storageClass = StorageClass(storageClassName)
butler.registry.storageClasses.registerStorageClass(storageClass)
dimensions = butler.registry.dimensions.extract(
(instrumentDimension, visitDimension))
datasetType = DatasetType(datasetName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
# check not-verbose output:
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check glob output:
result = runner.invoke(cli, ["query-dataset-types", "here", "t*"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertAstropyTablesEqual(readTable(result.output),
expectedNotVerbose)
# check verbose output:
result = runner.invoke(
cli, ["query-dataset-types", "here", "--verbose"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
expected = AstropyTable(array((
"test",
"['band', 'instrument', 'physical_filter', 'visit_system', 'visit']",
"testDatasetType")),
names=("name", "dimensions",
"storage class"))
self.assertAstropyTablesEqual(readTable(result.output), expected)
# Now remove and check that it was removed
# First a non-existent one
result = runner.invoke(cli,
["remove-dataset-type", "here", "unreal"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Now one we now has been registered
result = runner.invoke(
cli, ["remove-dataset-type", "here", datasetName])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# and check that it has gone
result = runner.invoke(cli, ["query-dataset-types", "here"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertIn("No results", result.output)
def test_help(self):
"""Tests `utils.addArgumentHelp` and its use in repo_argument and
directory_argument; verifies that the argument help gets added to the
command fucntion help, and that it's added in the correct order. See
addArgumentHelp for more details."""
runner = LogCliRunner()
result = runner.invoke(ArgumentHelpGeneratorTestCase.cli, ["--help"])
expected = """Usage: cli [OPTIONS] REPO DIRECTORY
The cli help message.
repo help text
directory help text
Options:
--help Show this message and exit.
"""
self.assertIn(expected, result.output)
def testRetrieveAll(self):
runner = LogCliRunner()
with runner.isolated_filesystem():
# When preserving the path the run will be in the directory along
# with a . in the component name. When not preserving paths the
# filename will have an underscore rather than dot.
for counter, (preserve_path, prefix) in enumerate(
(("--preserve-path", "ingest/run/test_metric_comp."),
("--no-preserve-path", "test_metric_comp_"))):
destdir = f"tmp{counter}/"
result = runner.invoke(
cli,
["retrieve-artifacts", self.root, destdir, preserve_path])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
self.assertTrue(result.stdout.endswith(": 6\n"),
f"Expected 6 got: {result.stdout}")
artifacts = self.find_files(destdir)
self.assertEqual(len(artifacts), 6,
f"Expected 6 artifacts: {artifacts}")
self.assertIn(f"{destdir}{prefix}", str(artifacts[1]))
def _ingestRaws(self, transfer, file=None):
"""Use the Click `testing` module to call the butler command line api
to ingest raws.
Parameters
----------
transfer : `str`
The external data transfer type.
file : `str`
Path to a file to ingest instead of the default associated with
the object.
"""
if file is None:
file = self.file
runner = LogCliRunner()
result = runner.invoke(butlerCli, [
"ingest-raws", self.root, file, "--output-run", self.outputRun,
"--transfer", transfer, "--ingest-task", self.rawIngestTask
])
self.assertEqual(
result.exit_code, 0,
f"output: {result.output} exception: {result.exception}")
class QueryDimensionRecordsTest(unittest.TestCase, ButlerTestHelper):
mockFuncName = "lsst.daf.butler.cli.cmd.commands.script.queryDimensionRecords"
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
storageClassFactory = StorageClassFactory()
expectedColumnNames = ("instrument", "id", "physical_filter",
"visit_system", "name", "day_obs", "exposure_time",
"target_name", "observation_reason",
"science_program", "zenith_angle", "region",
"timespan [2]")
def setUp(self):
self.root = makeTestTempDir(TESTDIR)
self.testRepo = MetricTestRepo(self.root,
configFile=os.path.join(
TESTDIR,
"config/basic/butler.yaml"))
self.runner = LogCliRunner()
def tearDown(self):
removeTestTempDir(self.root)
def testBasic(self):
result = self.runner.invoke(
butlerCli, ["query-dimension-records", self.root, "visit"])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array(
(("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None",
"None", "None", "None", "None", "None", "None", "None .. None"),
("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None",
"None", "None", "None", "None", "None", "None", "None .. None")))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testWhere(self):
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--where",
"instrument='DummyCamComp' AND visit.name='fourtwentythree'"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "423", "d-r", "1", "fourtwentythree",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def testCollection(self):
butler = Butler(self.root, run="foo")
# try replacing the testRepo's butler with the one with the "foo" run.
self.testRepo.butler = butler
self.testRepo.butler.registry.insertDimensionData(
"visit", {
"instrument": "DummyCamComp",
"id": 425,
"name": "fourtwentyfive",
"physical_filter": "d-r",
"visit_system": 1
})
self.testRepo.addDataset(dataId={
"instrument": "DummyCamComp",
"visit": 425
},
run="foo")
# verify getting records from the "ingest/run" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"ingest/run", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array(
(("DummyCamComp", "423", "d-r", "1", "fourtwentythree", "None",
"None", "None", "None", "None", "None", "None", "None .. None"),
("DummyCamComp", "424", "d-r", "1", "fourtwentyfour", "None",
"None", "None", "None", "None", "None", "None", "None .. None")))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
# verify getting records from the "foo" collection
result = self.runner.invoke(butlerCli, [
"query-dimension-records", self.root, "visit", "--collections",
"foo", "--datasets", "test_metric_comp"
])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
rows = array((("DummyCamComp", "425", "d-r", "1", "fourtwentyfive",
"None", "None", "None", "None", "None", "None", "None",
"None .. None"), ))
expected = AstropyTable(rows, names=self.expectedColumnNames)
self.assertAstropyTablesEqual(readTable(result.output), expected)
def run_test(self,
mockPruneDatasets,
mockQueryDatasets_init,
mockQueryDatasets_getDatasets,
mockQueryDatasets_getTables,
cliArgs,
exMsgs,
exPruneDatasetsCallArgs,
exGetTablesCalled,
exQueryDatasetsCallArgs,
invokeInput=None,
exPruneDatasetsExitCode=0):
"""Execute the test.
Makes a temporary repo, invokes ``prune-datasets``. Verifies expected
output, exit codes, and mock calls.
Parameters
----------
mockPruneDatasets : `MagicMock`
The MagicMock for the ``Butler.pruneDatasets`` function.
mockQueryDatasets_init : `MagicMock`
The MagicMock for the ``QueryDatasets.__init__`` function.
mockQueryDatasets_getDatasets : `MagicMock`
The MagicMock for the ``QueryDatasets.getDatasets`` function.
mockQueryDatasets_getTables : `MagicMock`
The MagicMock for the ``QueryDatasets.getTables`` function.
cliArgs : `list` [`str`]
The arguments to pass to the command line. Do not include the
subcommand name or the repo.
exMsgs : `list` [`str`] or None
A list of text fragments that should appear in the text output
after calling the CLI command, or None if no output should be
produced.
exPruneDatasetsCallArgs : `dict` [`str`, `Any`]
The arguments that ``Butler.pruneDatasets`` should have been called
with, or None if that function should not have been called.
exGetTablesCalled : bool
`True` if ``QueryDatasets.getTables`` should have been called, else
`False`.
exQueryDatasetsCallArgs : `dict` [`str`, `Any`]
The arguments that ``QueryDatasets.__init__`` should have bene
called with, or `None` if the function should not have been called.
invokeInput : `str`, optional.
As string to pass to the ``CliRunner.invoke`` `input` argument. By
default None.
exPruneDatasetsExitCode : `int`
The expected exit code returned from invoking ``prune-datasets``.
"""
runner = LogCliRunner()
with runner.isolated_filesystem():
# Make a repo so a butler can be created
result = runner.invoke(butlerCli, ["create", self.repo])
self.assertEqual(result.exit_code, 0, clickResultMsg(result))
# Run the prune-datasets CLI command, this will call all of our
# mocks:
cliArgs = ["prune-datasets", self.repo] + cliArgs
result = runner.invoke(butlerCli, cliArgs, input=invokeInput)
self.assertEqual(result.exit_code, exPruneDatasetsExitCode, clickResultMsg(result))
# Verify the Butler.pruneDatasets was called exactly once with
# expected arguments. The datasets argument is the value returned
# by QueryDatasets, which we've mocked with side effect
# ``getDatasets()``.
if exPruneDatasetsCallArgs:
mockPruneDatasets.assert_called_once_with(**exPruneDatasetsCallArgs)
else:
mockPruneDatasets.assert_not_called()
# Less critical, but do a quick verification that the QueryDataset
# member function mocks were called, in this case we expect one
# time each.
if exQueryDatasetsCallArgs:
mockQueryDatasets_init.assert_called_once_with(**exQueryDatasetsCallArgs)
else:
mockQueryDatasets_init.assert_not_called()
# If Butler.pruneDatasets was not called, then
# QueryDatasets.getDatasets also does not get called.
if exPruneDatasetsCallArgs:
mockQueryDatasets_getDatasets.assert_called_once()
else:
mockQueryDatasets_getDatasets.assert_not_called()
if exGetTablesCalled:
mockQueryDatasets_getTables.assert_called_once()
else:
mockQueryDatasets_getTables.assert_not_called()
if exMsgs is None:
self.assertEqual("", result.output)
else:
for expectedMsg in exMsgs:
self.assertIn(expectedMsg, result.output)
