def setUp(self):
"""Create a new butler root for each test."""
if self.useTempRoot:
self.root = tempfile.mkdtemp(dir=TESTDIR)
Butler.makeRepo(self.root, config=Config(self.configFile))
self.tmpConfigFile = os.path.join(self.root, "butler.yaml")
else:
self.root = None
self.tmpConfigFile = self.configFile
def testMakeRepo(self):
"""Test that we can write butler configuration to a new repository via
the Butler.makeRepo interface and then instantiate a butler from the
repo root.
"""
# Do not run the test if we know this datastore configuration does
# not support a file system root
if self.fullConfigKey is None:
return
# Remove the file created in setUp
os.unlink(self.tmpConfigFile)
butlerConfig = Butler.makeRepo(self.root,
config=Config(self.configFile))
limited = Config(self.configFile)
butler1 = Butler(butlerConfig)
butlerConfig = Butler.makeRepo(self.root,
standalone=True,
createRegistry=False,
config=Config(self.configFile))
full = Config(self.tmpConfigFile)
butler2 = Butler(butlerConfig)
# Butlers should have the same configuration regardless of whether
# defaults were expanded.
self.assertEqual(butler1._config, butler2._config)
# Config files loaded directly should not be the same.
self.assertNotEqual(limited, full)
# Make sure "limited" doesn't have a few keys we know it should be
# inheriting from defaults.
self.assertIn(self.fullConfigKey, full)
self.assertNotIn(self.fullConfigKey, limited)
# Collections don't appear until something is put in them
collections1 = butler1.registry.getAllCollections()
self.assertEqual(collections1, set())
self.assertEqual(butler2.registry.getAllCollections(), collections1)
# Check that a config with no associated file name will not
# work properly with relocatable Butler repo
butlerConfig.configFile = None
with self.assertRaises(ValueError):
Butler(butlerConfig)
def testEscape(self):
c = Config({"a": {"foo.bar": 1}, "b😂c": {"bar_baz": 2}})
self.assertEqual(c[r".a.foo\.bar"], 1)
self.assertEqual(c[":a:foo.bar"], 1)
self.assertEqual(c[".b😂c.bar_baz"], 2)
self.assertEqual(c[r"😂b\😂c😂bar_baz"], 2)
self.assertEqual(c[r"\a\foo.bar"], 1)
self.assertEqual(c["\ra\rfoo.bar"], 1)
with self.assertRaises(ValueError):
c[".a.foo\\.bar\r"]
def testPickle(self):
"""Test pickle support.
"""
self.root = tempfile.mkdtemp(dir=TESTDIR)
Butler.makeRepo(self.root, config=Config(self.configFile))
self.tmpConfigFile = os.path.join(self.root, "butler.yaml")
butler = Butler(self.tmpConfigFile, run="ingest")
butlerOut = pickle.loads(pickle.dumps(butler))
self.assertIsInstance(butlerOut, Butler)
self.assertEqual(butlerOut.config, butler.config)
def testIncludeConfigs(self):
"""Test the special includeConfigs key for pulling in additional
files."""
c = Config(os.path.join(self.configDir, "configIncludes.yaml"))
self.assertEqual(c["comp", "item2"], "hello")
self.assertEqual(c["comp", "item50"], 5000)
self.assertEqual(c["comp", "item1"], "first")
self.assertEqual(c["comp", "item10"], "tenth")
self.assertEqual(c["comp", "item11"], "eleventh")
self.assertEqual(c["unrelated"], 1)
self.assertEqual(c["addon", "comp", "item1"], "posix")
self.assertEqual(c["addon", "comp", "item11"], -1)
self.assertEqual(c["addon", "comp", "item50"], 500)
c = Config(os.path.join(self.configDir, "configIncludes.json"))
self.assertEqual(c["comp", "item2"], "hello")
self.assertEqual(c["comp", "item50"], 5000)
self.assertEqual(c["comp", "item1"], "first")
self.assertEqual(c["comp", "item10"], "tenth")
self.assertEqual(c["comp", "item11"], "eleventh")
self.assertEqual(c["unrelated"], 1)
self.assertEqual(c["addon", "comp", "item1"], "posix")
self.assertEqual(c["addon", "comp", "item11"], -1)
self.assertEqual(c["addon", "comp", "item50"], 500)
# Now test with an environment variable in includeConfigs
with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3):
c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
self.assertEqual(c["comp", "item2"], "hello")
self.assertEqual(c["comp", "item50"], 5000)
self.assertEqual(c["comp", "item1"], "first")
self.assertEqual(c["comp", "item10"], "tenth")
self.assertEqual(c["comp", "item11"], "eleventh")
self.assertEqual(c["unrelated"], 1)
self.assertEqual(c["addon", "comp", "item1"], "envvar")
self.assertEqual(c["addon", "comp", "item11"], -1)
self.assertEqual(c["addon", "comp", "item50"], 501)
# This will fail
with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2):
with self.assertRaises(FileNotFoundError):
Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
def testDict(self):
"""Test toDict()"""
c1 = Config({"a": {"b": 1}, "c": 2})
self.assertIsInstance(c1["a"], Config)
d1 = c1.toDict()
self.assertIsInstance(d1["a"], dict)
self.assertEqual(d1["a"], c1["a"])
# Modifying one does not change the other
d1["a"]["c"] = 2
self.assertNotEqual(d1["a"], c1["a"])
def _makeArgs(registryConfig=None, **kwargs):
"""Return parsed command line arguments.
By default butler_config is set to `Config` populated with some defaults,
it can be overridden completely by keyword argument.
Parameters
----------
cmd : `str`, optional
Produce arguments for this pipetask command.
registryConfig : `RegistryConfig`, optional
Override for registry configuration.
**kwargs
Overrides for other arguments.
"""
# Use a mock to get the default value of arguments to 'run'.
mock = unittest.mock.Mock()
@click.command(cls=PipetaskCommand)
@run_options()
def fake_run(ctx, **kwargs):
"""Fake "pipetask run" command for gathering input arguments.
The arguments & options should always match the arguments & options in
the "real" command function `lsst.ctrl.mpexec.cli.cmd.run`.
"""
mock(**kwargs)
runner = click.testing.CliRunner()
result = runner.invoke(fake_run)
if result.exit_code != 0:
raise RuntimeError(
f"Failure getting default args from 'fake_run': {result}")
mock.assert_called_once()
args = mock.call_args[1]
args["enableLsstDebug"] = args.pop("debug")
if "pipeline_actions" not in args:
args["pipeline_actions"] = []
args = SimpleNamespace(**args)
# override butler_config with our defaults
args.butler_config = Config()
if registryConfig:
args.butler_config["registry"] = registryConfig
# The default datastore has a relocatable root, so we need to specify
# some root here for it to use
args.butler_config.configFile = "."
# override arguments from keyword parameters
for key, value in kwargs.items():
setattr(args, key, value)
return args
def obscore_export(
repo: str,
destination: str,
config: str,
format: str,
where: Optional[str],
collections: Iterable[str],
dataset_type: Iterable[str],
) -> None:
"""Export Butler datasets as ObsCore Data Model in parquet format.
Parameters
----------
repo : `str`
URI to the butler repository.
destination : `str`
Location of the output file.
config : `str`
Location of the configuration file.
format : `str`
Output format, 'csv' or 'parquet'
where : `str`
Optional user expression, if provided overrides one in ``config``.
collections : `iterable` [ `str` ]
Optional collection names, if provided overrides one in ``config``.
"""
butler = Butler(repo, writeable=False)
config_data = Config(config)
cfg = ExporterConfig.parse_obj(config_data)
if where:
cfg.where = where
if collections:
cfg.collections = list(collections)
if dataset_type:
dataset_type_set = set(dataset_type)
# Check that configuration has all requested dataset types.
if not dataset_type_set.issubset(cfg.dataset_types):
extras = dataset_type_set - set(cfg.dataset_types)
raise ValueError(f"Dataset types {extras} are not defined in configuration file.")
# Remove dataset types that are not needed.
cfg.dataset_types = {
key: value for key, value in cfg.dataset_types.items() if key in dataset_type_set
}
exporter = ObscoreExporter(butler, cfg)
if format == "parquet":
exporter.to_parquet(destination)
elif format == "csv":
exporter.to_csv(destination)
else:
raise ValueError(f"Unexpected output format {format:r}")
def testIncludeConfigs(self):
"""Test the special includeConfigs key for pulling in additional
files."""
c = Config(os.path.join(self.configDir, "configIncludes.yaml"))
self.assertEqual(c["comp", "item2"], "hello")
self.assertEqual(c["comp", "item50"], 5000)
self.assertEqual(c["comp", "item1"], "first")
self.assertEqual(c["comp", "item10"], "tenth")
self.assertEqual(c["comp", "item11"], "eleventh")
self.assertEqual(c["unrelated"], 1)
self.assertEqual(c["addon", "comp", "item1"], "posix")
self.assertEqual(c["addon", "comp", "item11"], -1)
self.assertEqual(c["addon", "comp", "item50"], 500)
def testRegistryConfig(self):
configFile = os.path.join(TESTDIR, "config", "basic",
"posixDatastore.yaml")
config = Config(configFile)
universe = DimensionUniverse.fromConfig()
self.factory.registerFormatters(config["datastore", "formatters"],
universe=universe)
# Create a DatasetRef with and without instrument matching the
# one in the config file.
dimensions = universe.extract(
("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPviHscFmt = self.factory.getFormatter(refPviHsc)
self.assertIsInstance(refPviHscFmt, Formatter)
self.assertIn("JsonFormatter", refPviHscFmt.name())
refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {
"instrument": "DummyNotHSC",
"physical_filter": "v"
})
refPviNotHscFmt = self.factory.getFormatter(refPviNotHsc)
self.assertIsInstance(refPviNotHscFmt, Formatter)
self.assertIn("PickleFormatter", refPviNotHscFmt.name())
# Create a DatasetRef that should fall back to using Dimensions
refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPvixNotHscFmt = self.factory.getFormatter(refPvixHsc)
self.assertIsInstance(refPvixNotHscFmt, Formatter)
self.assertIn("PickleFormatter", refPvixNotHscFmt.name())
# Create a DatasetRef that should fall back to using StorageClass
dimensionsNoV = universe.extract(("physical_filter", "instrument"))
refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {
"instrument": "DummyHSC",
"physical_filter": "v"
})
refPvixNotHscDims_fmt = self.factory.getFormatter(refPvixNotHscDims)
self.assertIsInstance(refPvixNotHscDims_fmt, Formatter)
self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
def testResource(self):
c = Config("resource://lsst.daf.butler/configs/datastore.yaml")
self.assertIn("datastore", c)
# Test that we can include a resource URI
yaml = """
toplevel: true
resource: !include resource://lsst.daf.butler/configs/datastore.yaml
"""
c = Config.fromYaml(yaml)
self.assertIn(("resource", "datastore", "cls"), c)
# Test that we can include a resource URI with includeConfigs
yaml = """
toplevel: true
resource:
includeConfigs: resource://lsst.daf.butler/configs/datastore.yaml
"""
c = Config.fromYaml(yaml)
self.assertIn(("resource", "datastore", "cls"), c)
def testUpdate(self):
c = Config({"a": {"b": 1}})
c.update({"a": {"c": 2}})
self.assertEqual(c[".a.b"], 1)
self.assertEqual(c[".a.c"], 2)
c.update({"a": {"d": [3, 4]}})
self.assertEqual(c[".a.d.0"], 3)
c.update({"z": [5, 6, {"g": 2, "h": 3}]})
self.assertEqual(c[".z.1"], 6)
# This is detached from parent
c2 = c[".z.2"]
self.assertEqual(c2["g"], 2)
c2.update({"h": 4, "j": 5})
self.assertEqual(c2["h"], 4)
self.assertNotIn(".z.2.j", c)
self.assertNotEqual(c[".z.2.h"], 4)
with self.assertRaises(RuntimeError):
c.update([1, 2, 3])
def _setupNewButler(butler: Butler, outputLocation: ResourcePath, dirExists: bool) -> Butler:
# Set up the new butler object at the specified location
if dirExists:
# Remove the existing table, if the code got this far and this exists
# clobber must be true
executionRegistry = outputLocation.join("gen3.sqlite3")
if executionRegistry.exists():
executionRegistry.remove()
else:
outputLocation.mkdir()
# Copy the existing butler config, modifying the location of the
# registry to the specified location.
# Preserve the root path from the existing butler so things like
# file data stores continue to look at the old location.
config = Config(butler._config)
config["root"] = outputLocation.geturl()
config["allow_put_of_predefined_dataset"] = True
config["registry", "db"] = "sqlite:///<butlerRoot>/gen3.sqlite3"
# Remove any namespace that may be set in main registry.
config.pop(("registry", "namespace"), None)
# record the current root of the datastore if it is specified relative
# to the butler root
if config.get(("datastore", "root")) == BUTLER_ROOT_TAG and butler._config.configDir is not None:
config["datastore", "root"] = butler._config.configDir.geturl()
config["datastore", "trust_get_request"] = True
# Requires that we use the dimension configuration from the original
# butler and not use the defaults.
config = Butler.makeRepo(
root=outputLocation,
config=config,
dimensionConfig=butler.registry.dimensions.dimensionConfig,
overwrite=True,
forceConfigRoot=False,
)
# Return a newly created butler
return Butler(config, writeable=True)
def testInclude(self):
"""Read a config that has an include directive"""
c = Config(os.path.join(self.configDir, "testinclude.yaml"))
self.assertEqual(c[".comp1.item1"], 58)
self.assertEqual(c[".comp2.comp.item1"], 1)
self.assertEqual(c[".comp3.1.comp.item1"], "posix")
self.assertEqual(c[".comp4.0.comp.item1"], "posix")
self.assertEqual(c[".comp4.1.comp.item1"], 1)
self.assertEqual(c[".comp5.comp6.comp.item1"], "posix")
# Test a specific name and then test that all
# returned names are "in" the config.
names = c.names()
self.assertIn(c._D.join(("", "comp3", "1", "comp", "item1")), names)
for n in names:
self.assertIn(n, c)
# Test that override delimiter works
delimiter = "-"
names = c.names(delimiter=delimiter)
self.assertIn(delimiter.join(("", "comp3", "1", "comp", "item1")), names)
def setUp(self):
config = Config(self.configFile)
uri = ButlerURI(config[".datastore.datastore.root"])
self.bucketName = uri.netloc
if self.useTempRoot:
self.root = self.genRoot()
rooturi = f"s3://{self.bucketName}/{self.root}"
config.update({"datastore": {"datastore": {"root": rooturi}}})
# set up some fake credentials if they do not exist
self.usingDummyCredentials = setAwsEnvCredentials()
# MOTO needs to know that we expect Bucket bucketname to exist
# (this used to be the class attribute bucketName)
s3 = boto3.resource("s3")
s3.create_bucket(Bucket=self.bucketName)
self.datastoreStr = f"datastore={self.root}"
self.datastoreName = [f"S3Datastore@{rooturi}"]
Butler.makeRepo(rooturi, config=config, forceConfigRoot=False)
self.tmpConfigFile = os.path.join(rooturi, "butler.yaml")
def makeButlerRepo(root, config=None, standalone=False, override=False, outfile=None):
"""Make a new Butler repository.
Parameters
----------
root : `str`
Location to seed a butler repository.
config : `str`, optional
Configuration to seed the repository.
standalone : `bool`, optional
If `True` a fully expanded configuration will be written.
override : `bool`, optional
If `True` a root provided in the supplied config will not be
overwritten.
outfile : `str`, optional
Path to output configuration. This can be left `None`
if the configuration is to be written to ``root``.
"""
forceConfigRoot = not override
config = Config(config) if config is not None else None
Butler.makeRepo(root, config=config, standalone=standalone, forceConfigRoot=forceConfigRoot,
outfile=outfile)
parser.add_argument(
"--outfile",
"-f",
default=None,
type=str,
help="Name of output file to receive repository configuration."
" Default is to write butler.yaml into the specified root.")
parser.add_argument("--verbose",
"-v",
action="store_true",
help="Turn on debug reporting.")
parser.add_argument(
"--override",
"-o",
action="store_true",
help=
"Allow values in the supplied config to override any root settings.")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
forceConfigRoot = not args.override
config = Config(args.config) if args.config is not None else None
Butler.makeRepo(args.root,
config=config,
standalone=args.standalone,
forceConfigRoot=forceConfigRoot,
outfile=args.outfile)
#!/usr/bin/env python
import sys
import getpass
from lsst.daf.butler import Butler, Config
username = getpass.getuser()
config = Config()
config[
".registry.db"] = f"postgresql://{username}@lsst-pg-prod1.ncsa.illinois.edu:5432/lsstdb1"
config[".registry.namespace"] = username
root = sys.argv[1]
Butler.makeRepo(root, config=config)
def setUp(self):
config = Config(
{
"version": 1,
"namespace": "pipe_base_test",
"skypix": {
"common": "htm7",
"htm": {
"class": "lsst.sphgeom.HtmPixelization",
"max_level": 24,
},
},
"elements": {
"A": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
"B": {
"keys": [
{
"name": "id",
"type": "int",
}
],
"storage": {
"cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage",
},
},
},
"packers": {},
}
)
universe = DimensionUniverse(config=config)
# need to make a mapping of TaskDef to set of quantum
quantumMap = {}
tasks = []
for task, label in (
(Dummy1PipelineTask, "R"),
(Dummy2PipelineTask, "S"),
(Dummy3PipelineTask, "T"),
(Dummy4PipelineTask, "U"),
):
config = task.ConfigClass()
taskDef = TaskDef(get_full_type_name(task), config, task, label)
tasks.append(taskDef)
quantumSet = set()
connections = taskDef.connections
for a, b in ((1, 2), (3, 4)):
if connections.initInputs:
initInputDSType = DatasetType(
connections.initInput.name,
tuple(),
storageClass=connections.initInput.storageClass,
universe=universe,
)
initRefs = [DatasetRef(initInputDSType, DataCoordinate.makeEmpty(universe))]
else:
initRefs = None
inputDSType = DatasetType(
connections.input.name,
connections.input.dimensions,
storageClass=connections.input.storageClass,
universe=universe,
)
inputRefs = [
DatasetRef(inputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
outputDSType = DatasetType(
connections.output.name,
connections.output.dimensions,
storageClass=connections.output.storageClass,
universe=universe,
)
outputRefs = [
DatasetRef(outputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe))
]
quantumSet.add(
Quantum(
taskName=task.__qualname__,
dataId=DataCoordinate.standardize({"A": a, "B": b}, universe=universe),
taskClass=task,
initInputs=initRefs,
inputs={inputDSType: inputRefs},
outputs={outputDSType: outputRefs},
)
)
quantumMap[taskDef] = quantumSet
self.tasks = tasks
self.quantumMap = quantumMap
self.qGraph = QuantumGraph(quantumMap, metadata=METADATA)
self.universe = universe
def makeTestRepo(root, dataIds, *, config=None, **kwargs):
"""Create an empty repository with dummy data IDs.
Parameters
----------
root : `str`
The location of the root directory for the repository.
dataIds : `~collections.abc.Mapping` [`str`, `iterable`]
A mapping keyed by the dimensions used in the test. Each value
is an iterable of names for that dimension (e.g., detector IDs for
`"detector"`). Related dimensions (e.g., instruments and detectors)
are linked arbitrarily.
config : `lsst.daf.butler.Config`, optional
A configuration for the repository (for details, see
`lsst.daf.butler.Butler.makeRepo`). If omitted, creates a repository
with default dataset and storage types, but optimized for speed.
The defaults set ``.datastore.cls``, ``.datastore.checksum`` and
``.registry.db``. If a supplied config does not specify these values
the internal defaults will be used to ensure that we have a usable
configuration.
**kwargs
Extra arguments to `lsst.daf.butler.Butler.makeRepo`.
Returns
-------
butler : `lsst.daf.butler.Butler`
A Butler referring to the new repository. This Butler is provided only
for additional setup; to keep test cases isolated, it is highly
recommended that each test create its own Butler with a
unique run/collection. See `makeTestCollection`.
Notes
-----
This function provides a "quick and dirty" repository for simple unit
tests that don't depend on complex data relationships. Because it assigns
dimension relationships and other metadata abitrarily, it is ill-suited
for tests where the structure of the data matters. If you need such a
dataset, create it directly or use a saved test dataset.
Since the values in ``dataIds`` uniquely determine the repository's
data IDs, the fully linked IDs can be recovered by calling
`expandUniqueId`, so long as no other code has inserted dimensions into
the repository registry.
"""
defaults = Config()
defaults[
"datastore",
"cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore"
defaults["datastore", "checksum"] = False # In case of future changes
defaults["registry", "db"] = "sqlite:///:memory:"
if config:
defaults.update(config)
# Disable config root by default so that our registry override will
# not be ignored.
# newConfig guards against location-related keywords like outfile
newConfig = Butler.makeRepo(root,
config=defaults,
forceConfigRoot=False,
**kwargs)
butler = Butler(newConfig, writeable=True)
dimensionRecords = _makeRecords(dataIds, butler.registry.dimensions)
for dimension, records in dimensionRecords.items():
butler.registry.insertDimensionData(dimension, *records)
return butler
def testBadConfig(self):
for badArg in ([], "file.fits"):
with self.assertRaises(RuntimeError):
Config(badArg)
def testHierarchy(self):
c = Config()
# Simple dict
c["a"] = {"z": 52, "x": "string"}
self.assertIn(".a.z", c)
self.assertEqual(c[".a.x"], "string")
# Try different delimiters
self.assertEqual(c["⇛a⇛z"], 52)
self.assertEqual(c[("a", "z")], 52)
self.assertEqual(c["a", "z"], 52)
c[".b.new.thing1"] = "thing1"
c[".b.new.thing2"] = "thing2"
c[".b.new.thing3.supp"] = "supplemental"
self.assertEqual(c[".b.new.thing1"], "thing1")
tmp = c[".b.new"]
self.assertEqual(tmp["thing2"], "thing2")
self.assertEqual(c[".b.new.thing3.supp"], "supplemental")
# Test that we can index into lists
c[".a.b.c"] = [1, "7", 3, {"1": 4, "5": "Five"}, "hello"]
self.assertIn(".a.b.c.3.5", c)
self.assertNotIn(".a.b.c.10", c)
self.assertNotIn(".a.b.c.10.d", c)
self.assertEqual(c[".a.b.c.3.5"], "Five")
# Is the value in the list?
self.assertIn(".a.b.c.hello", c)
self.assertNotIn(".a.b.c.hello.not", c)
# And assign to an element in the list
self.assertEqual(c[".a.b.c.1"], "7")
c[".a.b.c.1"] = 8
self.assertEqual(c[".a.b.c.1"], 8)
self.assertIsInstance(c[".a.b.c"], collections.abc.Sequence)
# Test we do get lists back from asArray
a = c.asArray(".a.b.c")
self.assertIsInstance(a, list)
# Is it the *same* list as in the config
a.append("Sentinel")
self.assertIn("Sentinel", c[".a.b.c"])
self.assertIn(".a.b.c.Sentinel", c)
# Test we always get a list
for k in c.names():
a = c.asArray(k)
self.assertIsInstance(a, list)
# Check we get the same top level keys
self.assertEqual(set(c.names(topLevelOnly=True)), set(c._data.keys()))
# Check that we can iterate through items
for k, v in c.items():
self.assertEqual(c[k], v)
# Check that lists still work even if assigned a dict
c = Config({
"cls":
"lsst.daf.butler",
"formatters": {
"calexp.wcs": "{component}",
"calexp": "{datasetType}"
},
"datastores": [{
"datastore": {
"cls": "datastore1"
}
}, {
"datastore": {
"cls": "datastore2"
}
}]
})
c[".datastores.1.datastore"] = {"cls": "datastore2modified"}
self.assertEqual(c[".datastores.0.datastore.cls"], "datastore1")
self.assertEqual(c[".datastores.1.datastore.cls"],
"datastore2modified")
self.assertIsInstance(c["datastores"], collections.abc.Sequence)
# Test that we can get all the listed names.
# and also that they are marked as "in" the Config
# Try delimited names and tuples
for n in itertools.chain(c.names(), c.nameTuples()):
val = c[n]
self.assertIsNotNone(val)
self.assertIn(n, c)
names = c.names()
nameTuples = c.nameTuples()
self.assertEqual(len(names), len(nameTuples))
self.assertEqual(len(names), 11)
self.assertEqual(len(nameTuples), 11)
# Test that delimiter escaping works
names = c.names(delimiter=".")
for n in names:
self.assertIn(n, c)
self.assertIn(".formatters.calexp\\.wcs", names)
# Use a name that includes the internal default delimiter
# to test automatic adjustment of delimiter
strangeKey = f"calexp{c._D}wcs"
c["formatters", strangeKey] = "dynamic"
names = c.names()
self.assertIn(strangeKey, "-".join(names))
self.assertFalse(names[0].startswith(c._D))
for n in names:
self.assertIn(n, c)
top = c.nameTuples(topLevelOnly=True)
self.assertIsInstance(top[0], tuple)
# Investigate a possible delimeter in a key
c = Config({"formatters": {"calexp.wcs": 2, "calexp": 3}})
self.assertEqual(c[":formatters:calexp.wcs"], 2)
self.assertEqual(c[":formatters:calexp"], 3)
for k, v in c["formatters"].items():
self.assertEqual(c["formatters", k], v)
# Check internal delimiter inheritance
c._D = "."
c2 = c["formatters"]
self.assertEqual(c._D, c2._D) # Check that the child inherits
self.assertNotEqual(c2._D, Config._D)
def testRegistryConfig(self):
configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml")
config = Config(configFile)
universe = DimensionUniverse()
self.factory.registerFormatters(config["datastore", "formatters"], universe=universe)
# Create a DatasetRef with and without instrument matching the
# one in the config file.
dimensions = universe.extract(("visit", "physical_filter", "instrument"))
sc = StorageClass("DummySC", dict, None)
refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC",
"physical_filter": "v"},
conform=False)
refPviHscFmt = self.factory.getFormatterClass(refPviHsc)
self.assertIsFormatter(refPviHscFmt)
self.assertIn("JsonFormatter", refPviHscFmt.name())
refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC",
"physical_filter": "v"},
conform=False)
refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc)
self.assertIsFormatter(refPviNotHscFmt)
self.assertIn("PickleFormatter", refPviNotHscFmt.name())
# Create a DatasetRef that should fall back to using Dimensions
refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC",
"physical_filter": "v"},
conform=False)
refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc)
self.assertIsFormatter(refPvixNotHscFmt)
self.assertIn("PickleFormatter", refPvixNotHscFmt.name())
# Create a DatasetRef that should fall back to using StorageClass
dimensionsNoV = DimensionGraph(universe, names=("physical_filter", "instrument"))
refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC",
"physical_filter": "v"},
conform=False)
refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims)
self.assertIsFormatter(refPvixNotHscDims_fmt)
self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
# Check that parameters are stored
refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC",
"physical_filter": "v"},
conform=False)
lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam)
self.assertIn("writeParameters", kwargs)
expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"}
self.assertEqual(kwargs["writeParameters"], expected)
self.assertIn("FormatterTest", refParam_fmt.name())
f = self.factory.getFormatter(refParam, self.fileDescriptor)
self.assertEqual(f.writeParameters, expected)
f = self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"min": 22,
"extra": 50})
self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary",
"extra": 50, "recipe": "recipe1"})
self.assertIn("recipe1", f.writeRecipes)
self.assertEqual(f.writeParameters["recipe"], "recipe1")
with self.assertRaises(ValueError):
# "new" is not allowed as a write parameter
self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"new": 1})
with self.assertRaises(RuntimeError):
# "mode" is a required recipe parameter
self.factory.getFormatter(refParam, self.fileDescriptor, writeRecipes={"recipe3": {"notmode": 1}})
def makeSimpleQGraph(nQuanta=5,
pipeline=None,
butler=None,
root=None,
skipExisting=False,
inMemory=True,
userQuery=""):
"""Make simple QuantumGraph for tests.
Makes simple one-task pipeline with AddTask, sets up in-memory
registry and butler, fills them with minimal data, and generates
QuantumGraph with all of that.
Parameters
----------
nQuanta : `int`
Number of quanta in a graph.
pipeline : `~lsst.pipe.base.Pipeline`
If `None` then one-task pipeline is made with `AddTask` and
default `AddTaskConfig`.
butler : `~lsst.daf.butler.Butler`, optional
Data butler instance, this should be an instance returned from a
previous call to this method.
root : `str`
Path or URI to the root location of the new repository. Only used if
``butler`` is None.
skipExisting : `bool`, optional
If `True` (default), a Quantum is not created if all its outputs
already exist.
inMemory : `bool`, optional
If true make in-memory repository.
userQuery : `str`, optional
The user query to pass to ``makeGraph``, by default an empty string.
Returns
-------
butler : `~lsst.daf.butler.Butler`
Butler instance
qgraph : `~lsst.pipe.base.QuantumGraph`
Quantum graph instance
"""
if pipeline is None:
pipeline = makeSimplePipeline(nQuanta=nQuanta)
if butler is None:
if root is None:
raise ValueError("Must provide `root` when `butler` is None")
config = Config()
if not inMemory:
config["registry", "db"] = f"sqlite:///{root}/gen3.sqlite"
config[
"datastore",
"cls"] = "lsst.daf.butler.datastores.posixDatastore.PosixDatastore"
repo = butlerTests.makeTestRepo(root, {}, config=config)
collection = "test"
butler = Butler(butler=repo, run=collection)
# Add dataset types to registry
registerDatasetTypes(butler.registry, pipeline.toExpandedPipeline())
instrument = pipeline.getInstrument()
if instrument is not None:
if isinstance(instrument, str):
instrument = doImport(instrument)
instrumentName = instrument.getName()
else:
instrumentName = "INSTR"
# Add all needed dimensions to registry
butler.registry.insertDimensionData("instrument",
dict(name=instrumentName))
butler.registry.insertDimensionData(
"detector", dict(instrument=instrumentName, id=0,
full_name="det0"))
# Add inputs to butler
data = numpy.array([0., 1., 2., 5.])
butler.put(data, "add_dataset0", instrument=instrumentName, detector=0)
# Make the graph
builder = pipeBase.GraphBuilder(registry=butler.registry,
skipExisting=skipExisting)
qgraph = builder.makeGraph(pipeline,
collections=[butler.run],
run=butler.run,
userQuery=userQuery)
return butler, qgraph
def testBasics(self):
c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}})
pretty = c.ppprint()
self.assertIn("key3", pretty)
r = repr(c)
self.assertIn("key3", r)
regex = r"^Config\(\{.*\}\)$"
self.assertRegex(r, regex)
c2 = eval(r)
self.assertIn("1", c)
for n in c.names():
self.assertEqual(c2[n], c[n])
self.assertEqual(c, c2)
s = str(c)
self.assertIn("\n", s)
self.assertNotRegex(s, regex)
self.assertCountEqual(c.keys(), ["1", "3", "key3", "dict"])
self.assertEqual(list(c), list(c.keys()))
self.assertEqual(list(c.values()), [c[k] for k in c.keys()])
self.assertEqual(list(c.items()), [(k, c[k]) for k in c.keys()])
newKeys = ("key4", ".dict.q", ("dict", "r"), "5")
oldKeys = ("key3", ".dict.a", ("dict", "b"), "3")
remainingKey = "1"
# Check get with existing key
for k in oldKeys:
self.assertEqual(c.get(k, "missing"), c[k])
# Check get, pop with nonexistent key
for k in newKeys:
self.assertEqual(c.get(k, "missing"), "missing")
self.assertEqual(c.pop(k, "missing"), "missing")
# Check setdefault with existing key
for k in oldKeys:
c.setdefault(k, 8)
self.assertNotEqual(c[k], 8)
# Check setdefault with nonexistent key (mutates c, adding newKeys)
for k in newKeys:
c.setdefault(k, 8)
self.assertEqual(c[k], 8)
# Check pop with existing key (mutates c, removing newKeys)
for k in newKeys:
v = c[k]
self.assertEqual(c.pop(k, "missing"), v)
# Check deletion (mutates c, removing oldKeys)
for k in ("key3", ".dict.a", ("dict", "b"), "3"):
self.assertIn(k, c)
del c[k]
self.assertNotIn(k, c)
# Check that `dict` still exists, but is now empty (then remove
# it, mutatic c)
self.assertIn("dict", c)
del c["dict"]
# Check popitem (mutates c, removing remainingKey)
v = c[remainingKey]
self.assertEqual(c.popitem(), (remainingKey, v))
# Check that c is now empty
self.assertFalse(c)
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
__all__ = ("Gen3ButlerWrapper", "walk", "write")
import os
from lsst.utils import getPackageDir
from lsst.daf.butler import Butler, ButlerConfig, Registry, Datastore, Config, StorageClassFactory
from lsst.daf.butler.gen2convert import ConversionWalker, ConversionWriter
REPO_ROOT = os.path.join(getPackageDir("ci_hsc"), "DATA")
converterConfig = Config(
os.path.join(getPackageDir("daf_butler"), "config/gen2convert.yaml"))
converterConfig["skymaps"] = {
"ci_hsc": os.path.join(REPO_ROOT, "rerun", "ci_hsc")
}
converterConfig["regions"][0]["collection"] = "shared/ci_hsc"
searchPaths = [
os.path.join(getPackageDir("ci_hsc"), "gen3config"),
]
class Gen3ButlerWrapper:
"""A class to simplify access to a gen 3 `~lsst.daf.butler.Butler`.
Parameters
----------