def setUp(self): """Create a new butler root for each test.""" if self.useTempRoot: self.root = tempfile.mkdtemp(dir=TESTDIR) Butler.makeRepo(self.root, config=Config(self.configFile)) self.tmpConfigFile = os.path.join(self.root, "butler.yaml") else: self.root = None self.tmpConfigFile = self.configFile
def testMakeRepo(self): """Test that we can write butler configuration to a new repository via the Butler.makeRepo interface and then instantiate a butler from the repo root. """ # Do not run the test if we know this datastore configuration does # not support a file system root if self.fullConfigKey is None: return # Remove the file created in setUp os.unlink(self.tmpConfigFile) butlerConfig = Butler.makeRepo(self.root, config=Config(self.configFile)) limited = Config(self.configFile) butler1 = Butler(butlerConfig) butlerConfig = Butler.makeRepo(self.root, standalone=True, createRegistry=False, config=Config(self.configFile)) full = Config(self.tmpConfigFile) butler2 = Butler(butlerConfig) # Butlers should have the same configuration regardless of whether # defaults were expanded. self.assertEqual(butler1._config, butler2._config) # Config files loaded directly should not be the same. self.assertNotEqual(limited, full) # Make sure "limited" doesn't have a few keys we know it should be # inheriting from defaults. self.assertIn(self.fullConfigKey, full) self.assertNotIn(self.fullConfigKey, limited) # Collections don't appear until something is put in them collections1 = butler1.registry.getAllCollections() self.assertEqual(collections1, set()) self.assertEqual(butler2.registry.getAllCollections(), collections1) # Check that a config with no associated file name will not # work properly with relocatable Butler repo butlerConfig.configFile = None with self.assertRaises(ValueError): Butler(butlerConfig)
def testEscape(self): c = Config({"a": {"foo.bar": 1}, "b😂c": {"bar_baz": 2}}) self.assertEqual(c[r".a.foo\.bar"], 1) self.assertEqual(c[":a:foo.bar"], 1) self.assertEqual(c[".b😂c.bar_baz"], 2) self.assertEqual(c[r"😂b\😂c😂bar_baz"], 2) self.assertEqual(c[r"\a\foo.bar"], 1) self.assertEqual(c["\ra\rfoo.bar"], 1) with self.assertRaises(ValueError): c[".a.foo\\.bar\r"]
def testPickle(self): """Test pickle support. """ self.root = tempfile.mkdtemp(dir=TESTDIR) Butler.makeRepo(self.root, config=Config(self.configFile)) self.tmpConfigFile = os.path.join(self.root, "butler.yaml") butler = Butler(self.tmpConfigFile, run="ingest") butlerOut = pickle.loads(pickle.dumps(butler)) self.assertIsInstance(butlerOut, Butler) self.assertEqual(butlerOut.config, butler.config)
def testIncludeConfigs(self): """Test the special includeConfigs key for pulling in additional files.""" c = Config(os.path.join(self.configDir, "configIncludes.yaml")) self.assertEqual(c["comp", "item2"], "hello") self.assertEqual(c["comp", "item50"], 5000) self.assertEqual(c["comp", "item1"], "first") self.assertEqual(c["comp", "item10"], "tenth") self.assertEqual(c["comp", "item11"], "eleventh") self.assertEqual(c["unrelated"], 1) self.assertEqual(c["addon", "comp", "item1"], "posix") self.assertEqual(c["addon", "comp", "item11"], -1) self.assertEqual(c["addon", "comp", "item50"], 500) c = Config(os.path.join(self.configDir, "configIncludes.json")) self.assertEqual(c["comp", "item2"], "hello") self.assertEqual(c["comp", "item50"], 5000) self.assertEqual(c["comp", "item1"], "first") self.assertEqual(c["comp", "item10"], "tenth") self.assertEqual(c["comp", "item11"], "eleventh") self.assertEqual(c["unrelated"], 1) self.assertEqual(c["addon", "comp", "item1"], "posix") self.assertEqual(c["addon", "comp", "item11"], -1) self.assertEqual(c["addon", "comp", "item50"], 500) # Now test with an environment variable in includeConfigs with modified_environment(SPECIAL_BUTLER_DIR=self.configDir3): c = Config(os.path.join(self.configDir, "configIncludesEnv.yaml")) self.assertEqual(c["comp", "item2"], "hello") self.assertEqual(c["comp", "item50"], 5000) self.assertEqual(c["comp", "item1"], "first") self.assertEqual(c["comp", "item10"], "tenth") self.assertEqual(c["comp", "item11"], "eleventh") self.assertEqual(c["unrelated"], 1) self.assertEqual(c["addon", "comp", "item1"], "envvar") self.assertEqual(c["addon", "comp", "item11"], -1) self.assertEqual(c["addon", "comp", "item50"], 501) # This will fail with modified_environment(SPECIAL_BUTLER_DIR=self.configDir2): with self.assertRaises(FileNotFoundError): Config(os.path.join(self.configDir, "configIncludesEnv.yaml"))
def testDict(self): """Test toDict()""" c1 = Config({"a": {"b": 1}, "c": 2}) self.assertIsInstance(c1["a"], Config) d1 = c1.toDict() self.assertIsInstance(d1["a"], dict) self.assertEqual(d1["a"], c1["a"]) # Modifying one does not change the other d1["a"]["c"] = 2 self.assertNotEqual(d1["a"], c1["a"])
def _makeArgs(registryConfig=None, **kwargs): """Return parsed command line arguments. By default butler_config is set to `Config` populated with some defaults, it can be overridden completely by keyword argument. Parameters ---------- cmd : `str`, optional Produce arguments for this pipetask command. registryConfig : `RegistryConfig`, optional Override for registry configuration. **kwargs Overrides for other arguments. """ # Use a mock to get the default value of arguments to 'run'. mock = unittest.mock.Mock() @click.command(cls=PipetaskCommand) @run_options() def fake_run(ctx, **kwargs): """Fake "pipetask run" command for gathering input arguments. The arguments & options should always match the arguments & options in the "real" command function `lsst.ctrl.mpexec.cli.cmd.run`. """ mock(**kwargs) runner = click.testing.CliRunner() result = runner.invoke(fake_run) if result.exit_code != 0: raise RuntimeError( f"Failure getting default args from 'fake_run': {result}") mock.assert_called_once() args = mock.call_args[1] args["enableLsstDebug"] = args.pop("debug") if "pipeline_actions" not in args: args["pipeline_actions"] = [] args = SimpleNamespace(**args) # override butler_config with our defaults args.butler_config = Config() if registryConfig: args.butler_config["registry"] = registryConfig # The default datastore has a relocatable root, so we need to specify # some root here for it to use args.butler_config.configFile = "." # override arguments from keyword parameters for key, value in kwargs.items(): setattr(args, key, value) return args
def obscore_export( repo: str, destination: str, config: str, format: str, where: Optional[str], collections: Iterable[str], dataset_type: Iterable[str], ) -> None: """Export Butler datasets as ObsCore Data Model in parquet format. Parameters ---------- repo : `str` URI to the butler repository. destination : `str` Location of the output file. config : `str` Location of the configuration file. format : `str` Output format, 'csv' or 'parquet' where : `str` Optional user expression, if provided overrides one in ``config``. collections : `iterable` [ `str` ] Optional collection names, if provided overrides one in ``config``. """ butler = Butler(repo, writeable=False) config_data = Config(config) cfg = ExporterConfig.parse_obj(config_data) if where: cfg.where = where if collections: cfg.collections = list(collections) if dataset_type: dataset_type_set = set(dataset_type) # Check that configuration has all requested dataset types. if not dataset_type_set.issubset(cfg.dataset_types): extras = dataset_type_set - set(cfg.dataset_types) raise ValueError(f"Dataset types {extras} are not defined in configuration file.") # Remove dataset types that are not needed. cfg.dataset_types = { key: value for key, value in cfg.dataset_types.items() if key in dataset_type_set } exporter = ObscoreExporter(butler, cfg) if format == "parquet": exporter.to_parquet(destination) elif format == "csv": exporter.to_csv(destination) else: raise ValueError(f"Unexpected output format {format:r}")
def testIncludeConfigs(self): """Test the special includeConfigs key for pulling in additional files.""" c = Config(os.path.join(self.configDir, "configIncludes.yaml")) self.assertEqual(c["comp", "item2"], "hello") self.assertEqual(c["comp", "item50"], 5000) self.assertEqual(c["comp", "item1"], "first") self.assertEqual(c["comp", "item10"], "tenth") self.assertEqual(c["comp", "item11"], "eleventh") self.assertEqual(c["unrelated"], 1) self.assertEqual(c["addon", "comp", "item1"], "posix") self.assertEqual(c["addon", "comp", "item11"], -1) self.assertEqual(c["addon", "comp", "item50"], 500)
def testRegistryConfig(self): configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") config = Config(configFile) universe = DimensionUniverse.fromConfig() self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) # Create a DatasetRef with and without instrument matching the # one in the config file. dimensions = universe.extract( ("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPviHscFmt = self.factory.getFormatter(refPviHsc) self.assertIsInstance(refPviHscFmt, Formatter) self.assertIn("JsonFormatter", refPviHscFmt.name()) refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, { "instrument": "DummyNotHSC", "physical_filter": "v" }) refPviNotHscFmt = self.factory.getFormatter(refPviNotHsc) self.assertIsInstance(refPviNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPviNotHscFmt.name()) # Create a DatasetRef that should fall back to using Dimensions refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscFmt = self.factory.getFormatter(refPvixHsc) self.assertIsInstance(refPvixNotHscFmt, Formatter) self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) # Create a DatasetRef that should fall back to using StorageClass dimensionsNoV = universe.extract(("physical_filter", "instrument")) refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, { "instrument": "DummyHSC", "physical_filter": "v" }) refPvixNotHscDims_fmt = self.factory.getFormatter(refPvixNotHscDims) self.assertIsInstance(refPvixNotHscDims_fmt, Formatter) self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name())
def testResource(self): c = Config("resource://lsst.daf.butler/configs/datastore.yaml") self.assertIn("datastore", c) # Test that we can include a resource URI yaml = """ toplevel: true resource: !include resource://lsst.daf.butler/configs/datastore.yaml """ c = Config.fromYaml(yaml) self.assertIn(("resource", "datastore", "cls"), c) # Test that we can include a resource URI with includeConfigs yaml = """ toplevel: true resource: includeConfigs: resource://lsst.daf.butler/configs/datastore.yaml """ c = Config.fromYaml(yaml) self.assertIn(("resource", "datastore", "cls"), c)
def testUpdate(self): c = Config({"a": {"b": 1}}) c.update({"a": {"c": 2}}) self.assertEqual(c[".a.b"], 1) self.assertEqual(c[".a.c"], 2) c.update({"a": {"d": [3, 4]}}) self.assertEqual(c[".a.d.0"], 3) c.update({"z": [5, 6, {"g": 2, "h": 3}]}) self.assertEqual(c[".z.1"], 6) # This is detached from parent c2 = c[".z.2"] self.assertEqual(c2["g"], 2) c2.update({"h": 4, "j": 5}) self.assertEqual(c2["h"], 4) self.assertNotIn(".z.2.j", c) self.assertNotEqual(c[".z.2.h"], 4) with self.assertRaises(RuntimeError): c.update([1, 2, 3])
def _setupNewButler(butler: Butler, outputLocation: ResourcePath, dirExists: bool) -> Butler: # Set up the new butler object at the specified location if dirExists: # Remove the existing table, if the code got this far and this exists # clobber must be true executionRegistry = outputLocation.join("gen3.sqlite3") if executionRegistry.exists(): executionRegistry.remove() else: outputLocation.mkdir() # Copy the existing butler config, modifying the location of the # registry to the specified location. # Preserve the root path from the existing butler so things like # file data stores continue to look at the old location. config = Config(butler._config) config["root"] = outputLocation.geturl() config["allow_put_of_predefined_dataset"] = True config["registry", "db"] = "sqlite:///<butlerRoot>/gen3.sqlite3" # Remove any namespace that may be set in main registry. config.pop(("registry", "namespace"), None) # record the current root of the datastore if it is specified relative # to the butler root if config.get(("datastore", "root")) == BUTLER_ROOT_TAG and butler._config.configDir is not None: config["datastore", "root"] = butler._config.configDir.geturl() config["datastore", "trust_get_request"] = True # Requires that we use the dimension configuration from the original # butler and not use the defaults. config = Butler.makeRepo( root=outputLocation, config=config, dimensionConfig=butler.registry.dimensions.dimensionConfig, overwrite=True, forceConfigRoot=False, ) # Return a newly created butler return Butler(config, writeable=True)
def testInclude(self): """Read a config that has an include directive""" c = Config(os.path.join(self.configDir, "testinclude.yaml")) self.assertEqual(c[".comp1.item1"], 58) self.assertEqual(c[".comp2.comp.item1"], 1) self.assertEqual(c[".comp3.1.comp.item1"], "posix") self.assertEqual(c[".comp4.0.comp.item1"], "posix") self.assertEqual(c[".comp4.1.comp.item1"], 1) self.assertEqual(c[".comp5.comp6.comp.item1"], "posix") # Test a specific name and then test that all # returned names are "in" the config. names = c.names() self.assertIn(c._D.join(("", "comp3", "1", "comp", "item1")), names) for n in names: self.assertIn(n, c) # Test that override delimiter works delimiter = "-" names = c.names(delimiter=delimiter) self.assertIn(delimiter.join(("", "comp3", "1", "comp", "item1")), names)
def setUp(self): config = Config(self.configFile) uri = ButlerURI(config[".datastore.datastore.root"]) self.bucketName = uri.netloc if self.useTempRoot: self.root = self.genRoot() rooturi = f"s3://{self.bucketName}/{self.root}" config.update({"datastore": {"datastore": {"root": rooturi}}}) # set up some fake credentials if they do not exist self.usingDummyCredentials = setAwsEnvCredentials() # MOTO needs to know that we expect Bucket bucketname to exist # (this used to be the class attribute bucketName) s3 = boto3.resource("s3") s3.create_bucket(Bucket=self.bucketName) self.datastoreStr = f"datastore={self.root}" self.datastoreName = [f"S3Datastore@{rooturi}"] Butler.makeRepo(rooturi, config=config, forceConfigRoot=False) self.tmpConfigFile = os.path.join(rooturi, "butler.yaml")
def makeButlerRepo(root, config=None, standalone=False, override=False, outfile=None): """Make a new Butler repository. Parameters ---------- root : `str` Location to seed a butler repository. config : `str`, optional Configuration to seed the repository. standalone : `bool`, optional If `True` a fully expanded configuration will be written. override : `bool`, optional If `True` a root provided in the supplied config will not be overwritten. outfile : `str`, optional Path to output configuration. This can be left `None` if the configuration is to be written to ``root``. """ forceConfigRoot = not override config = Config(config) if config is not None else None Butler.makeRepo(root, config=config, standalone=standalone, forceConfigRoot=forceConfigRoot, outfile=outfile)
parser.add_argument( "--outfile", "-f", default=None, type=str, help="Name of output file to receive repository configuration." " Default is to write butler.yaml into the specified root.") parser.add_argument("--verbose", "-v", action="store_true", help="Turn on debug reporting.") parser.add_argument( "--override", "-o", action="store_true", help= "Allow values in the supplied config to override any root settings.") args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) forceConfigRoot = not args.override config = Config(args.config) if args.config is not None else None Butler.makeRepo(args.root, config=config, standalone=args.standalone, forceConfigRoot=forceConfigRoot, outfile=args.outfile)
#!/usr/bin/env python import sys import getpass from lsst.daf.butler import Butler, Config username = getpass.getuser() config = Config() config[ ".registry.db"] = f"postgresql://{username}@lsst-pg-prod1.ncsa.illinois.edu:5432/lsstdb1" config[".registry.namespace"] = username root = sys.argv[1] Butler.makeRepo(root, config=config)
def setUp(self): config = Config( { "version": 1, "namespace": "pipe_base_test", "skypix": { "common": "htm7", "htm": { "class": "lsst.sphgeom.HtmPixelization", "max_level": 24, }, }, "elements": { "A": { "keys": [ { "name": "id", "type": "int", } ], "storage": { "cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage", }, }, "B": { "keys": [ { "name": "id", "type": "int", } ], "storage": { "cls": "lsst.daf.butler.registry.dimensions.table.TableDimensionRecordStorage", }, }, }, "packers": {}, } ) universe = DimensionUniverse(config=config) # need to make a mapping of TaskDef to set of quantum quantumMap = {} tasks = [] for task, label in ( (Dummy1PipelineTask, "R"), (Dummy2PipelineTask, "S"), (Dummy3PipelineTask, "T"), (Dummy4PipelineTask, "U"), ): config = task.ConfigClass() taskDef = TaskDef(get_full_type_name(task), config, task, label) tasks.append(taskDef) quantumSet = set() connections = taskDef.connections for a, b in ((1, 2), (3, 4)): if connections.initInputs: initInputDSType = DatasetType( connections.initInput.name, tuple(), storageClass=connections.initInput.storageClass, universe=universe, ) initRefs = [DatasetRef(initInputDSType, DataCoordinate.makeEmpty(universe))] else: initRefs = None inputDSType = DatasetType( connections.input.name, connections.input.dimensions, storageClass=connections.input.storageClass, universe=universe, ) inputRefs = [ DatasetRef(inputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe)) ] outputDSType = DatasetType( connections.output.name, connections.output.dimensions, storageClass=connections.output.storageClass, universe=universe, ) outputRefs = [ DatasetRef(outputDSType, DataCoordinate.standardize({"A": a, "B": b}, universe=universe)) ] quantumSet.add( Quantum( taskName=task.__qualname__, dataId=DataCoordinate.standardize({"A": a, "B": b}, universe=universe), taskClass=task, initInputs=initRefs, inputs={inputDSType: inputRefs}, outputs={outputDSType: outputRefs}, ) ) quantumMap[taskDef] = quantumSet self.tasks = tasks self.quantumMap = quantumMap self.qGraph = QuantumGraph(quantumMap, metadata=METADATA) self.universe = universe
def makeTestRepo(root, dataIds, *, config=None, **kwargs): """Create an empty repository with dummy data IDs. Parameters ---------- root : `str` The location of the root directory for the repository. dataIds : `~collections.abc.Mapping` [`str`, `iterable`] A mapping keyed by the dimensions used in the test. Each value is an iterable of names for that dimension (e.g., detector IDs for `"detector"`). Related dimensions (e.g., instruments and detectors) are linked arbitrarily. config : `lsst.daf.butler.Config`, optional A configuration for the repository (for details, see `lsst.daf.butler.Butler.makeRepo`). If omitted, creates a repository with default dataset and storage types, but optimized for speed. The defaults set ``.datastore.cls``, ``.datastore.checksum`` and ``.registry.db``. If a supplied config does not specify these values the internal defaults will be used to ensure that we have a usable configuration. **kwargs Extra arguments to `lsst.daf.butler.Butler.makeRepo`. Returns ------- butler : `lsst.daf.butler.Butler` A Butler referring to the new repository. This Butler is provided only for additional setup; to keep test cases isolated, it is highly recommended that each test create its own Butler with a unique run/collection. See `makeTestCollection`. Notes ----- This function provides a "quick and dirty" repository for simple unit tests that don't depend on complex data relationships. Because it assigns dimension relationships and other metadata abitrarily, it is ill-suited for tests where the structure of the data matters. If you need such a dataset, create it directly or use a saved test dataset. Since the values in ``dataIds`` uniquely determine the repository's data IDs, the fully linked IDs can be recovered by calling `expandUniqueId`, so long as no other code has inserted dimensions into the repository registry. """ defaults = Config() defaults[ "datastore", "cls"] = "lsst.daf.butler.datastores.inMemoryDatastore.InMemoryDatastore" defaults["datastore", "checksum"] = False # In case of future changes defaults["registry", "db"] = "sqlite:///:memory:" if config: defaults.update(config) # Disable config root by default so that our registry override will # not be ignored. # newConfig guards against location-related keywords like outfile newConfig = Butler.makeRepo(root, config=defaults, forceConfigRoot=False, **kwargs) butler = Butler(newConfig, writeable=True) dimensionRecords = _makeRecords(dataIds, butler.registry.dimensions) for dimension, records in dimensionRecords.items(): butler.registry.insertDimensionData(dimension, *records) return butler
def testBadConfig(self): for badArg in ([], "file.fits"): with self.assertRaises(RuntimeError): Config(badArg)
def testHierarchy(self): c = Config() # Simple dict c["a"] = {"z": 52, "x": "string"} self.assertIn(".a.z", c) self.assertEqual(c[".a.x"], "string") # Try different delimiters self.assertEqual(c["⇛a⇛z"], 52) self.assertEqual(c[("a", "z")], 52) self.assertEqual(c["a", "z"], 52) c[".b.new.thing1"] = "thing1" c[".b.new.thing2"] = "thing2" c[".b.new.thing3.supp"] = "supplemental" self.assertEqual(c[".b.new.thing1"], "thing1") tmp = c[".b.new"] self.assertEqual(tmp["thing2"], "thing2") self.assertEqual(c[".b.new.thing3.supp"], "supplemental") # Test that we can index into lists c[".a.b.c"] = [1, "7", 3, {"1": 4, "5": "Five"}, "hello"] self.assertIn(".a.b.c.3.5", c) self.assertNotIn(".a.b.c.10", c) self.assertNotIn(".a.b.c.10.d", c) self.assertEqual(c[".a.b.c.3.5"], "Five") # Is the value in the list? self.assertIn(".a.b.c.hello", c) self.assertNotIn(".a.b.c.hello.not", c) # And assign to an element in the list self.assertEqual(c[".a.b.c.1"], "7") c[".a.b.c.1"] = 8 self.assertEqual(c[".a.b.c.1"], 8) self.assertIsInstance(c[".a.b.c"], collections.abc.Sequence) # Test we do get lists back from asArray a = c.asArray(".a.b.c") self.assertIsInstance(a, list) # Is it the *same* list as in the config a.append("Sentinel") self.assertIn("Sentinel", c[".a.b.c"]) self.assertIn(".a.b.c.Sentinel", c) # Test we always get a list for k in c.names(): a = c.asArray(k) self.assertIsInstance(a, list) # Check we get the same top level keys self.assertEqual(set(c.names(topLevelOnly=True)), set(c._data.keys())) # Check that we can iterate through items for k, v in c.items(): self.assertEqual(c[k], v) # Check that lists still work even if assigned a dict c = Config({ "cls": "lsst.daf.butler", "formatters": { "calexp.wcs": "{component}", "calexp": "{datasetType}" }, "datastores": [{ "datastore": { "cls": "datastore1" } }, { "datastore": { "cls": "datastore2" } }] }) c[".datastores.1.datastore"] = {"cls": "datastore2modified"} self.assertEqual(c[".datastores.0.datastore.cls"], "datastore1") self.assertEqual(c[".datastores.1.datastore.cls"], "datastore2modified") self.assertIsInstance(c["datastores"], collections.abc.Sequence) # Test that we can get all the listed names. # and also that they are marked as "in" the Config # Try delimited names and tuples for n in itertools.chain(c.names(), c.nameTuples()): val = c[n] self.assertIsNotNone(val) self.assertIn(n, c) names = c.names() nameTuples = c.nameTuples() self.assertEqual(len(names), len(nameTuples)) self.assertEqual(len(names), 11) self.assertEqual(len(nameTuples), 11) # Test that delimiter escaping works names = c.names(delimiter=".") for n in names: self.assertIn(n, c) self.assertIn(".formatters.calexp\\.wcs", names) # Use a name that includes the internal default delimiter # to test automatic adjustment of delimiter strangeKey = f"calexp{c._D}wcs" c["formatters", strangeKey] = "dynamic" names = c.names() self.assertIn(strangeKey, "-".join(names)) self.assertFalse(names[0].startswith(c._D)) for n in names: self.assertIn(n, c) top = c.nameTuples(topLevelOnly=True) self.assertIsInstance(top[0], tuple) # Investigate a possible delimeter in a key c = Config({"formatters": {"calexp.wcs": 2, "calexp": 3}}) self.assertEqual(c[":formatters:calexp.wcs"], 2) self.assertEqual(c[":formatters:calexp"], 3) for k, v in c["formatters"].items(): self.assertEqual(c["formatters", k], v) # Check internal delimiter inheritance c._D = "." c2 = c["formatters"] self.assertEqual(c._D, c2._D) # Check that the child inherits self.assertNotEqual(c2._D, Config._D)
def testRegistryConfig(self): configFile = os.path.join(TESTDIR, "config", "basic", "posixDatastore.yaml") config = Config(configFile) universe = DimensionUniverse() self.factory.registerFormatters(config["datastore", "formatters"], universe=universe) # Create a DatasetRef with and without instrument matching the # one in the config file. dimensions = universe.extract(("visit", "physical_filter", "instrument")) sc = StorageClass("DummySC", dict, None) refPviHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyHSC", "physical_filter": "v"}, conform=False) refPviHscFmt = self.factory.getFormatterClass(refPviHsc) self.assertIsFormatter(refPviHscFmt) self.assertIn("JsonFormatter", refPviHscFmt.name()) refPviNotHsc = self.makeDatasetRef("pvi", dimensions, sc, {"instrument": "DummyNotHSC", "physical_filter": "v"}, conform=False) refPviNotHscFmt = self.factory.getFormatterClass(refPviNotHsc) self.assertIsFormatter(refPviNotHscFmt) self.assertIn("PickleFormatter", refPviNotHscFmt.name()) # Create a DatasetRef that should fall back to using Dimensions refPvixHsc = self.makeDatasetRef("pvix", dimensions, sc, {"instrument": "DummyHSC", "physical_filter": "v"}, conform=False) refPvixNotHscFmt = self.factory.getFormatterClass(refPvixHsc) self.assertIsFormatter(refPvixNotHscFmt) self.assertIn("PickleFormatter", refPvixNotHscFmt.name()) # Create a DatasetRef that should fall back to using StorageClass dimensionsNoV = DimensionGraph(universe, names=("physical_filter", "instrument")) refPvixNotHscDims = self.makeDatasetRef("pvix", dimensionsNoV, sc, {"instrument": "DummyHSC", "physical_filter": "v"}, conform=False) refPvixNotHscDims_fmt = self.factory.getFormatterClass(refPvixNotHscDims) self.assertIsFormatter(refPvixNotHscDims_fmt) self.assertIn("YamlFormatter", refPvixNotHscDims_fmt.name()) # Check that parameters are stored refParam = self.makeDatasetRef("paramtest", dimensions, sc, {"instrument": "DummyNotHSC", "physical_filter": "v"}, conform=False) lookup, refParam_fmt, kwargs = self.factory.getFormatterClassWithMatch(refParam) self.assertIn("writeParameters", kwargs) expected = {"max": 5, "min": 2, "comment": "Additional commentary", "recipe": "recipe1"} self.assertEqual(kwargs["writeParameters"], expected) self.assertIn("FormatterTest", refParam_fmt.name()) f = self.factory.getFormatter(refParam, self.fileDescriptor) self.assertEqual(f.writeParameters, expected) f = self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"min": 22, "extra": 50}) self.assertEqual(f.writeParameters, {"max": 5, "min": 22, "comment": "Additional commentary", "extra": 50, "recipe": "recipe1"}) self.assertIn("recipe1", f.writeRecipes) self.assertEqual(f.writeParameters["recipe"], "recipe1") with self.assertRaises(ValueError): # "new" is not allowed as a write parameter self.factory.getFormatter(refParam, self.fileDescriptor, writeParameters={"new": 1}) with self.assertRaises(RuntimeError): # "mode" is a required recipe parameter self.factory.getFormatter(refParam, self.fileDescriptor, writeRecipes={"recipe3": {"notmode": 1}})
def makeSimpleQGraph(nQuanta=5, pipeline=None, butler=None, root=None, skipExisting=False, inMemory=True, userQuery=""): """Make simple QuantumGraph for tests. Makes simple one-task pipeline with AddTask, sets up in-memory registry and butler, fills them with minimal data, and generates QuantumGraph with all of that. Parameters ---------- nQuanta : `int` Number of quanta in a graph. pipeline : `~lsst.pipe.base.Pipeline` If `None` then one-task pipeline is made with `AddTask` and default `AddTaskConfig`. butler : `~lsst.daf.butler.Butler`, optional Data butler instance, this should be an instance returned from a previous call to this method. root : `str` Path or URI to the root location of the new repository. Only used if ``butler`` is None. skipExisting : `bool`, optional If `True` (default), a Quantum is not created if all its outputs already exist. inMemory : `bool`, optional If true make in-memory repository. userQuery : `str`, optional The user query to pass to ``makeGraph``, by default an empty string. Returns ------- butler : `~lsst.daf.butler.Butler` Butler instance qgraph : `~lsst.pipe.base.QuantumGraph` Quantum graph instance """ if pipeline is None: pipeline = makeSimplePipeline(nQuanta=nQuanta) if butler is None: if root is None: raise ValueError("Must provide `root` when `butler` is None") config = Config() if not inMemory: config["registry", "db"] = f"sqlite:///{root}/gen3.sqlite" config[ "datastore", "cls"] = "lsst.daf.butler.datastores.posixDatastore.PosixDatastore" repo = butlerTests.makeTestRepo(root, {}, config=config) collection = "test" butler = Butler(butler=repo, run=collection) # Add dataset types to registry registerDatasetTypes(butler.registry, pipeline.toExpandedPipeline()) instrument = pipeline.getInstrument() if instrument is not None: if isinstance(instrument, str): instrument = doImport(instrument) instrumentName = instrument.getName() else: instrumentName = "INSTR" # Add all needed dimensions to registry butler.registry.insertDimensionData("instrument", dict(name=instrumentName)) butler.registry.insertDimensionData( "detector", dict(instrument=instrumentName, id=0, full_name="det0")) # Add inputs to butler data = numpy.array([0., 1., 2., 5.]) butler.put(data, "add_dataset0", instrument=instrumentName, detector=0) # Make the graph builder = pipeBase.GraphBuilder(registry=butler.registry, skipExisting=skipExisting) qgraph = builder.makeGraph(pipeline, collections=[butler.run], run=butler.run, userQuery=userQuery) return butler, qgraph
def testBasics(self): c = Config({"1": 2, "3": 4, "key3": 6, "dict": {"a": 1, "b": 2}}) pretty = c.ppprint() self.assertIn("key3", pretty) r = repr(c) self.assertIn("key3", r) regex = r"^Config\(\{.*\}\)$" self.assertRegex(r, regex) c2 = eval(r) self.assertIn("1", c) for n in c.names(): self.assertEqual(c2[n], c[n]) self.assertEqual(c, c2) s = str(c) self.assertIn("\n", s) self.assertNotRegex(s, regex) self.assertCountEqual(c.keys(), ["1", "3", "key3", "dict"]) self.assertEqual(list(c), list(c.keys())) self.assertEqual(list(c.values()), [c[k] for k in c.keys()]) self.assertEqual(list(c.items()), [(k, c[k]) for k in c.keys()]) newKeys = ("key4", ".dict.q", ("dict", "r"), "5") oldKeys = ("key3", ".dict.a", ("dict", "b"), "3") remainingKey = "1" # Check get with existing key for k in oldKeys: self.assertEqual(c.get(k, "missing"), c[k]) # Check get, pop with nonexistent key for k in newKeys: self.assertEqual(c.get(k, "missing"), "missing") self.assertEqual(c.pop(k, "missing"), "missing") # Check setdefault with existing key for k in oldKeys: c.setdefault(k, 8) self.assertNotEqual(c[k], 8) # Check setdefault with nonexistent key (mutates c, adding newKeys) for k in newKeys: c.setdefault(k, 8) self.assertEqual(c[k], 8) # Check pop with existing key (mutates c, removing newKeys) for k in newKeys: v = c[k] self.assertEqual(c.pop(k, "missing"), v) # Check deletion (mutates c, removing oldKeys) for k in ("key3", ".dict.a", ("dict", "b"), "3"): self.assertIn(k, c) del c[k] self.assertNotIn(k, c) # Check that `dict` still exists, but is now empty (then remove # it, mutatic c) self.assertIn("dict", c) del c["dict"] # Check popitem (mutates c, removing remainingKey) v = c[remainingKey] self.assertEqual(c.popitem(), (remainingKey, v)) # Check that c is now empty self.assertFalse(c)
# GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. __all__ = ("Gen3ButlerWrapper", "walk", "write") import os from lsst.utils import getPackageDir from lsst.daf.butler import Butler, ButlerConfig, Registry, Datastore, Config, StorageClassFactory from lsst.daf.butler.gen2convert import ConversionWalker, ConversionWriter REPO_ROOT = os.path.join(getPackageDir("ci_hsc"), "DATA") converterConfig = Config( os.path.join(getPackageDir("daf_butler"), "config/gen2convert.yaml")) converterConfig["skymaps"] = { "ci_hsc": os.path.join(REPO_ROOT, "rerun", "ci_hsc") } converterConfig["regions"][0]["collection"] = "shared/ci_hsc" searchPaths = [ os.path.join(getPackageDir("ci_hsc"), "gen3config"), ] class Gen3ButlerWrapper: """A class to simplify access to a gen 3 `~lsst.daf.butler.Butler`. Parameters ----------