def test_cache_remove_all():
config = Config("""{
"a": {
"b": {
"c": true
}
}
}""")
config.lookup(("a", "b", "c"), {}, remove=True)
assert not config.data
def test_cache_config():
config = Config("""{
"caching": {
"enabled": true,
"size_limit": 5e9,
"directory": "/tmp"
}
}""")
cache_config = config.lookup(("caching",), {})
assert cache_config["enabled"]
assert cache_config["size_limit"] == 5 * 10 ** 9
def test_lookup_deep():
config = Config(deep_str)
assert config.lookup(["a"]) == {"b": {"c": [1, 2, 3]}}
assert config.lookup(["a", "b"]) == {"c": [1, 2, 3]}
assert config.lookup(["a", "b", "c"]) == [1, 2, 3]
with raises(AttributeError):
config.lookup(["a", "b", "c", "d"])
assert config.lookup(["a", "b", "c", "d"], "x") == "x"
def test_cache_remove_step_wise():
config = Config("""{
"a": {
"b": {
"c": true,
"d": true
},
"e": 1
},
"f": 2
}""")
config.lookup(("a", "b", "c"), {}, remove=True)
assert "c" not in config.data["a"]["b"]
config.lookup(("a", "b"), {}, remove=True)
assert "b" not in config.data["a"]
config.lookup(("a", ), {}, remove=True)
assert "a" not in config.data
def from_json(cls,
json_url: str,
strict: bool = None,
config: Optional[Union[str, Dict]] = None) -> "Experiment":
"""
Construct an `Experiment` from an experiment.json file format specifier
Parameters
----------
json_url : str
file path or web link to an experiment.json file
strict : bool
if true, then all JSON loaded by this method will be
passed to the appropriate validator
config : str or dict
configuration property that will be passed to
starfish.util.config.Config
STARISH_CONFIG :
This parameter is read from the environment to permit setting configuration
values either directly or via a file. Keys read include:
- cache.allow_caching
STARFISH_STRICT_LOADING :
This parameter is read from the environment. If set, then all JSON loaded by this
method will be passed to the appropriate validator. The `strict` parameter to this
method has priority over the environment variable.
Returns
-------
Experiment :
Experiment object serving the requested experiment data
"""
if strict is None:
strict = "STARFISH_STRICT_LOADING" in os.environ
if strict:
valid = validate_sptx.validate(json_url)
if not valid:
raise Exception("validation failed")
config_obj = Config(config) # STARFISH_CONFIG is assumed
allow_caching = config_obj.lookup(["cache", "allow_caching"], True)
backend, name, baseurl = resolve_path_or_url(json_url, allow_caching)
with backend.read_contextmanager(name) as fh:
experiment_document = json.load(fh)
version = cls.verify_version(experiment_document['version'])
_, codebook_name, codebook_baseurl = resolve_url(
experiment_document['codebook'], baseurl, allow_caching)
codebook_absolute_url = pathjoin(codebook_baseurl, codebook_name)
codebook = Codebook.from_json(codebook_absolute_url)
extras = experiment_document['extras']
fovs: MutableSequence[FieldOfView] = list()
fov_tilesets: MutableMapping[str, TileSet] = dict()
if version < Version("5.0.0"):
primary_image: Collection = Reader.parse_doc(
experiment_document['primary_images'], baseurl)
auxiliary_images: MutableMapping[str, Collection] = dict()
for aux_image_type, aux_image_url in experiment_document[
'auxiliary_images'].items():
auxiliary_images[aux_image_type] = Reader.parse_doc(
aux_image_url, baseurl)
for fov_name, primary_tileset in primary_image.all_tilesets():
fov_tilesets[FieldOfView.PRIMARY_IMAGES] = primary_tileset
for aux_image_type, aux_image_collection in auxiliary_images.items(
):
aux_image_tileset = aux_image_collection.find_tileset(
fov_name)
if aux_image_tileset is not None:
fov_tilesets[aux_image_type] = aux_image_tileset
fov = FieldOfView(fov_name, image_tilesets=fov_tilesets)
fovs.append(fov)
else:
images: MutableMapping[str, Collection] = dict()
all_fov_names: MutableSet[str] = set()
for image_type, image_url in experiment_document['images'].items():
image = Reader.parse_doc(image_url, baseurl)
images[image_type] = image
for fov_name, _ in image.all_tilesets():
all_fov_names.add(fov_name)
for fov_name in all_fov_names:
for image_type, image_collection in images.items():
image_tileset = image_collection.find_tileset(fov_name)
if image_tileset is not None:
fov_tilesets[image_type] = image_tileset
fov = FieldOfView(fov_name, image_tilesets=fov_tilesets)
fovs.append(fov)
return Experiment(fovs, codebook, extras, src_doc=experiment_document)
def test_lookup_dne():
config = Config(simple_str)
with raises(KeyError):
config.lookup(["foo"])
assert config.lookup(["foo"], 1) == 1
assert config.lookup(["foo", "bar"], 2) == 2
class StarfishConfig(object):
"""
Application specific configuration settings which can be loaded throughout
the starfish codebase.
Attributes
----------
slicedimage : dictionary
Subdictionary that can be passed to slicedimage.io methods.
strict : bool
Whether or not loaded json should be validated.
verbose : bool
Controls output like from tqdm
Examples
--------
Check strict property
>>> from starfish.config import StarfishConfig
>>> config = StarfishConfig()
>>> if config.strict:
>>> validate(json)
Default starfish configuration equivalent:
>>> {
>>> "slicedimage": {
>>> "caching": {
>>> "debug": false,
>>> "directory": "~/.starfish/cache",
>>> "size_limit": 5e9
>>> },
>>> },
>>> "validation": {
>>> "strict": false
>>> },
>>> "verbose": true
>>> }
Example of a ~/.starfish.config file to disable caching:
>>> {
>>> "slicedimage": {
>>> "caching": {
>>> "size_limit": 0
>>> }
>>> }
>>> }
"""
def __init__(self) -> None:
"""
Loads the configuration specified by the STARFISH_CONFIG environment variable.
Parameters
----------
STARISH_CONFIG :
This parameter is read from the environment to permit setting configuration
values either directly or via a file. Keys read include:
- ["slicedimage"]["caching"]["directory"] (default: ~/.starfish/cache)
- ["slicedimage"]["caching"]["size_limit"] (default: None; 0 disables caching)
- ["validation"]["strict"] (default: False)
- ["verbose"] (default: True)
Note: all keys can also be set by and environment variable constructed from the
key parts and prefixed with STARFISH, e.g. STARFISH_VALIDATION_STRICT.
"""
config = os.environ.get("STARFISH_CONFIG", "@~/.starfish/config")
self._config_obj = Config(config)
self._env_keys = [
x for x in os.environ.keys()
if special_prefix(x) and x != "STARFISH_CONFIG"]
# If no directory is set, then force the default
self._slicedimage = self._config_obj.lookup(("slicedimage",), NestedDict(), remove=True)
if not self._slicedimage["caching"]["directory"]:
self._slicedimage["caching"]["directory"] = "~/.starfish/cache"
self._slicedimage_update(('caching', 'directory'))
self._slicedimage_update(('caching', 'size_limit'), int)
self._strict = self._config_obj.lookup(
("validation", "strict"), self.flag("STARFISH_VALIDATION_STRICT", "false"), remove=True)
self._verbose = self._config_obj.lookup(
("verbose",), self.flag("STARFISH_VERBOSE", "true"), remove=True)
if self._config_obj.data:
warnings.warn(f"unknown configuration: {self._config_obj.data}")
if self._env_keys:
warnings.warn(f"unknown environment variables: {self._env_keys}")
def _slicedimage_update(self, lookup, parse=lambda x: x):
""" accept STARFISH_SLICEDIMAGE_ or SLICEDIMAGE_ prefixes"""
value = None
name1 = "SLICEDIMAGE_" + "_".join([x.upper() for x in lookup])
if name1 in os.environ:
self._env_keys.remove(name1)
value = parse(os.environ[name1])
name2 = "STARFISH_" + name1
if name2 in os.environ:
if value:
warnings.warn(f"duplicate variable: (STARFISH_){name1}")
self._env_keys.remove(name2)
value = parse(os.environ[name2])
if value is None:
return # Nothing found
v = self._slicedimage
for k in lookup[:-1]:
v = v[k]
v[lookup[-1]] = value
def flag(self, name, default_value=""):
if name in os.environ:
value = os.environ[name]
self._env_keys.remove(name)
else:
value = default_value
if isinstance(value, str):
value = value.lower()
return value in ("true", "1", "yes", "y", "on", "active", "enabled")
@property
def slicedimage(self):
return dict(self._slicedimage)
@property
def strict(self):
return self._strict
@property
def verbose(self):
return self._verbose