def parse_dataset_definition(config: DictConfig):
"""Parse and instantiate a dataset class using the configuration provided.
Parameters
__________
config: An omegaconf.dictconfig.DictConfig object
This is the dataset config dictionary. It must contain the `type` key
with fully qualified class name.
Raise
_____
DataSetError: If the function fails to parse the configuration provided.
Return
______
2-tuple: (Dataset class object, configuration dictionary)
"""
if "type" not in config:
raise DataSetError(
"`type` is missing from DataSet catalog configuration")
class_obj = config.pop("type")
if isinstance(class_obj, str):
if len(class_obj.strip(".")) != len(
class_obj): #check if starts or ends with a dot
raise DataSetError("`type` class path does not support relative "
"paths or paths ending with a dot.")
class_obj = load_obj(class_obj)
if not issubclass(class_obj, AbstractDataSet):
raise DataSetError(
f"DataSet type `{class_obj.__module__}.{class_obj.__qualname__}` "
f"is invalid: all data set types must extend `AbstractDataSet`.")
return class_obj, config
def merge_json(
doe_directory: Path = CONFIG["doe_directory"],
gds_directory: Path = CONFIG["gds_directory"],
extra_directories: Optional[Iterable[Path]] = None,
jsonpath: Path = CONFIG["mask_directory"] / "metadata.json",
json_version: int = 6,
config: DictConfig = TECH,
) -> Dict[str, Any]:
"""Combine several JSON files from config.yml
in the root of the mask directory, gets mask_name from there
Args:
doe_directory: defaults to current working directory
extra_directories: list of extra_directories
jsonpath
json_version:
config
"""
logger.debug("Merging JSON files:")
cells = {}
extra_directories = extra_directories or []
config = dataclasses.asdict(config)
config.pop("library", "")
for directory in extra_directories + [doe_directory]:
for filename in directory.glob("*/*.json"):
logger.debug(filename)
with open(filename, "r") as f:
data = json.load(f)
cells.update(data.get("cells"))
does = {
d.stem: json.loads(open(d).read())
for d in doe_directory.glob("*.json")
}
metadata = dict(
json_version=json_version,
cells=cells,
does=does,
config=config,
)
write_config(metadata, jsonpath)
logger.info(f"Wrote metadata in {jsonpath}")
return metadata
