def store_cfg(filename: Union[str, Path], d: dict): filename = str(filename) data = Box(d) if "yml" in filename or "yaml" in filename: Box(XConfig.decode(data.to_dict())).to_yaml(filename) if "json" in filename: Box(XConfig.decode(data.to_dict())).to_json(filename) if "toml" in filename: Box(XConfig.decode(data.to_dict())).to_toml(filename)
def compile( configuration_file: str, output_file: str, check: bool, options: Sequence[Tuple[str, str]], ): from choixe.configurations import XConfig import rich try: cfg = XConfig(filename=configuration_file) except Exception as e: rich.print(f"[red]Invalid configuration file: {e}[/red]") import sys sys.exit(1) placeholders = cfg.available_placeholders() for key, value in options: if key in placeholders: placeholder = placeholders[key] cfg.deep_set(key, placeholder.cast(value)) if check: cfg.check_available_placeholders(close_app=True) cfg.save_to(output_file)
def acquire( calibration: Path, device_cfg: Path, output_folder: Path, scale_factor: int, disparity: bool, max_disparity: int, max_depth: int, max_frames: int, skip: int, ): if device_cfg is None: device_cfg = oakeye.data_folder / "device" / "device.yml" device_cfg = XConfig(device_cfg) device = OakDeviceFactory().create(device_cfg) keys = ["left", "center", "right", "depth", "disparityCL", "disparityCR"] acquirer = DeviceAcquirer(device) if calibration is not None: calib = XConfig(calibration) acquirer = RectifiedAcquirer(acquirer, calib) if disparity: acquirer = DisparityAcquirer(acquirer, disp_diff=max_disparity) acquirer = GuiAcquirer( acquirer, keys, scale_factor=scale_factor, ranges={ "disparityCL": [0, max_disparity], "disparityCR": [0, max_disparity], "depth": [0, max_depth], }, ) dataset = acquirer(max_frames=max_frames, skip=skip) device.close() if output_folder is not None: ext_map = { "left": "png", "center": "png", "right": "png", "depth": "png", "disparityCL": "png", "disparityCR": "png", "device": "yml", } root_files = ["device"] for d in dataset: d["device"] = device_cfg.to_dict() UnderfolderWriter(output_folder, root_files_keys=root_files, extensions_map=ext_map)(SamplesSequence(dataset))
def test_prompt(self, monkeypatch, tmpdir_factory): d = { "a_int": "@int(a_int)", "another_int": "@int(a_int)", "a_float": "@float(a_float)", "a_bool": "@bool(a_bool)", "a_str1": "@str(a_str1)", "a_str2": "@str(a_str2,hello,hi)", "a_str3": "@str(a_str3,one,two,three,default=one)", } fills = { "a_int": 2, "another_int": 666, "a_float": 2.22, "a_bool": True, "a_str1": "hello", "a_str2": "hi", "a_str3": "three", } def custom_prompt(questions): return fills # old_prompt = XInquirer._system_prompt monkeypatch.setattr(XInquirer, "_system_prompt", custom_prompt, raising=True) # from dict xconfig = XConfig.from_dict(d) xconfig.set_schema(Schema({})) new_xconfig = XInquirer.prompt(xconfig, close_app=False) assert len(new_xconfig.available_placeholders()) == 0 assert xconfig._filename == new_xconfig._filename assert xconfig.get_schema() == new_xconfig.get_schema() # from file filename = tmpdir_factory.mktemp("configuration") / "cfg.yml" xconfig.save_to(filename) xconfig = XConfig(filename=filename) new_xconfig = XInquirer.prompt(xconfig, close_app=False) assert len(new_xconfig.available_placeholders()) == 0 assert xconfig._filename == new_xconfig._filename assert xconfig.get_schema() == new_xconfig.get_schema()
def test_deepset(self): cfg = { "l1": "L1", "l2": { "l2_0": "L20", "l2_1": "L21", }, "l3": { "l3_0": { "l3_0_0": "L300", "l3_0_1": "L301", "l3_0_2": "L302", } }, } xcfg = XConfig.from_dict(cfg) for k, old_value in xcfg.chunks_as_lists(): row_dict = {} pydash.set_(row_dict, k, old_value) xrow_dict = XConfig.from_dict(row_dict) for newk, newv in xrow_dict.chunks_as_lists(): # VALID KEYS # Try to replace present keys only with ONLY_VALID_KEYS = TRUE, should be equal! new_xcfg = xcfg.copy() new_xcfg.deep_set(newk, newv, only_valid_keys=True) assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict()) # Try to replace present keys only with ONLY_VALID_KEYS = FALSE, should be equal! new_xcfg = xcfg.copy() new_xcfg.deep_set(newk, newv, only_valid_keys=False) assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict()) # INVALID KEYS new_keys = ["xxIMPOSSIBLE_KEY"] * 10 # Try to replace not present keys but ONLY_VALID_KEYS = TRUE, should be equal! new_xcfg = xcfg.copy() new_xcfg.deep_set(new_keys, newv, only_valid_keys=True) assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict()) # Try to replace not present keys but ONLY_VALID_KEYS = False, should be different! new_xcfg = xcfg.copy() new_xcfg.deep_set(new_keys, newv, only_valid_keys=False) assert DeepDiff(xcfg.to_dict(), new_xcfg.to_dict())
def test_replace_ignore_defaults(self, generic_temp_folder, data): sample_dict = data["data"] placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data environment_variables = data["environment_variables"] conf = XConfig.from_dict(sample_dict) rich.print(conf) np.random.seed(66) to_replace = {} to_replace_with_defaults = {} for p in placeholders: _p = Placeholder.from_string(p) if _p.default_value is not None: to_replace_with_defaults[_p.name] = np.random.randint(0, 10) else: to_replace[_p.name] = np.random.randint(0, 10) rich.print("NAME", _p.name, _p.default_value is None) defaults_placeholders_counter = len(conf.available_placeholders()) - len( conf.available_placeholders(ignore_defaults=True) ) assert defaults_placeholders_counter == len(to_replace_with_defaults) if defaults_placeholders_counter > 0: conf_copy_A = conf.copy() conf_copy_A.replace_variables_map(to_replace, replace_defaults=False) conf_copy_B = conf.copy() conf_copy_B.replace_variables_map(to_replace, replace_defaults=True) assert len(conf_copy_A.available_placeholders()) > len( conf_copy_B.available_placeholders() )
def check(configuration_file, close): from choixe.configurations import XConfig import rich try: cfg = XConfig(filename=configuration_file) rich.print(cfg.to_dict()) except Exception as e: rich.print(f"[red]Invalid configuration file: {e}[/red]") import sys sys.exit(1) rich.print("[green]Configuration is good![/green]") # d = cfg.to_dict(discard_private_qualifiers=False) cfg.check_available_placeholders(close_app=close)
def test_nosweeps(self): root_cfg = { "first_level": 1, "second": { "alpha": [1, 2, 3], }, } cfg = XConfig.from_dict(root_cfg) assert len(cfg.sweep()) == 1
def create_from_file(cls, filename: str) -> any: """Creates an object starting from its serialized representation in a file loadable as XConfig (yml, json, etc.) :param filename: config filename :type filename: str :return: hydrated object :rtype: any """ return cls.create(XConfig(filename))
def test_xinquirer(self): assert XInquirer.safe_cast_placeholder(10, PlaceholderType.INT) assert XInquirer.safe_cast_placeholder(10, PlaceholderType.FLOAT) assert XInquirer.safe_cast_placeholder(10, PlaceholderType.STR) assert XInquirer.safe_cast_placeholder("10", PlaceholderType.INT) assert not XInquirer.safe_cast_placeholder("a string", PlaceholderType.INT) assert not XInquirer.safe_cast_placeholder("another string", PlaceholderType.FLOAT) assert XInquirer.safe_cast_placeholder(True, PlaceholderType.BOOL) assert XInquirer.safe_cast_placeholder(0, PlaceholderType.BOOL) assert XInquirer.safe_cast_placeholder("hello", PlaceholderType.PATH) assert XInquirer.safe_cast_placeholder("what time is it?", PlaceholderType.DATE) d = { "a_int": "@int(a_int)", "another_int": "@int(a_int)", "a_float": "@float(a_float)", "a_bool1": "@bool(a_bool1)", "a_bool2": "@bool(a_bool2, default=AA)", "a_bool3": "@bool(a_bool3, default=yEs)", "a_str1": "@str(a_str1)", "a_str2": "@str(a_str2,hello,hi)", "a_str3": "@str(a_str3,one,two,three,default=one)", } xconfig = XConfig.from_dict(d) placeholders = list(xconfig.available_placeholders().values()) for ph in placeholders: q = XInquirer.placeholder_to_question(ph) assert q.name == ph.name if ph.type == PlaceholderType.BOOL: assert isinstance(q, inquirer.Confirm) assert isinstance(q.default, bool) assert q.default == XInquirer.to_bool(ph.default_value) elif len(ph.options) > 0: assert isinstance(q, inquirer.List) assert len(q.choices) == len(ph.options) assert q.default == ph.default_value else: assert isinstance(q, inquirer.Text) assert q.default == ph.default_value unique_phs = XInquirer.unique_placeholders_with_order( xconfig.available_placeholders()) assert len(unique_phs) == 8 assert unique_phs[0].name == "a_int" assert unique_phs[1].name == "a_float" assert unique_phs[2].name == "a_bool1" assert unique_phs[3].name == "a_bool2" assert unique_phs[4].name == "a_bool3" assert unique_phs[5].name == "a_str1" assert unique_phs[6].name == "a_str2" assert unique_phs[7].name == "a_str3"
def test_env_variables(self, generic_temp_folder, data): sample_dict = data["data"] placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data environment_variables = data["environment_variables"] conf = XConfig.from_dict(sample_dict) np.random.seed(66) assert len(conf.available_placeholders()) == len(placeholders) + len( environment_variables ) for envv in environment_variables: pl = Placeholder.from_string(envv) assert ( pl.name not in os.environ ), f"this PYTEST needs no environment variable with name '{pl.name}' " # Load and parse environment variables but nobody set them! conf = XConfig.from_dict(sample_dict, replace_environment_variables=True) assert len(conf.available_placeholders()) == len(placeholders) + len( environment_variables ) # Load and parse environment variables with manual set for envv in environment_variables: pl = Placeholder.from_string(envv) os.environ[pl.name] = str(np.random.uniform(0, 1, (1,))) conf = XConfig.from_dict(sample_dict, replace_environment_variables=True) import rich rich.print(conf) assert len(conf.available_placeholders()) == len(placeholders) for envv in environment_variables: pl = Placeholder.from_string(envv) del os.environ[pl.name]
def test_deep_update(self): cfg = { "l1": "L1", "l2": { "l2_0": "L20", "l2_1": "L21", }, "l3": { "l3_0": { "l3_0_0": "L300", "l3_0_1": "L301", "l3_0_2": "L302", } }, } cfg_to_replace = { "l3": { "l3_0": { "l3_0_2": "NEW_L302", } }, "not_present_key": {"depth": {"alpha": "alpha"}}, } xcfg = XConfig.from_dict(cfg) xcfg_to_replace = XConfig.from_dict(cfg_to_replace) # No Full merge new_cfg = xcfg.copy() new_cfg.deep_update(xcfg_to_replace, full_merge=False) assert len(xcfg.chunks_as_lists()) == len(new_cfg.chunks_as_lists()) assert DeepDiff(xcfg.to_dict(), new_cfg.to_dict()) # Full merge new_cfg = xcfg.copy() new_cfg.deep_update(xcfg_to_replace, full_merge=True) assert len(xcfg.chunks_as_lists()) < len(new_cfg.chunks_as_lists())
def test_creation_root_replace(self, generic_temp_folder, data, cfg_extension): """Test with double @@ -> replace value with content of content""" generic_temp_folder = Path(generic_temp_folder) sample_dict = data["data"] to_be_raplaced_keys = data[ "to_be_replaced" ] # , _, _, to_be_raplaced_keys = data volatile_dict = copy.deepcopy(sample_dict) subtitutions_values = {} for k in to_be_raplaced_keys: random_name = k + f".{cfg_extension}" random_name = DirectiveAT.generate_directive_string( "import_root", [random_name] ) subtitutions_values[random_name] = pydash.get(volatile_dict, k) pydash.set_(volatile_dict, k, random_name) output_cfg_filename = generic_temp_folder / f"out_config.{cfg_extension}" subtitutions_values[str(output_cfg_filename)] = volatile_dict saved_cfgs = [] for directive_value, d in subtitutions_values.items(): directive = DirectiveAT(value=directive_value) if directive.valid: output_filename = generic_temp_folder / directive.args[0] else: output_filename = generic_temp_folder / directive_value store_cfg(output_filename, d) saved_cfgs.append(output_filename) yconf = XConfig(output_cfg_filename) assert DeepDiff(yconf.to_dict(), sample_dict)
def test_chunks_view(self, sample_configurations_data): for config in sample_configurations_data: filename = config["filename"] xcfg = XConfig(filename=filename, no_deep_parse=True) assert len(xcfg.chunks_as_lists()) > 0 assert len(xcfg.chunks_as_lists()) == len(xcfg.chunks_as_tuples()) assert len(xcfg.chunks()) == len(xcfg.chunks_as_tuples()) ls = xcfg.chunks_as_lists() ts = xcfg.chunks_as_tuples() for idx in range(len(ls)): l, _ = ls[idx] t, _ = ts[idx] for kidx in range(len(l)): assert l[kidx] == t[kidx]
def test_cfg_placeholder(self, sample_configurations_data): for sample_cfg in sample_configurations_data: fname = sample_cfg["filename"] root_cfg = { "first_level": 1, "second": { "second.1": f"@cfg(TO_IMPORT,{fname})", "second.2": f"@cfg_root(TO_IMPORT_ROOT,{fname})", }, } print(root_cfg) root_cfg = XConfig.from_dict(root_cfg) copy_cfg = root_cfg.copy() assert len(root_cfg.available_placeholders()) == 2 # IMPORT for key, placeholder in root_cfg.available_placeholders().items(): assert placeholder is not None assert len(placeholder.options) == 1 importer_string = placeholder.cast(placeholder.options[0]) print(importer_string) importer = Importer.from_string(importer_string) assert importer is not None print("REPLACING", key) root_cfg.replace_variable(placeholder.name, placeholder.options[0]) assert len(root_cfg.available_placeholders()) == 0 print("\nNEW_CFG", root_cfg) root_cfg.deep_parse() print("\nNEW_CFG", root_cfg) new_keys = [ x[0] for x in root_cfg.chunks_as_tuples( discard_private_qualifiers=True) ] old_keys = [ x[0] for x in copy_cfg.chunks_as_tuples( discard_private_qualifiers=True) ] assert len(new_keys) > len(old_keys)
def test_copy(self, sample_configurations_data): for config in sample_configurations_data: filename = config["filename"] xcfg = XConfig(filename=filename, no_deep_parse=True) print("#" * 10) print(filename) print(xcfg) xcfg_copy = xcfg.copy() xcfg_copy.deep_parse() xcfg_correct = XConfig(filename=filename) assert not DeepDiff(xcfg_correct.to_dict(), xcfg_copy.to_dict()) assert xcfg_copy == xcfg_correct
def test_validation(self, generic_temp_folder, data): sample_dict = data["data"] schema = data["schema"] # , _, _, to_be_raplaced_keys = data conf = XConfig.from_dict(sample_dict) conf.set_schema(schema) print(type(conf.get_schema())) conf.validate() assert conf.is_valid() conf.set_schema(None) assert conf.is_valid() invalid_schema = Schema({"fake": int}) conf.set_schema(invalid_schema) with pytest.raises(SchemaMissingKeyError): conf.validate() assert not conf.is_valid()
def test_validate_with_replace(self): for replace in [True, False]: cfg = {"one": {"s0": "1", "s1": "22.2", "s2": "True"}} conf = XConfig.from_dict(cfg) schema = Schema( { "one": { "s0": Use(int), "s1": Use(float), "s2": Use(bool), } } ) conf.set_schema(schema) conf.validate(replace=replace) assert isinstance(conf.one.s0, int) == replace assert isinstance(conf.one.s1, float) == replace assert isinstance(conf.one.s2, bool) == replace
def test_sweeps(self): ints = [1, 2, 3] floats = [1.0, 2.0, 3.0] strings = ['"a"', '"b"', '"c"'] bools = [True, False] root_cfg = { "first_level": 1, "second": { "ints": self._build_sweep(ints), "floats": self._build_sweep(floats), "strings": self._build_sweep(strings), "bools": self._build_sweep(bools), }, } cfg = XConfig.from_dict(root_cfg) assert len(cfg.available_placeholders()) == 4 sweeped_cfgs = cfg.sweep() total = len(ints) * len(floats) * len(strings) * len(bools) assert len(sweeped_cfgs) == total
def test_replace(self, generic_temp_folder, data): sample_dict = data["data"] placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data environment_variables = data["environment_variables"] conf = XConfig.from_dict(sample_dict) np.random.seed(66) to_replace = {} for p in placeholders: _p = Placeholder.from_string(p) to_replace[_p.name] = np.random.randint(0, 10) assert len(conf.available_placeholders()) == len(placeholders) + len( environment_variables ) conf.check_available_placeholders(close_app=False) if len(conf.available_placeholders()) > 0: for name, p in conf.available_placeholders().items(): assert isinstance(name, str) assert isinstance(p, Placeholder) with pytest.raises(SystemExit): conf.check_available_placeholders(close_app=True) conf.replace_variables_map(to_replace) chunks = conf.chunks_as_lists() for key, value in chunks: if not isinstance(value, Box) and not isinstance(value, BoxList): assert value not in to_replace.keys() assert len(conf.available_placeholders()) == len(environment_variables)
def prompt(cls, xconfig: XConfig, close_app: bool = True) -> XConfig: """Prompts the placeholders of an xconfig and fill them with the user answers, it returns a copy of the original xconfig :param xconfig: the xconfig :type xconfig: XConfig :param close_app: if True the app will be closed if all the placeholder are not filled, defaults to True :type close_app: bool, optional :return: the filled xconfig :rtype: XConfig """ xconfig = xconfig.copy() unique_phs = cls.unique_placeholders_with_order( xconfig.available_placeholders() ) # user interaction questions = [cls.placeholder_to_question(x) for x in unique_phs] answers = cls._system_prompt(questions) # replaces placeholders with inquirer answers xconfig.replace_variables_map(answers) xconfig.check_available_placeholders(close_app=close_app) return xconfig
from choixe.configurations import XConfig from rich import print cfg = XConfig(filename="cfg.yml") print(cfg.to_dict())
from choixe.configurations import XConfig import rich ################################## # SETs ENVIRONMENT VARIABLE FIRST # to pass XConfig check # # > export V0=0.0 # > export V1=2.0 # > export V2=3.0 # ################################## cfg = XConfig(filename="cfg.yml") cfg.check_available_placeholders(close_app=True) rich.print(cfg.to_dict())
# Create Object bundle = MyBundle( name="custom_bundle", images=[ MyImage(name="image", image_size=[800, 600]), MyImage(name="mask", image_size=[256, 256]), MyImage(name="crop", image_size=[32, 32]), ], ) # Serialization bundle_serialized = bundle.serialize() rich.print("Serialized object:", bundle_serialized) # IO Write cfg = XConfig.from_dict(bundle_serialized) cfg_filename = Path(tempfile.mkdtemp()) / "cfg.yml" cfg.save_to(cfg_filename) rich.print("Stored config in:", cfg_filename) # IO Read cfg = XConfig(filename=cfg_filename) hydrated_bundle: MyBundle = Spook.create(cfg) rich.print("Hydrated object:", hydrated_bundle.serialize()) for image in hydrated_bundle.images: image.my_method()
def generate_labels( data_folder, scale_factor, distance_threshold, ransac_n, num_iterations, offset, offset_up, eps, min_points, min_area, ): VISUALIZER_SIZE = [800, 800] try: from screeninfo import get_monitors max_height = int(get_monitors()[0].height) max_width = int(get_monitors()[0].width * 0.48) VISUALIZER_SIZE = [max_width, max_height] except: pass # Check for filenames image_filename = data_folder / "image.png" disparity_filename = data_folder / "disparity.png" calibration_filename = data_folder / "calibration.yml" for _filename in [ image_filename, disparity_filename, calibration_filename ]: assert _filename.exists(), f"Filename: {str(_filename)} not found!" # Load camera calibration data calibration = XConfig(calibration_filename) camera_matrix = calibration["camera_matrix"]["center"] baseline = calibration["baseline"]["center_right"] focal = camera_matrix[0][0] # Load image/disparity image = np.array(imageio.imread(str(image_filename))) if len(image.shape) == 2: image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) disparity = np.array(imageio.imread(str(disparity_filename))) # Compute depth from disparity depth = (baseline * focal / disparity * 1000).astype(np.float32) # Segmentation Pipeline segmenter = Segmenter(image, depth, camera_matrix, f=scale_factor) segmentation_output = segmenter.segment( distance_threshold, ransac_n, num_iterations, offset, offset_up, eps, min_points, min_area, ) # Visualization loop control visualization_running = True def stop_visualization(event=None): nonlocal visualization_running visualization_running = False # Visualize 3D PointCloud with plane segmentation visualizer_3d = create_3d_visualizer( point_cloud=segmentation_output["full_point_cloud"], inliers=segmentation_output["plane_inliers"], visualizer_size=VISUALIZER_SIZE, ) visualizer_3d.register_key_callback(81, stop_visualization) # Visualize 2D Final Output colored_seg_mask = segmentation_output["colored_final_mask"] plane_seg_mask = segmentation_output["colored_plane_mask"] crops = segmentation_output["crops"] _, _, bounding_boxes = zip(*crops) out = create_2d_stacked_image(image, colored_seg_mask, plane_seg_mask, bounding_boxes) out = cv2.cvtColor(out, cv2.COLOR_RGB2BGR) cv2.namedWindow("output", cv2.WINDOW_NORMAL) cv2.resizeWindow("output", VISUALIZER_SIZE[0], VISUALIZER_SIZE[1]) cv2.moveWindow("output", int(VISUALIZER_SIZE[0] * 1.1), 0) # Visualization loop while visualization_running: visualizer_3d.poll_events() visualizer_3d.update_renderer() cv2.imshow("output", out) c = cv2.waitKey(10) if ord("q") == c: stop_visualization() # teardown visualizers visualizer_3d.destroy_window() cv2.destroyWindow("output")
from choixe.configurations import XConfig import rich import tempfile from pathlib import Path cfg = XConfig(filename="cfg.yml") # Create sweeps from single cfg sweeped_cfgs = cfg.sweep() folder = Path(tempfile.mkdtemp()) for idx, cfg in enumerate(sweeped_cfgs): rich.print("Sweep", "-" * 10) rich.print(cfg) filename = folder / f"sweep_{str(idx).zfill(5)}.yml" cfg.save_to(filename=filename) rich.print("Output saved to:", folder)
from choixe.configurations import XConfig import rich cfg = XConfig(filename="cfg.yml") # Checks for placeholders before replace rich.print("\nBefore Replace", cfg.to_dict()) cfg.check_available_placeholders(close_app=False) # Iterate Placeholders placeholders = cfg.available_placeholders() for key, p in placeholders.items(): # Replace default value if any if p.default_value is not None: cfg.replace_variable( p.name, p.default_value ) # Replace automatically Casts for CFG # Checks for placeholders after replace rich.print("\nAfter Replace", cfg.to_dict()) # Checks for placeholders after replace cfg.deep_parse() rich.print("\nAfter Second Deep Parse", cfg.to_dict())
from choixe.configurations import XConfig import rich from schema import Schema, Use cfg = { "one": {"s0": "1", "s1": "22.2", "s2": "True"}, "nested": {"alphabet": {"zero": "0", "one": "1"}}, } conf = XConfig.from_dict(cfg) rich.print(conf) schema = Schema( { "one": { "s0": Use(int), "s1": Use(float), "s2": Use(bool), }, "nested": {"alphabet": {"zero": Use(int), "one": Use(float)}}, } ) conf.set_schema(schema) conf.validate() rich.print(conf)
def complex_data(): np.random.seed(666) placeholders = [] env_variables = [] pl_generator = PlaceholderGenerator() sample_dict = { "one": pl_generator.create_placeholder( placeholders, "int", [6, 7, 8, "default=10"] ), "two": np.random.randint(-50, 50, (3, 3)).tolist(), "three": { "3.1": "TrueValue", "2.1": [False, False], "name": { "name_1": pl_generator.create_placeholder(placeholders, "float"), "name_2": 2, "name_3": { "this_is_a_list": [3.3, 3.3], "this_is_A_dict": { "a": { "f": True, "s": False, "numpy_array": np.array([1.0, 2, 3]), "numpy_data": np.array([1.0, 2, 3])[0], "tuple": ("a", "b", "2"), "boxlist": BoxList([1, 2, 3, 4]), "boxdict": Box({"a": 2.2}), } }, }, }, }, "first": { "f1": pl_generator.create_placeholder( placeholders, "str", ["alpha", "beta", "gamma"] ), # placeholders[2], "f2": 2.22, "f3": [3, 3, 3, 3, 3, 3], "external": { "ext": np.random.uniform(-2, 2, (2, 3)).tolist(), "ext_name": [pl_generator.create_placeholder(placeholders, "bool")], }, "ops": { "o1": pl_generator.create_placeholder(placeholders), # placeholders[4], "o2": pl_generator.create_placeholder( placeholders, "path" ), # placeholders[5], "o3": pl_generator.create_placeholder( placeholders, "date" ), # placeholders[6], "o4": pl_generator.create_placeholder( placeholders, "str", ["A", "B", "C", "default=C"] ), # placeholders[7], "o5": pl_generator.create_placeholder(placeholders), # placeholders[8], "o6": pl_generator.create_placeholder(placeholders), # placeholders[9] }, "env": [ pl_generator.create_placeholder( env_variables, "env" ), # env_variables[0], pl_generator.create_placeholder(env_variables, "env"), ], "key": { "with": 120, "dots": pl_generator.create_placeholder(placeholders, "object"), }, "key.with.dots": pl_generator.create_placeholder( placeholders ), # placeholders[10], "key.with...many.....dots": pl_generator.create_placeholder(placeholders), "nested.key": {"with.dots": pl_generator.create_placeholder(placeholders)}, "placeholder_object": pl_generator.create_placeholder(placeholders), "placeholder_cfg": pl_generator.create_placeholder( placeholders, "cfg" ), # placeholders[15], "placeholder_cfg_root": pl_generator.create_placeholder( placeholders, "cfg_root" ), # , }, } schema = Schema( { "one": Or(int, str), "two": list, "three": { "3.1": str, "2.1": [bool], "name": dict, }, "first": {Regex(""): Or(str, int, list, float, dict)}, } ) to_be_raplaced_keys = sorted( [ "three.name", "three.name.name_3", "three.name.name_3.this_is_A_dict", "first", "first.external", ], reverse=True, ) return { "data": XConfig.decode(sample_dict), "schema": schema, "placeholders": placeholders, "environment_variables": env_variables, "to_be_replaced": to_be_raplaced_keys, }
def test_creation( self, generic_temp_folder, data, cfg_extension ): # TODO: toml 0.10.2 needed! toml has a bug otherwise generic_temp_folder = Path(generic_temp_folder) sample_dict = data["data"] to_be_raplaced_keys = data[ "to_be_replaced" ] # , _, _, to_be_raplaced_keys = data volatile_dict = copy.deepcopy(sample_dict) subtitutions_values = {} for k in to_be_raplaced_keys: random_name = k + f".{cfg_extension}" random_name = DirectiveAT.generate_directive_string("import", [random_name]) subtitutions_values[random_name] = pydash.get(volatile_dict, k) pydash.set_(volatile_dict, k, random_name) output_cfg_filename = generic_temp_folder / f"out_config.{cfg_extension}" output_cfg_filename2 = generic_temp_folder / f"out_config2.{cfg_extension}" subtitutions_values[str(output_cfg_filename)] = volatile_dict saved_cfgs = [] for directive_value, d in subtitutions_values.items(): directive = DirectiveAT(value=directive_value) if directive.valid: output_filename = generic_temp_folder / directive.args[0] else: output_filename = generic_temp_folder / directive_value store_cfg(output_filename, d) saved_cfgs.append(output_filename) yconf = XConfig(output_cfg_filename) yconf.save_to(output_cfg_filename2) yconf_reloaded = XConfig(output_cfg_filename2) with pytest.raises(NotImplementedError): yconf.save_to(str(output_cfg_filename2) + "#IMPOSSIBLE.EXTENSION") assert not DeepDiff(yconf.to_dict(), sample_dict) assert not DeepDiff(yconf_reloaded.to_dict(), sample_dict) assert not DeepDiff( XConfig.from_dict(yconf_reloaded.to_dict()).to_dict(), yconf_reloaded.to_dict(), ) assert len(XConfig.from_dict(yconf_reloaded.to_dict())) > len( sample_dict ) # YConf contains 2 more private keys! # remove last cfg file saved_cfgs[0].unlink() with pytest.raises(OSError): yconf = XConfig(output_cfg_filename)