def store_cfg(filename: Union[str, Path], d: dict):
filename = str(filename)
data = Box(d)
if "yml" in filename or "yaml" in filename:
Box(XConfig.decode(data.to_dict())).to_yaml(filename)
if "json" in filename:
Box(XConfig.decode(data.to_dict())).to_json(filename)
if "toml" in filename:
Box(XConfig.decode(data.to_dict())).to_toml(filename)
def compile(
configuration_file: str,
output_file: str,
check: bool,
options: Sequence[Tuple[str, str]],
):
from choixe.configurations import XConfig
import rich
try:
cfg = XConfig(filename=configuration_file)
except Exception as e:
rich.print(f"[red]Invalid configuration file: {e}[/red]")
import sys
sys.exit(1)
placeholders = cfg.available_placeholders()
for key, value in options:
if key in placeholders:
placeholder = placeholders[key]
cfg.deep_set(key, placeholder.cast(value))
if check:
cfg.check_available_placeholders(close_app=True)
cfg.save_to(output_file)
def acquire(
calibration: Path,
device_cfg: Path,
output_folder: Path,
scale_factor: int,
disparity: bool,
max_disparity: int,
max_depth: int,
max_frames: int,
skip: int,
):
if device_cfg is None:
device_cfg = oakeye.data_folder / "device" / "device.yml"
device_cfg = XConfig(device_cfg)
device = OakDeviceFactory().create(device_cfg)
keys = ["left", "center", "right", "depth", "disparityCL", "disparityCR"]
acquirer = DeviceAcquirer(device)
if calibration is not None:
calib = XConfig(calibration)
acquirer = RectifiedAcquirer(acquirer, calib)
if disparity:
acquirer = DisparityAcquirer(acquirer, disp_diff=max_disparity)
acquirer = GuiAcquirer(
acquirer,
keys,
scale_factor=scale_factor,
ranges={
"disparityCL": [0, max_disparity],
"disparityCR": [0, max_disparity],
"depth": [0, max_depth],
},
)
dataset = acquirer(max_frames=max_frames, skip=skip)
device.close()
if output_folder is not None:
ext_map = {
"left": "png",
"center": "png",
"right": "png",
"depth": "png",
"disparityCL": "png",
"disparityCR": "png",
"device": "yml",
}
root_files = ["device"]
for d in dataset:
d["device"] = device_cfg.to_dict()
UnderfolderWriter(output_folder,
root_files_keys=root_files,
extensions_map=ext_map)(SamplesSequence(dataset))
def test_prompt(self, monkeypatch, tmpdir_factory):
d = {
"a_int": "@int(a_int)",
"another_int": "@int(a_int)",
"a_float": "@float(a_float)",
"a_bool": "@bool(a_bool)",
"a_str1": "@str(a_str1)",
"a_str2": "@str(a_str2,hello,hi)",
"a_str3": "@str(a_str3,one,two,three,default=one)",
}
fills = {
"a_int": 2,
"another_int": 666,
"a_float": 2.22,
"a_bool": True,
"a_str1": "hello",
"a_str2": "hi",
"a_str3": "three",
}
def custom_prompt(questions):
return fills
# old_prompt = XInquirer._system_prompt
monkeypatch.setattr(XInquirer,
"_system_prompt",
custom_prompt,
raising=True)
# from dict
xconfig = XConfig.from_dict(d)
xconfig.set_schema(Schema({}))
new_xconfig = XInquirer.prompt(xconfig, close_app=False)
assert len(new_xconfig.available_placeholders()) == 0
assert xconfig._filename == new_xconfig._filename
assert xconfig.get_schema() == new_xconfig.get_schema()
# from file
filename = tmpdir_factory.mktemp("configuration") / "cfg.yml"
xconfig.save_to(filename)
xconfig = XConfig(filename=filename)
new_xconfig = XInquirer.prompt(xconfig, close_app=False)
assert len(new_xconfig.available_placeholders()) == 0
assert xconfig._filename == new_xconfig._filename
assert xconfig.get_schema() == new_xconfig.get_schema()
def test_deepset(self):
cfg = {
"l1": "L1",
"l2": {
"l2_0": "L20",
"l2_1": "L21",
},
"l3": {
"l3_0": {
"l3_0_0": "L300",
"l3_0_1": "L301",
"l3_0_2": "L302",
}
},
}
xcfg = XConfig.from_dict(cfg)
for k, old_value in xcfg.chunks_as_lists():
row_dict = {}
pydash.set_(row_dict, k, old_value)
xrow_dict = XConfig.from_dict(row_dict)
for newk, newv in xrow_dict.chunks_as_lists():
# VALID KEYS
# Try to replace present keys only with ONLY_VALID_KEYS = TRUE, should be equal!
new_xcfg = xcfg.copy()
new_xcfg.deep_set(newk, newv, only_valid_keys=True)
assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict())
# Try to replace present keys only with ONLY_VALID_KEYS = FALSE, should be equal!
new_xcfg = xcfg.copy()
new_xcfg.deep_set(newk, newv, only_valid_keys=False)
assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict())
# INVALID KEYS
new_keys = ["xxIMPOSSIBLE_KEY"] * 10
# Try to replace not present keys but ONLY_VALID_KEYS = TRUE, should be equal!
new_xcfg = xcfg.copy()
new_xcfg.deep_set(new_keys, newv, only_valid_keys=True)
assert not DeepDiff(xcfg.to_dict(), new_xcfg.to_dict())
# Try to replace not present keys but ONLY_VALID_KEYS = False, should be different!
new_xcfg = xcfg.copy()
new_xcfg.deep_set(new_keys, newv, only_valid_keys=False)
assert DeepDiff(xcfg.to_dict(), new_xcfg.to_dict())
def test_replace_ignore_defaults(self, generic_temp_folder, data):
sample_dict = data["data"]
placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data
environment_variables = data["environment_variables"]
conf = XConfig.from_dict(sample_dict)
rich.print(conf)
np.random.seed(66)
to_replace = {}
to_replace_with_defaults = {}
for p in placeholders:
_p = Placeholder.from_string(p)
if _p.default_value is not None:
to_replace_with_defaults[_p.name] = np.random.randint(0, 10)
else:
to_replace[_p.name] = np.random.randint(0, 10)
rich.print("NAME", _p.name, _p.default_value is None)
defaults_placeholders_counter = len(conf.available_placeholders()) - len(
conf.available_placeholders(ignore_defaults=True)
)
assert defaults_placeholders_counter == len(to_replace_with_defaults)
if defaults_placeholders_counter > 0:
conf_copy_A = conf.copy()
conf_copy_A.replace_variables_map(to_replace, replace_defaults=False)
conf_copy_B = conf.copy()
conf_copy_B.replace_variables_map(to_replace, replace_defaults=True)
assert len(conf_copy_A.available_placeholders()) > len(
conf_copy_B.available_placeholders()
)
def check(configuration_file, close):
from choixe.configurations import XConfig
import rich
try:
cfg = XConfig(filename=configuration_file)
rich.print(cfg.to_dict())
except Exception as e:
rich.print(f"[red]Invalid configuration file: {e}[/red]")
import sys
sys.exit(1)
rich.print("[green]Configuration is good![/green]")
# d = cfg.to_dict(discard_private_qualifiers=False)
cfg.check_available_placeholders(close_app=close)
def test_nosweeps(self):
root_cfg = {
"first_level": 1,
"second": {
"alpha": [1, 2, 3],
},
}
cfg = XConfig.from_dict(root_cfg)
assert len(cfg.sweep()) == 1
def create_from_file(cls, filename: str) -> any:
"""Creates an object starting from its serialized representation in a file loadable
as XConfig (yml, json, etc.)
:param filename: config filename
:type filename: str
:return: hydrated object
:rtype: any
"""
return cls.create(XConfig(filename))
def test_xinquirer(self):
assert XInquirer.safe_cast_placeholder(10, PlaceholderType.INT)
assert XInquirer.safe_cast_placeholder(10, PlaceholderType.FLOAT)
assert XInquirer.safe_cast_placeholder(10, PlaceholderType.STR)
assert XInquirer.safe_cast_placeholder("10", PlaceholderType.INT)
assert not XInquirer.safe_cast_placeholder("a string",
PlaceholderType.INT)
assert not XInquirer.safe_cast_placeholder("another string",
PlaceholderType.FLOAT)
assert XInquirer.safe_cast_placeholder(True, PlaceholderType.BOOL)
assert XInquirer.safe_cast_placeholder(0, PlaceholderType.BOOL)
assert XInquirer.safe_cast_placeholder("hello", PlaceholderType.PATH)
assert XInquirer.safe_cast_placeholder("what time is it?",
PlaceholderType.DATE)
d = {
"a_int": "@int(a_int)",
"another_int": "@int(a_int)",
"a_float": "@float(a_float)",
"a_bool1": "@bool(a_bool1)",
"a_bool2": "@bool(a_bool2, default=AA)",
"a_bool3": "@bool(a_bool3, default=yEs)",
"a_str1": "@str(a_str1)",
"a_str2": "@str(a_str2,hello,hi)",
"a_str3": "@str(a_str3,one,two,three,default=one)",
}
xconfig = XConfig.from_dict(d)
placeholders = list(xconfig.available_placeholders().values())
for ph in placeholders:
q = XInquirer.placeholder_to_question(ph)
assert q.name == ph.name
if ph.type == PlaceholderType.BOOL:
assert isinstance(q, inquirer.Confirm)
assert isinstance(q.default, bool)
assert q.default == XInquirer.to_bool(ph.default_value)
elif len(ph.options) > 0:
assert isinstance(q, inquirer.List)
assert len(q.choices) == len(ph.options)
assert q.default == ph.default_value
else:
assert isinstance(q, inquirer.Text)
assert q.default == ph.default_value
unique_phs = XInquirer.unique_placeholders_with_order(
xconfig.available_placeholders())
assert len(unique_phs) == 8
assert unique_phs[0].name == "a_int"
assert unique_phs[1].name == "a_float"
assert unique_phs[2].name == "a_bool1"
assert unique_phs[3].name == "a_bool2"
assert unique_phs[4].name == "a_bool3"
assert unique_phs[5].name == "a_str1"
assert unique_phs[6].name == "a_str2"
assert unique_phs[7].name == "a_str3"
def test_env_variables(self, generic_temp_folder, data):
sample_dict = data["data"]
placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data
environment_variables = data["environment_variables"]
conf = XConfig.from_dict(sample_dict)
np.random.seed(66)
assert len(conf.available_placeholders()) == len(placeholders) + len(
environment_variables
)
for envv in environment_variables:
pl = Placeholder.from_string(envv)
assert (
pl.name not in os.environ
), f"this PYTEST needs no environment variable with name '{pl.name}' "
# Load and parse environment variables but nobody set them!
conf = XConfig.from_dict(sample_dict, replace_environment_variables=True)
assert len(conf.available_placeholders()) == len(placeholders) + len(
environment_variables
)
# Load and parse environment variables with manual set
for envv in environment_variables:
pl = Placeholder.from_string(envv)
os.environ[pl.name] = str(np.random.uniform(0, 1, (1,)))
conf = XConfig.from_dict(sample_dict, replace_environment_variables=True)
import rich
rich.print(conf)
assert len(conf.available_placeholders()) == len(placeholders)
for envv in environment_variables:
pl = Placeholder.from_string(envv)
del os.environ[pl.name]
def test_deep_update(self):
cfg = {
"l1": "L1",
"l2": {
"l2_0": "L20",
"l2_1": "L21",
},
"l3": {
"l3_0": {
"l3_0_0": "L300",
"l3_0_1": "L301",
"l3_0_2": "L302",
}
},
}
cfg_to_replace = {
"l3": {
"l3_0": {
"l3_0_2": "NEW_L302",
}
},
"not_present_key": {"depth": {"alpha": "alpha"}},
}
xcfg = XConfig.from_dict(cfg)
xcfg_to_replace = XConfig.from_dict(cfg_to_replace)
# No Full merge
new_cfg = xcfg.copy()
new_cfg.deep_update(xcfg_to_replace, full_merge=False)
assert len(xcfg.chunks_as_lists()) == len(new_cfg.chunks_as_lists())
assert DeepDiff(xcfg.to_dict(), new_cfg.to_dict())
# Full merge
new_cfg = xcfg.copy()
new_cfg.deep_update(xcfg_to_replace, full_merge=True)
assert len(xcfg.chunks_as_lists()) < len(new_cfg.chunks_as_lists())
def test_creation_root_replace(self, generic_temp_folder, data, cfg_extension):
"""Test with double @@ -> replace value with content of content"""
generic_temp_folder = Path(generic_temp_folder)
sample_dict = data["data"]
to_be_raplaced_keys = data[
"to_be_replaced"
] # , _, _, to_be_raplaced_keys = data
volatile_dict = copy.deepcopy(sample_dict)
subtitutions_values = {}
for k in to_be_raplaced_keys:
random_name = k + f".{cfg_extension}"
random_name = DirectiveAT.generate_directive_string(
"import_root", [random_name]
)
subtitutions_values[random_name] = pydash.get(volatile_dict, k)
pydash.set_(volatile_dict, k, random_name)
output_cfg_filename = generic_temp_folder / f"out_config.{cfg_extension}"
subtitutions_values[str(output_cfg_filename)] = volatile_dict
saved_cfgs = []
for directive_value, d in subtitutions_values.items():
directive = DirectiveAT(value=directive_value)
if directive.valid:
output_filename = generic_temp_folder / directive.args[0]
else:
output_filename = generic_temp_folder / directive_value
store_cfg(output_filename, d)
saved_cfgs.append(output_filename)
yconf = XConfig(output_cfg_filename)
assert DeepDiff(yconf.to_dict(), sample_dict)
def test_chunks_view(self, sample_configurations_data):
for config in sample_configurations_data:
filename = config["filename"]
xcfg = XConfig(filename=filename, no_deep_parse=True)
assert len(xcfg.chunks_as_lists()) > 0
assert len(xcfg.chunks_as_lists()) == len(xcfg.chunks_as_tuples())
assert len(xcfg.chunks()) == len(xcfg.chunks_as_tuples())
ls = xcfg.chunks_as_lists()
ts = xcfg.chunks_as_tuples()
for idx in range(len(ls)):
l, _ = ls[idx]
t, _ = ts[idx]
for kidx in range(len(l)):
assert l[kidx] == t[kidx]
def test_cfg_placeholder(self, sample_configurations_data):
for sample_cfg in sample_configurations_data:
fname = sample_cfg["filename"]
root_cfg = {
"first_level": 1,
"second": {
"second.1": f"@cfg(TO_IMPORT,{fname})",
"second.2": f"@cfg_root(TO_IMPORT_ROOT,{fname})",
},
}
print(root_cfg)
root_cfg = XConfig.from_dict(root_cfg)
copy_cfg = root_cfg.copy()
assert len(root_cfg.available_placeholders()) == 2
# IMPORT
for key, placeholder in root_cfg.available_placeholders().items():
assert placeholder is not None
assert len(placeholder.options) == 1
importer_string = placeholder.cast(placeholder.options[0])
print(importer_string)
importer = Importer.from_string(importer_string)
assert importer is not None
print("REPLACING", key)
root_cfg.replace_variable(placeholder.name,
placeholder.options[0])
assert len(root_cfg.available_placeholders()) == 0
print("\nNEW_CFG", root_cfg)
root_cfg.deep_parse()
print("\nNEW_CFG", root_cfg)
new_keys = [
x[0] for x in root_cfg.chunks_as_tuples(
discard_private_qualifiers=True)
]
old_keys = [
x[0] for x in copy_cfg.chunks_as_tuples(
discard_private_qualifiers=True)
]
assert len(new_keys) > len(old_keys)
def test_copy(self, sample_configurations_data):
for config in sample_configurations_data:
filename = config["filename"]
xcfg = XConfig(filename=filename, no_deep_parse=True)
print("#" * 10)
print(filename)
print(xcfg)
xcfg_copy = xcfg.copy()
xcfg_copy.deep_parse()
xcfg_correct = XConfig(filename=filename)
assert not DeepDiff(xcfg_correct.to_dict(), xcfg_copy.to_dict())
assert xcfg_copy == xcfg_correct
def test_validation(self, generic_temp_folder, data):
sample_dict = data["data"]
schema = data["schema"] # , _, _, to_be_raplaced_keys = data
conf = XConfig.from_dict(sample_dict)
conf.set_schema(schema)
print(type(conf.get_schema()))
conf.validate()
assert conf.is_valid()
conf.set_schema(None)
assert conf.is_valid()
invalid_schema = Schema({"fake": int})
conf.set_schema(invalid_schema)
with pytest.raises(SchemaMissingKeyError):
conf.validate()
assert not conf.is_valid()
def test_validate_with_replace(self):
for replace in [True, False]:
cfg = {"one": {"s0": "1", "s1": "22.2", "s2": "True"}}
conf = XConfig.from_dict(cfg)
schema = Schema(
{
"one": {
"s0": Use(int),
"s1": Use(float),
"s2": Use(bool),
}
}
)
conf.set_schema(schema)
conf.validate(replace=replace)
assert isinstance(conf.one.s0, int) == replace
assert isinstance(conf.one.s1, float) == replace
assert isinstance(conf.one.s2, bool) == replace
def test_sweeps(self):
ints = [1, 2, 3]
floats = [1.0, 2.0, 3.0]
strings = ['"a"', '"b"', '"c"']
bools = [True, False]
root_cfg = {
"first_level": 1,
"second": {
"ints": self._build_sweep(ints),
"floats": self._build_sweep(floats),
"strings": self._build_sweep(strings),
"bools": self._build_sweep(bools),
},
}
cfg = XConfig.from_dict(root_cfg)
assert len(cfg.available_placeholders()) == 4
sweeped_cfgs = cfg.sweep()
total = len(ints) * len(floats) * len(strings) * len(bools)
assert len(sweeped_cfgs) == total
def test_replace(self, generic_temp_folder, data):
sample_dict = data["data"]
placeholders = data["placeholders"] # , _, _, to_be_raplaced_keys = data
environment_variables = data["environment_variables"]
conf = XConfig.from_dict(sample_dict)
np.random.seed(66)
to_replace = {}
for p in placeholders:
_p = Placeholder.from_string(p)
to_replace[_p.name] = np.random.randint(0, 10)
assert len(conf.available_placeholders()) == len(placeholders) + len(
environment_variables
)
conf.check_available_placeholders(close_app=False)
if len(conf.available_placeholders()) > 0:
for name, p in conf.available_placeholders().items():
assert isinstance(name, str)
assert isinstance(p, Placeholder)
with pytest.raises(SystemExit):
conf.check_available_placeholders(close_app=True)
conf.replace_variables_map(to_replace)
chunks = conf.chunks_as_lists()
for key, value in chunks:
if not isinstance(value, Box) and not isinstance(value, BoxList):
assert value not in to_replace.keys()
assert len(conf.available_placeholders()) == len(environment_variables)
def prompt(cls, xconfig: XConfig, close_app: bool = True) -> XConfig:
"""Prompts the placeholders of an xconfig and fill them
with the user answers, it returns a copy of the original xconfig
:param xconfig: the xconfig
:type xconfig: XConfig
:param close_app: if True the app will be closed if all the placeholder are not filled, defaults to True
:type close_app: bool, optional
:return: the filled xconfig
:rtype: XConfig
"""
xconfig = xconfig.copy()
unique_phs = cls.unique_placeholders_with_order(
xconfig.available_placeholders()
)
# user interaction
questions = [cls.placeholder_to_question(x) for x in unique_phs]
answers = cls._system_prompt(questions)
# replaces placeholders with inquirer answers
xconfig.replace_variables_map(answers)
xconfig.check_available_placeholders(close_app=close_app)
return xconfig
from choixe.configurations import XConfig from rich import print cfg = XConfig(filename="cfg.yml") print(cfg.to_dict())
from choixe.configurations import XConfig import rich ################################## # SETs ENVIRONMENT VARIABLE FIRST # to pass XConfig check # # > export V0=0.0 # > export V1=2.0 # > export V2=3.0 # ################################## cfg = XConfig(filename="cfg.yml") cfg.check_available_placeholders(close_app=True) rich.print(cfg.to_dict())
# Create Object
bundle = MyBundle(
name="custom_bundle",
images=[
MyImage(name="image", image_size=[800, 600]),
MyImage(name="mask", image_size=[256, 256]),
MyImage(name="crop", image_size=[32, 32]),
],
)
# Serialization
bundle_serialized = bundle.serialize()
rich.print("Serialized object:", bundle_serialized)
# IO Write
cfg = XConfig.from_dict(bundle_serialized)
cfg_filename = Path(tempfile.mkdtemp()) / "cfg.yml"
cfg.save_to(cfg_filename)
rich.print("Stored config in:", cfg_filename)
# IO Read
cfg = XConfig(filename=cfg_filename)
hydrated_bundle: MyBundle = Spook.create(cfg)
rich.print("Hydrated object:", hydrated_bundle.serialize())
for image in hydrated_bundle.images:
image.my_method()
def generate_labels(
data_folder,
scale_factor,
distance_threshold,
ransac_n,
num_iterations,
offset,
offset_up,
eps,
min_points,
min_area,
):
VISUALIZER_SIZE = [800, 800]
try:
from screeninfo import get_monitors
max_height = int(get_monitors()[0].height)
max_width = int(get_monitors()[0].width * 0.48)
VISUALIZER_SIZE = [max_width, max_height]
except:
pass
# Check for filenames
image_filename = data_folder / "image.png"
disparity_filename = data_folder / "disparity.png"
calibration_filename = data_folder / "calibration.yml"
for _filename in [
image_filename, disparity_filename, calibration_filename
]:
assert _filename.exists(), f"Filename: {str(_filename)} not found!"
# Load camera calibration data
calibration = XConfig(calibration_filename)
camera_matrix = calibration["camera_matrix"]["center"]
baseline = calibration["baseline"]["center_right"]
focal = camera_matrix[0][0]
# Load image/disparity
image = np.array(imageio.imread(str(image_filename)))
if len(image.shape) == 2:
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
disparity = np.array(imageio.imread(str(disparity_filename)))
# Compute depth from disparity
depth = (baseline * focal / disparity * 1000).astype(np.float32)
# Segmentation Pipeline
segmenter = Segmenter(image, depth, camera_matrix, f=scale_factor)
segmentation_output = segmenter.segment(
distance_threshold,
ransac_n,
num_iterations,
offset,
offset_up,
eps,
min_points,
min_area,
)
# Visualization loop control
visualization_running = True
def stop_visualization(event=None):
nonlocal visualization_running
visualization_running = False
# Visualize 3D PointCloud with plane segmentation
visualizer_3d = create_3d_visualizer(
point_cloud=segmentation_output["full_point_cloud"],
inliers=segmentation_output["plane_inliers"],
visualizer_size=VISUALIZER_SIZE,
)
visualizer_3d.register_key_callback(81, stop_visualization)
# Visualize 2D Final Output
colored_seg_mask = segmentation_output["colored_final_mask"]
plane_seg_mask = segmentation_output["colored_plane_mask"]
crops = segmentation_output["crops"]
_, _, bounding_boxes = zip(*crops)
out = create_2d_stacked_image(image, colored_seg_mask, plane_seg_mask,
bounding_boxes)
out = cv2.cvtColor(out, cv2.COLOR_RGB2BGR)
cv2.namedWindow("output", cv2.WINDOW_NORMAL)
cv2.resizeWindow("output", VISUALIZER_SIZE[0], VISUALIZER_SIZE[1])
cv2.moveWindow("output", int(VISUALIZER_SIZE[0] * 1.1), 0)
# Visualization loop
while visualization_running:
visualizer_3d.poll_events()
visualizer_3d.update_renderer()
cv2.imshow("output", out)
c = cv2.waitKey(10)
if ord("q") == c:
stop_visualization()
# teardown visualizers
visualizer_3d.destroy_window()
cv2.destroyWindow("output")
from choixe.configurations import XConfig
import rich
import tempfile
from pathlib import Path
cfg = XConfig(filename="cfg.yml")
# Create sweeps from single cfg
sweeped_cfgs = cfg.sweep()
folder = Path(tempfile.mkdtemp())
for idx, cfg in enumerate(sweeped_cfgs):
rich.print("Sweep", "-" * 10)
rich.print(cfg)
filename = folder / f"sweep_{str(idx).zfill(5)}.yml"
cfg.save_to(filename=filename)
rich.print("Output saved to:", folder)
from choixe.configurations import XConfig
import rich
cfg = XConfig(filename="cfg.yml")
# Checks for placeholders before replace
rich.print("\nBefore Replace", cfg.to_dict())
cfg.check_available_placeholders(close_app=False)
# Iterate Placeholders
placeholders = cfg.available_placeholders()
for key, p in placeholders.items():
# Replace default value if any
if p.default_value is not None:
cfg.replace_variable(
p.name, p.default_value
) # Replace automatically Casts for CFG
# Checks for placeholders after replace
rich.print("\nAfter Replace", cfg.to_dict())
# Checks for placeholders after replace
cfg.deep_parse()
rich.print("\nAfter Second Deep Parse", cfg.to_dict())
from choixe.configurations import XConfig
import rich
from schema import Schema, Use
cfg = {
"one": {"s0": "1", "s1": "22.2", "s2": "True"},
"nested": {"alphabet": {"zero": "0", "one": "1"}},
}
conf = XConfig.from_dict(cfg)
rich.print(conf)
schema = Schema(
{
"one": {
"s0": Use(int),
"s1": Use(float),
"s2": Use(bool),
},
"nested": {"alphabet": {"zero": Use(int), "one": Use(float)}},
}
)
conf.set_schema(schema)
conf.validate()
rich.print(conf)
def complex_data():
np.random.seed(666)
placeholders = []
env_variables = []
pl_generator = PlaceholderGenerator()
sample_dict = {
"one": pl_generator.create_placeholder(
placeholders, "int", [6, 7, 8, "default=10"]
),
"two": np.random.randint(-50, 50, (3, 3)).tolist(),
"three": {
"3.1": "TrueValue",
"2.1": [False, False],
"name": {
"name_1": pl_generator.create_placeholder(placeholders, "float"),
"name_2": 2,
"name_3": {
"this_is_a_list": [3.3, 3.3],
"this_is_A_dict": {
"a": {
"f": True,
"s": False,
"numpy_array": np.array([1.0, 2, 3]),
"numpy_data": np.array([1.0, 2, 3])[0],
"tuple": ("a", "b", "2"),
"boxlist": BoxList([1, 2, 3, 4]),
"boxdict": Box({"a": 2.2}),
}
},
},
},
},
"first": {
"f1": pl_generator.create_placeholder(
placeholders, "str", ["alpha", "beta", "gamma"]
), # placeholders[2],
"f2": 2.22,
"f3": [3, 3, 3, 3, 3, 3],
"external": {
"ext": np.random.uniform(-2, 2, (2, 3)).tolist(),
"ext_name": [pl_generator.create_placeholder(placeholders, "bool")],
},
"ops": {
"o1": pl_generator.create_placeholder(placeholders), # placeholders[4],
"o2": pl_generator.create_placeholder(
placeholders, "path"
), # placeholders[5],
"o3": pl_generator.create_placeholder(
placeholders, "date"
), # placeholders[6],
"o4": pl_generator.create_placeholder(
placeholders, "str", ["A", "B", "C", "default=C"]
), # placeholders[7],
"o5": pl_generator.create_placeholder(placeholders), # placeholders[8],
"o6": pl_generator.create_placeholder(placeholders), # placeholders[9]
},
"env": [
pl_generator.create_placeholder(
env_variables, "env"
), # env_variables[0],
pl_generator.create_placeholder(env_variables, "env"),
],
"key": {
"with": 120,
"dots": pl_generator.create_placeholder(placeholders, "object"),
},
"key.with.dots": pl_generator.create_placeholder(
placeholders
), # placeholders[10],
"key.with...many.....dots": pl_generator.create_placeholder(placeholders),
"nested.key": {"with.dots": pl_generator.create_placeholder(placeholders)},
"placeholder_object": pl_generator.create_placeholder(placeholders),
"placeholder_cfg": pl_generator.create_placeholder(
placeholders, "cfg"
), # placeholders[15],
"placeholder_cfg_root": pl_generator.create_placeholder(
placeholders, "cfg_root"
), # ,
},
}
schema = Schema(
{
"one": Or(int, str),
"two": list,
"three": {
"3.1": str,
"2.1": [bool],
"name": dict,
},
"first": {Regex(""): Or(str, int, list, float, dict)},
}
)
to_be_raplaced_keys = sorted(
[
"three.name",
"three.name.name_3",
"three.name.name_3.this_is_A_dict",
"first",
"first.external",
],
reverse=True,
)
return {
"data": XConfig.decode(sample_dict),
"schema": schema,
"placeholders": placeholders,
"environment_variables": env_variables,
"to_be_replaced": to_be_raplaced_keys,
}
def test_creation(
self, generic_temp_folder, data, cfg_extension
): # TODO: toml 0.10.2 needed! toml has a bug otherwise
generic_temp_folder = Path(generic_temp_folder)
sample_dict = data["data"]
to_be_raplaced_keys = data[
"to_be_replaced"
] # , _, _, to_be_raplaced_keys = data
volatile_dict = copy.deepcopy(sample_dict)
subtitutions_values = {}
for k in to_be_raplaced_keys:
random_name = k + f".{cfg_extension}"
random_name = DirectiveAT.generate_directive_string("import", [random_name])
subtitutions_values[random_name] = pydash.get(volatile_dict, k)
pydash.set_(volatile_dict, k, random_name)
output_cfg_filename = generic_temp_folder / f"out_config.{cfg_extension}"
output_cfg_filename2 = generic_temp_folder / f"out_config2.{cfg_extension}"
subtitutions_values[str(output_cfg_filename)] = volatile_dict
saved_cfgs = []
for directive_value, d in subtitutions_values.items():
directive = DirectiveAT(value=directive_value)
if directive.valid:
output_filename = generic_temp_folder / directive.args[0]
else:
output_filename = generic_temp_folder / directive_value
store_cfg(output_filename, d)
saved_cfgs.append(output_filename)
yconf = XConfig(output_cfg_filename)
yconf.save_to(output_cfg_filename2)
yconf_reloaded = XConfig(output_cfg_filename2)
with pytest.raises(NotImplementedError):
yconf.save_to(str(output_cfg_filename2) + "#IMPOSSIBLE.EXTENSION")
assert not DeepDiff(yconf.to_dict(), sample_dict)
assert not DeepDiff(yconf_reloaded.to_dict(), sample_dict)
assert not DeepDiff(
XConfig.from_dict(yconf_reloaded.to_dict()).to_dict(),
yconf_reloaded.to_dict(),
)
assert len(XConfig.from_dict(yconf_reloaded.to_dict())) > len(
sample_dict
) # YConf contains 2 more private keys!
# remove last cfg file
saved_cfgs[0].unlink()
with pytest.raises(OSError):
yconf = XConfig(output_cfg_filename)