ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
ERR_LEVELS = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
ERR_LEVELS = ERR_LEVELS + [x.lower() for x in ERR_LEVELS]
LOGGING = {
"logging": {
KPH("console", exact=True, populate=True, required=False): {
"level": Text(
default_value="critical",
is_in_list=ERR_LEVELS,
to_lower=True,
description="level to log information to the console",
),
"format_str": Text(
default_value="%(name)-12s: %(levelname)-8s %(message)s"
),
},
KPH("file", exact=True, populate=True, required=False): {
"level": Text(
default_value="critical",
is_in_list=ERR_LEVELS,
to_lower=True,
description="level to log information to a file",
),
"format_str": Text(
default_value="%(filename)s:%(lineno)s - %(funcName)20s()][%(levelname)-8s]: %(message)s"
import crummycm as ccm
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder as KPH,
ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
from crummycm.validation.types.values.compound.either import Either
MODEL = {
"model": {
"name": Text(),
"start_fresh": Bool(),
"layers": {
KPH("layer_name", multi=True): {
"type":
Text(),
KPH("source", required=False, exact=True):
Text(),
KPH("options", required=False, exact=True): {
KPH("layer_option_key", multi=True):
VPH("layer_option_value"),
KPH("activation", exact=True, required=False): {
"type":
Text(),
KPH("options_key", required=False, multi=True):
VPH("options_value"),
},
},
import crummycm as ccm
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder as KPH,
ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
from yeahml.build.components.callbacks.callbacks import return_available_callbacks
CALLBACKS = {
KPH("callbacks", exact=True, required=False): {
"objects": {
KPH("callback_name", multi=True): {
"type":
Text(to_lower=True, is_in_list=return_available_callbacks()),
KPH("options", exact=True, required=False): {
KPH("options_key", multi=True, required=False):
VPH("options_value")
},
}
}
}
}
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder,
ValuePlaceholder,
)
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
cd_outer = CD({
"my_mixed":
CD({
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
})
})
no_cd_single = {
"my_mixed": {
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
}
}
no_cd_single_nested = {
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder,
ValuePlaceholder,
)
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_mixed_all_single = {
"my_mixed": CD(
{
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
}
)
}
A_mixed_outter = CD(
{
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
}
)
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder as KPH,
ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
META = {
"meta": {
KPH("yeahml_dir", exact=True, required=False, populate=True):
Text(
default_value="yeahml",
description=("Root directory to store information\n"
" > e.g. meta:yeahml_dir: 'yeahml'"),
),
"data_name":
Text(description=(
"Description of the data used \n"
" > e.g. meta:data_name: 'mnist', or meta:data_name: 'V00'\n"
"this logic will likely change in the future")),
"experiment_name":
Text(description=("Name for the experiment being performed\n"
" > e.g. meta:experiment_name: 'trial_00'")),
KPH("start_fresh", exact=True, required=False, populate=True):
Bool(
default_value=False,
description=
("Used to determine whether previous experiments should be deleted\n"
" > e.g. meta:start_fresh: True"),
SINGLE_default = {"my_multi": Multi(default_value=["a", "b"], required=True)}
SINGLE_default_tup = {
"my_multi": Multi(default_value=("a", "b"), required=True)
}
SINGLE_req_false = {"my_multi": Multi(required=False)}
SINGLE_inner_str = {"my_multi": Multi(required=True, element_types=str)}
SINGLE_homogeneous = {"my_multi": Multi(required=True, homogeneous=True)}
SINGLE_homogeneous_float = {
"my_multi": Multi(required=True, homogeneous=True, element_types=float)
}
SINGLE_is_list = {"my_multi": Multi(required=True, is_type=list)}
SINGLE_multi_inner_types = {"my_multi": Multi(element_types=(int, str))}
SINGLE_Numeric_float = {
"my_multi": Multi(element_types=Numeric(is_type=float))
}
SINGLE_Text_lower = {"my_multi": Multi(element_types=Text(to_lower=True))}
SINGLE_Text_lower_tuple = {
"my_multi": Multi(element_types=Text(to_lower=True), is_type=tuple)
}
SINGLE_bool = {"my_multi": Multi(element_types=bool)}
multi_int_unique = {"my_multi": Multi(element_types=int, elements_unique=True)}
# fn
def add_letter_n_times(raw, letter="B", times=1):
for i, r in enumerate(raw):
raw[i] = r + f"{letter}" * times
return raw
def reversed_plus_letter(raw, letter="Z", times=1):
from crummycm.validation.types.values.compound.multi import Multi
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.compound.either import Either
Num_or_Text_list = {
"my_either":
Either(either_seq=[
Numeric(required=False, is_type=float),
Text(required=False)
])
}
Num_or_Text_tuple = {
"my_either":
Either(either_seq=(Numeric(required=False, is_type=float),
Text(required=False)))
}
return_list = {
"my_either":
Either(
either_seq=[
Numeric(required=False, is_type=float),
Text(required=False)
],
return_as_type=list,
)
}
text_or_list = {
ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
# NOTE: neither loss nor metric are required, and yet, without one it is useless
# the current config management does not allow for this.
PERFORMANCE = {
"performance": {
"objectives": {
KPH("objective_name", multi=True): {
KPH("loss", exact=True, required=False): {
"type": Text(),
KPH("options", exact=True, required=False):
VPH("loss_options"),
KPH("track", exact=True, required=False):
VPH("loss_track"),
},
KPH("metric", exact=True, required=False): {
"type": Text(),
KPH("options", exact=True, required=False):
VPH("loss_options"),
KPH("track", exact=True, required=False):
VPH("loss_track"),
},
"in_config": {
"type": Text(),
KPH("options", exact=True, required=False): {
# from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
required_num_false = {
"my_conf": CD({KeyPlaceholder("my_val"): ValuePlaceholder("some_val")})
}
required_num_true = {
"my_conf":
CD({KeyPlaceholder("my_val", required=True): ValuePlaceholder("some_val")})
}
required_num_true_true = {
"my_conf":
CD({KeyPlaceholder("my_text", required=True): Text(required=True)})
}
required_num_true_false = {
"my_conf":
CD({KeyPlaceholder("my_text", required=True): Text(required=False)})
}
required_num_true_false_default = {
"my_conf":
CD({KeyPlaceholder("my_text", required=True): Text(default_value="Jack")})
}
# if the key is present, the value is required
# if the key isn't present, ignore
required_num_false_true = {
KeyPlaceholder,
ValuePlaceholder,
)
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
flat_a = {
"my_mixed":
CD({
"kd_num":
Numeric(default_value=int(0), is_type=int),
KeyPlaceholder("some_str", ends_with="_str", required=False):
Text(default_value="DIESEL"),
KeyPlaceholder("some_num", required=True):
ValuePlaceholder("user_num"),
"wild_card":
ValuePlaceholder("wild_card_value"),
})
}
nested_a = {
"my_mixed":
CD({
"kd_num":
Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"):
Text(),
KeyPlaceholder("some_num"):
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.placeholders.placeholder import KeyPlaceholder as KPH
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
ACCEPTED_DTYPES = ["int32", "float32", "int64", "float64"]
DATA = CD({
"data": {
"name": Text(required=True, to_lower=True),
"schema": {
KPH(name="feature_name", multi=True, required=True): {
"shape":
Multi(required=True,
is_type=list,
element_types=Numeric(is_type=int)),
"dtype":
Text(required=True, is_in_list=ACCEPTED_DTYPES),
KPH("label", exact=True, required=False):
Bool(required=True),
}
},
"source": Text(required=True, starts_with="http:"),
}
})
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
from yeahml.build.components.dtype import return_available_dtypes
DATA = {
"data": {
"datasets": {
KPH("dataset_name", multi=True): {
"in": {
KPH("feat_name", multi=True): {
# required false is a bit of a hacky
"shape":
Multi(element_types=int, required=False),
"dtype":
Text(is_in_list=return_available_dtypes()),
KPH("startpoint",
exact=True,
required=False,
populate=True):
Bool(default_value=True),
KPH("endpoint",
exact=True,
required=False,
populate=True):
Bool(default_value=False),
KPH("label", exact=True, required=False, populate=True):
Bool(default_value=False),
}
},
"split": {
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
from yeahml.build.components.optimizer import return_available_optimizers
OPTIMIZE = {
"optimize": {
"optimizers": {
KPH("optimizer_name", multi=True): {
"type":
Text(
is_in_list=return_available_optimizers(),
to_lower=True,
description=(
"The type of optimizer being used\n"
" > e.g. optimize:optimizers:'name':type: 'adam'"),
),
KPH("options", exact=True, required=False): {
"learning_rate": Numeric(is_type=float),
KPH("other_options", required=False):
VPH("optimizer_options"),
},
"objectives":
VPH("optimizer_objectives"),
}
}
# "directive": {"instructions": "SS"},
}
}
from crummycm.validation.types.values.element.text import Text
A_EX_TEMP = {"my_text": Text(default_value="Jack")}
# type
A_required_EX_TEMP = {"my_text": Text(required=True)}
# built_in
A_to_lower = {"my_text": Text(to_lower=True)}
A_is_in_list_true = {"my_text": Text(is_in_list=["Jack", "Diesel", "Abby"])}
A_contains_true = {"my_text": Text(contains=".jpg")}
A_contains_one_of_true = {"my_text": Text(contains_one_of=[".jpg", ".png"])}
A_starts_with_True = {"my_text": Text(starts_with="j")}
A_starts_with_False = {"my_text": Text(starts_with="x")}
A_ends_with_True = {"my_text": Text(ends_with=".jpg")}
A_ends_with_False = {"my_text": Text(ends_with=".xxx")}
# fn
def add_letter_n_times(raw, letter="B", times=1):
out = raw + f"{letter}" * times
return out
A_fn = {"my_text": Text(fn=add_letter_n_times)}
A_fn_kwargs = {
"my_text": Text(fn=add_letter_n_times,
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_ex = {
"my_num": Numeric(default_value=int(0), is_type=int),
"my_str": Text(default_value="Jack", is_type=str),
}
A_named_ex = {
"config":
KnownDict({
"my_num": Numeric(default_value=int(0), is_type=int),
"my_str": Text(default_value="Jack", is_type=str),
})
}
A_named_out = KnownDict(
{"my_num": Numeric(default_value=int(0), required=False, is_type=int)})
A_nested_known_ex = {
"config":
KnownDict({
"my_num":
Numeric(default_value=int(0), is_type=int),
"my_dict":
KnownDict({
"my_num": Numeric(default_value=0, is_type=int),
"my_str": Text(default_value="Jack", is_type=str),