default_value="critical",
is_in_list=ERR_LEVELS,
to_lower=True,
description="level to log information to the console",
),
"format_str": Text(
default_value="%(name)-12s: %(levelname)-8s %(message)s"
),
},
KPH("file", exact=True, populate=True, required=False): {
"level": Text(
default_value="critical",
is_in_list=ERR_LEVELS,
to_lower=True,
description="level to log information to a file",
),
"format_str": Text(
default_value="%(filename)s:%(lineno)s - %(funcName)20s()][%(levelname)-8s]: %(message)s"
),
},
KPH("track", exact=True, populate=True, required=False): {
"tracker_steps": Numeric(
default_value=0,
is_type=int,
description="the frequency (as a number of training steps) at which to log tracker information",
),
"tensorboard": {"param_steps": Numeric(default_value=0, is_type=int)},
},
}
}
SINGLE = {"my_multi": Multi(required=True)}
SINGLE_default = {"my_multi": Multi(default_value=["a", "b"], required=True)}
SINGLE_default_tup = {
"my_multi": Multi(default_value=("a", "b"), required=True)
}
SINGLE_req_false = {"my_multi": Multi(required=False)}
SINGLE_inner_str = {"my_multi": Multi(required=True, element_types=str)}
SINGLE_homogeneous = {"my_multi": Multi(required=True, homogeneous=True)}
SINGLE_homogeneous_float = {
"my_multi": Multi(required=True, homogeneous=True, element_types=float)
}
SINGLE_is_list = {"my_multi": Multi(required=True, is_type=list)}
SINGLE_multi_inner_types = {"my_multi": Multi(element_types=(int, str))}
SINGLE_Numeric_float = {
"my_multi": Multi(element_types=Numeric(is_type=float))
}
SINGLE_Text_lower = {"my_multi": Multi(element_types=Text(to_lower=True))}
SINGLE_Text_lower_tuple = {
"my_multi": Multi(element_types=Text(to_lower=True), is_type=tuple)
}
SINGLE_bool = {"my_multi": Multi(element_types=bool)}
multi_int_unique = {"my_multi": Multi(element_types=int, elements_unique=True)}
# fn
def add_letter_n_times(raw, letter="B", times=1):
for i, r in enumerate(raw):
raw[i] = r + f"{letter}" * times
return raw
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder,
ValuePlaceholder,
)
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_mixed_all_single = {
"my_mixed": CD(
{
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
}
)
}
A_mixed_outter = CD(
{
"kd_num": Numeric(default_value=int(0), required=False, is_type=int),
KeyPlaceholder("some_str", ends_with="_str"): Text(),
KeyPlaceholder("some_num"): ValuePlaceholder("user_num"),
"wild_card": ValuePlaceholder("wild_card_value"),
}
)
from crummycm.validation.types.placeholders.placeholder import KeyPlaceholder
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_ex = {"val": Numeric(default_value=int(0), required=False, is_type=int)}
A_unnamed_single_num_ex = {
"config":
UnnamedDict({
KeyPlaceholder("some_key"):
Numeric(default_value=int(0), required=False, is_type=int)
})
}
A_unnamed_single_exact = {
"config":
UnnamedDict({
KeyPlaceholder("some_key", exact=True):
Numeric(default_value=2, required=True, is_type=int)
})
}
A_unnamed_single_num_multi_ex = {
"config":
UnnamedDict({
KeyPlaceholder("some_key", multi=True):
Numeric(default_value=int(0), required=False, is_type=int)
})
}
" > e.g. meta:yeahml_dir: 'yeahml'"),
),
"data_name":
Text(description=(
"Description of the data used \n"
" > e.g. meta:data_name: 'mnist', or meta:data_name: 'V00'\n"
"this logic will likely change in the future")),
"experiment_name":
Text(description=("Name for the experiment being performed\n"
" > e.g. meta:experiment_name: 'trial_00'")),
KPH("start_fresh", exact=True, required=False, populate=True):
Bool(
default_value=False,
description=
("Used to determine whether previous experiments should be deleted\n"
" > e.g. meta:start_fresh: True"),
),
KPH("random_seed", exact=True, required=False):
Numeric(
is_type=int,
description=("Used to set the random seed for tensorflow\n"
" > e.g. meta:rand_seed: 42"),
),
# TODO: ensure path exists
KPH("default_load_params_path", exact=True, required=False):
Text(),
# TODO: tracing
# "trace_level": Text(description="meta:trace_level: <str>"),
}
}
from crummycm.validation.types.dicts.config_dict import ConfigDict as CD
from crummycm.validation.types.placeholders.placeholder import KeyPlaceholder as KPH
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
ACCEPTED_DTYPES = ["int32", "float32", "int64", "float64"]
DATA = CD({
"data": {
"name": Text(required=True, to_lower=True),
"schema": {
KPH(name="feature_name", multi=True, required=True): {
"shape":
Multi(required=True,
is_type=list,
element_types=Numeric(is_type=int)),
"dtype":
Text(required=True, is_in_list=ACCEPTED_DTYPES),
KPH("label", exact=True, required=False):
Bool(required=True),
}
},
"source": Text(required=True, starts_with="http:"),
}
})
from crummycm.validation.types.values.compound.multi import Multi
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.compound.either import Either
Num_or_Text_list = {
"my_either":
Either(either_seq=[
Numeric(required=False, is_type=float),
Text(required=False)
])
}
Num_or_Text_tuple = {
"my_either":
Either(either_seq=(Numeric(required=False, is_type=float),
Text(required=False)))
}
return_list = {
"my_either":
Either(
either_seq=[
Numeric(required=False, is_type=float),
Text(required=False)
],
return_as_type=list,
)
}
text_or_list = {
from crummycm.validation.types.placeholders.placeholder import KeyPlaceholder
from crummycm.validation.types.dicts.foundation.unnamed_dict import UnnamedDict
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_known_uk = {
"known_dict":
KnownDict({
"kd_num":
Numeric(default_value=int(0), required=False, is_type=int),
"uk_dict":
UnnamedDict({
KeyPlaceholder("a_num"):
Numeric(default_value=int(0), required=False, is_type=int)
}),
})
}
A_uk_k = {
"my_conf":
UnnamedDict({
KeyPlaceholder("a_num"):
KnownDict({
"my_num":
Numeric(default_value=int(0), required=False, is_type=int)
})
})
}
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
from yeahml.build.components.optimizer import return_available_optimizers
OPTIMIZE = {
"optimize": {
"optimizers": {
KPH("optimizer_name", multi=True): {
"type":
Text(
is_in_list=return_available_optimizers(),
to_lower=True,
description=(
"The type of optimizer being used\n"
" > e.g. optimize:optimizers:'name':type: 'adam'"),
),
KPH("options", exact=True, required=False): {
"learning_rate": Numeric(is_type=float),
KPH("other_options", required=False):
VPH("optimizer_options"),
},
"objectives":
VPH("optimizer_objectives"),
}
}
# "directive": {"instructions": "SS"},
}
}
import crummycm as ccm
from crummycm.validation.types.placeholders.placeholder import (
KeyPlaceholder as KPH,
ValuePlaceholder as VPH,
)
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
from crummycm.validation.types.values.element.bool import Bool
from crummycm.validation.types.values.compound.multi import Multi
HYPER_PARAMETERS = {
"hyper_parameters": {
"dataset": {
"batch":
Numeric(is_type=int,
description="hyper_parameters:dataset:batch: <int>"),
KPH("shuffle_buffer", exact=True, required=False):
Numeric(
is_type=int,
description="hyper_parameters:dataset:shuffle_buffer: <int>",
),
},
"epochs":
Numeric(is_type=int, description="hyper_parameters:epochs: <int>"),
KPH("early_stopping", exact=True, required=False): {
"epochs":
Numeric(is_type=int, description="patience"),
"warm_up":
Numeric(
is_type=int,
description="allow training to 'warm up' before keeping track",
from crummycm.validation.types.values.element.numeric import Numeric
A_EX_TEMP = {
"my_num": Numeric(default_value=int(0), required=False, is_type=int)
}
# type
A_required_EX_TEMP = {"my_num": Numeric(required=True, is_type=float)}
# # type
A_int_EX_TEMP = {"my_num": Numeric(is_type=int)}
A_float_EX_TEMP = {"my_num": Numeric(is_type=float)}
# fn
def add_number_n_times(raw, number=1, times=1):
out = raw + (number * times)
return out
A_fn_EX_TEMP = {"my_num": Numeric(fn=add_number_n_times, is_type=int)}
A_fn_kwargs_EX_TEMP = {
"my_num":
Numeric(fn=add_number_n_times,
fn_kwargs={
"number": 1,
"times": 10
},
is_type=int)
}
A_fn_bad_kwargs_EX_TEMP = {
"my_num":
Numeric(fn=add_number_n_times,
from crummycm.validation.types.dicts.foundation.known_dict import KnownDict
from crummycm.validation.types.values.element.numeric import Numeric
from crummycm.validation.types.values.element.text import Text
# from crummycm.validation.types.values.base import BaseValue
A_ex = {
"my_num": Numeric(default_value=int(0), is_type=int),
"my_str": Text(default_value="Jack", is_type=str),
}
A_named_ex = {
"config":
KnownDict({
"my_num": Numeric(default_value=int(0), is_type=int),
"my_str": Text(default_value="Jack", is_type=str),
})
}
A_named_out = KnownDict(
{"my_num": Numeric(default_value=int(0), required=False, is_type=int)})
A_nested_known_ex = {
"config":
KnownDict({
"my_num":
Numeric(default_value=int(0), is_type=int),
"my_dict":
KnownDict({
"my_num": Numeric(default_value=0, is_type=int),
"my_str": Text(default_value="Jack", is_type=str),