class BasicTrainConfig(BaseConfig):
"""Basic training configuration
Required parameters:
- train_dataloader (Iterable with length): training data loader
- model (`torch.nn.Module`): model to train
- optimizer (`torch.optim.Optimizer`): optimizer to use for training
- criterion (`torch.nn.Module`): loss function to use for training
- num_epochs (int): number of epochs
Optional parameters:
- metrics (dict): dictionary with ignite metrics, e.g `{'precision': Precision()}`
- ...
"""
train_dataloader = attr.ib(validator=is_iterable_with_length, default=None)
model = attr.ib(validator=instance_of(nn.Module), default=None)
optimizer = attr.ib(validator=instance_of(Optimizer), default=None)
criterion = attr.ib(validator=instance_of(nn.Module), default=None)
num_epochs = attr.ib(validator=and_(instance_of(int), is_positive),
default=None)
metrics = attr.ib(default={},
validator=optional(is_dict_of_key_value_type(
str, Metric)))
log_interval = attr.ib(default=100,
validator=optional(
and_(instance_of(int), is_positive)))
trainer_checkpoint_interval = attr.ib(default=1000,
validator=optional(
and_(instance_of(int),
is_positive)))
model_checkpoint_kwargs = attr.ib(default=None,
validator=optional(instance_of(dict)))
lr_scheduler = attr.ib(default=None,
validator=optional(instance_of(_LRScheduler)))
reduce_lr_on_plateau = attr.ib(default=None,
validator=optional(
instance_of(ReduceLROnPlateau)))
reduce_lr_on_plateau_var = attr.ib(default='loss',
validator=optional(instance_of(str)))
val_dataloader = attr.ib(default=None,
validator=optional(is_iterable_with_length))
val_metrics = attr.ib(default=None,
validator=optional(
is_dict_of_key_value_type(str, Metric)))
val_interval_epochs = attr.ib(default=1,
validator=optional(
and_(instance_of(int), is_positive)))
train_eval_dataloader = attr.ib(
default=None, validator=optional(is_iterable_with_length))
early_stopping_kwargs = attr.ib(default=None,
validator=optional(instance_of(dict)))
class SolverConfig(_BaseConfig):
"""
"""
optimizer = attr.ib(default=_dummy_optim, validator=instance_of(Optimizer))
criterion = attr.ib(default=nn.Module(), validator=instance_of(nn.Module))
num_epochs = attr.ib(default=None,
validator=optional(and_(instance_of(int),
is_positive)))
num_iterations = attr.ib(default=None,
validator=optional(
and_(instance_of(int), is_positive)))
class InventoryModel:
goods: Dict[str, float] = attrib(validator=validators.deep_mapping(
key_validator=validators.and_(validators.instance_of(str),
non_empty_str),
value_validator=validators.and_(
validators.instance_of(float),
positive_int # type: ignore
),
mapping_validator=validators.instance_of(dict),
))
discounts: Dict[str, DiscountModel] = attrib(
validator=validators.deep_mapping(
key_validator=validators.instance_of(str),
value_validator=validators.instance_of(DiscountModel),
mapping_validator=validators.instance_of(dict),
),
converter=defaultdict_of_classes( # type: ignore
DiscountModel, lambda: NO_DISCOUNT),
)
multibuy: Dict[str, MultibuyModel] = attrib(
validator=validators.deep_mapping(
key_validator=validators.instance_of(str),
value_validator=validators.instance_of(MultibuyModel),
mapping_validator=validators.instance_of(dict),
),
converter=defaultdict_of_classes( # type: ignore
MultibuyModel, lambda: NO_MULTIBUY),
)
def __attrs_post_init__(self):
goods = set(self.goods.keys())
# checks if not discounting unknown items
for attr in ["discounts", "multibuy"]:
unknown = set(getattr(self, attr).keys()).difference(goods)
if len(unknown) > 0:
raise ValueError(
f"Unknown goods as key in '{attr}': {unknown}")
unknown = {
multibuy_item.discounts_goods
for multibuy_item in self.multibuy.values()
}.difference(goods)
if len(unknown) > 0:
raise ValueError(
f"Unknown goods as value for 'multibuy.*.discounted_goods': {unknown}"
)
def sequence_of(validator, allow_empty=False):
if isinstance(validator, list):
validator = and_(*validator)
def validate(inst, attr, value):
is_seq(inst, attr, value)
if not allow_empty and not value:
raise ValueError("'{name}' cannot be empty".format(name=attr.name,
value=value))
for i, item in enumerate(value):
try:
validator(inst, attr, item)
except (ValueError, TypeError):
# now that we know it's invalid, let's re-do the validation
# with a better attribute name. Since we have to copy data,
# we only do it when we know we have to raise an exception.
item_attr = copy.copy(attr)
object.__setattr__(item_attr, 'name',
'{}[{}]'.format(attr.name, i))
try:
validator(inst, item_attr, item)
except Exception as exc:
six.raise_from(exc, None)
else:
# if we somehow got here, raise original exception
raise
return validate
def test_success(self):
"""
Succeeds if all wrapped validators succeed.
"""
v = and_(instance_of(int), always_pass)
v(None, simple_attr("test"), 42)
def test_success(self):
"""
Succeeds if all wrapped validators succeed.
"""
v = and_(instance_of(int), always_pass)
v(None, simple_attr("test"), 42)
def test_fail(self):
"""
Fails if any wrapped validator fails.
"""
v = and_(instance_of(int), always_fail)
with pytest.raises(ZeroDivisionError):
v(None, simple_attr("test"), 42)
def test_fail(self):
"""
Fails if any wrapped validator fails.
"""
v = and_(instance_of(int), always_fail)
with pytest.raises(ZeroDivisionError):
v(None, simple_attr("test"), 42)
def implicit_or(validator):
if isinstance(validator, list):
validator = and_(*validator)
def validate(inst, attr, value):
if value == IMPLICIT_ZERO:
return
validator(inst, attr, value)
return validate
class MemoryAmount:
"""
An amount of memory, consisting of an *amount*
paired with its corresponding `MemoryUnit` *unit*.
"""
amount: int = attrib(validator=and_(instance_of(int), in_(Range.at_least(1))))
unit: MemoryUnit = attrib(validator=None)
_PARSE_PATTERN = re.compile(r"(\d+) ?([TtGgMmKk])[bB]?")
@staticmethod
def parse(memory_string: str) -> "MemoryAmount":
parts = MemoryAmount._PARSE_PATTERN.match(memory_string)
if parts:
return MemoryAmount(
amount=int(parts.group(1)), unit=MemoryUnit.parse(parts.group(2))
)
else:
raise RuntimeError(
f"Cannot parse {memory_string} as an amount of memory. "
f"Expected an integer followed by K, M, G, or T"
)
class C(object):
a1 = attr.ib("a1", validator=and_(instance_of(int), ))
a2 = attr.ib("a2", validator=[
instance_of(int),
])
class C:
a1 = attr.ib("a1", validator=and_(instance_of(int)))
a2 = attr.ib("a2", validator=[instance_of(int)])
def sequence_of(types):
def validator(_, attrib, value):
assert all(isinstance(x, types) for x in value), attrib.name
return and_(sequence, validator)
class ConfigDocFormat():
"""
Class to store formats for printing out configuration setting information.
Tries to handle a lot of heavy lifting and provide a nicer interface for
common case definitions and a convenient interface for template authors.
Attributes:
doc (str): the base docstring to be used by the template.
template (str or jinja2.Template): Template that is used to render
the final docstring for a full config group.
??? Info "Available Template Variables"
These will be variables the template has access to:
- `doc`: primary documentation for the root config group
- `val_table`: dict w/ various info on each var.
- `exists`: Does the table exist? Other entries will not be
there if this is false.
- `name`: The name of the table
- `headers`: column headings (in definition order)
- `entries`: list of dicts for each table entry, where the
keys are the column headings and the values are the
formatted strings.
**Note:** Any variables available to format specifiers, like
`is_group` or `is_val`, will also be made available
to the template under each entry.
- `group_table`: dict w/ various info on groups
- `exists`: Does the table exist
- `name`: The name of the table
- `headers`: column headings (in definition order)
- `entries`: list of dicts for each table entry, where the
keys are the column headings and the values are the
formatted strings.
**Note:** Any variables available to format specifiers, like
`is_group` or `is_val`, will also be made available
to the template under each entry.
- `combined_table`: A table of combined vars and groups
- `exists`: Does the table exist
- `name`: The name of the table
- `headers`: column headings (in definition order)
- `entries`: list of dicts for each table entry, where the
keys are the column headings and the values are the
formatted strings.
**Note:** Any variables available to format specifiers, like
`is_group` or `is_val`, will also be made available
to the template under each entry.
- `group_col_widths`: map from keys to colwidths for a group.
Should be used to set the `colspan` parameter in table
cells if needed.
The width of a column will be `0` if it should be
omitted.
- `val_col_widths`: map from keys to colwidths for a value
Should be used to set the `colspan` parameter in table
cells if needed.
The width of a column will be `0` if it should be
omitted.
val_table_name (dict): The title for the value table.
val_table_format (dict): Dictionary used to set the columns the val
table. Each key should be a column heading, with the value being
a format string used to format the entry.
Use `None` if you don't want the val table to appear at all. The
table will also be ommitted if there are no entries for it.
??? Info "Available Format Specifiers"
These are the keys that will be available to the format strings.
- `is_val`: is this a variable?
- `is_group`: is this a group?
- `defined_in`: the module where their was defined / last updated
- `val`: the value the term was set to
- `val_repr`: thst string representation of a value.
- `doc`: the provided docstring for the term
- `name`: the name of the term, only the final component
- `path`: the full pathname of the value
group_table_name (dict): The title for the subgroup table.
group_table_format (dict): Dictionary used to set the columns the group
table. Each key should be a column heading, with the value being
a format string used to format the entry.
Use `None` if you don't want the group table to appear at all. The
table will also be ommitted if there are no entries for it.
??? Info "Available Format Specifiers"
These are the keys that will be available to the format strings.
- `is_val`: is this a variable?
- `is_group`: is this a group?
- `defined_in`: the module where their was defined / last updated
- `doc`: the provided docstring for the term
- `name`: the name of the term, only the final component
- `path`: the full pathname of the value
combine_tables (bool): Do we combine the value and group tables into a
single joint table?
combined_table_name (str): The title for the combined table.
combined_table_header (list[str]): An ordered list of headers to use for
a combined entry table. These should match keys in the `table_format`
parameters.
recurse_entries (bool): If `True` we recurse to all entries and subgroups
otherwise we only print the direct children of the config group.
other_vars (dict): other variables to be passed to the template directly.
"""
template = attr.ib()
doc = attr.ib(default="")
@template.validator
def __template_validator(self, attr, val):
if isinstance(val, str):
self.template = jinja2.Template(textwrap.dedent(val))
return None
elif isinstance(val, jinja2.Template):
self.template = val
else:
raise TypeError(
("ConfigDocFormat expects a template of type `str` or of " +
"type `jinja2.Template`. Instead got a template of type {}"
).format(type(val)))
val_table_name = attr.ib(
default="",
validator=valid.instance_of(str),
)
def __normalize_format_dict(format_dict):
return {
key: textwrap.dedent(val)
for (key, val) in format_dict.items()
}
val_table_format = attr.ib(
default=None,
converter=__normalize_format_dict,
validator=valid.optional(
valid.deep_mapping(
key_validator=valid.instance_of(str),
value_validator=valid.instance_of(str),
)),
)
val_table_headers = attr.ib(validator=valid.optional(
valid.and_(
valid.instance_of(list),
valid.deep_iterable(valid.instance_of(str)),
)), )
@val_table_headers.default
def val_table_headers_default(self):
hide_doc = True # Docs must be the first statement to show up
"""
List of headers for the value table.
"""
if self.val_table_format != None:
keys = list(self.val_table_format)
log.debug(
textwrap.dedent("""
Attempting to get ordered headers for val table:
Dict:
%s
List:
%s
"""),
pformat(self.val_table_format),
pformat(keys),
)
return keys
else:
return []
group_table_headers = attr.ib(validator=valid.optional(
valid.and_(
valid.instance_of(list),
valid.deep_iterable(valid.instance_of(str)),
)), )
@group_table_headers.default
def group_table_headers_default(self):
hide_doc = True # Docs must be the first statement to show up
"""
List of headers for the group table
"""
if self.group_table_format != None:
return list(self.group_table_format)
else:
return []
group_table_name = attr.ib(
default="",
validator=valid.instance_of(str),
)
group_table_format = attr.ib(
default=None,
converter=__normalize_format_dict,
validator=valid.optional(
valid.deep_mapping(
key_validator=valid.instance_of(str),
value_validator=valid.instance_of(str),
)),
)
combine_tables = attr.ib(
default=False,
validator=valid.instance_of(bool),
)
combined_table_name = attr.ib(
default="",
validator=valid.instance_of(str),
)
combined_table_headers = attr.ib(validator=valid.optional(
valid.and_(
valid.instance_of(list),
valid.deep_iterable(valid.instance_of(str)),
)), )
@combined_table_headers.default
def combined_table_headers_default(self):
hide_doc = True # Docs must be the first statement to show up
graph = graphlib.TopologicalSorter()
table_edges = list(
zip(self.val_table_headers[::2], self.val_table_headers[1::2]))
table_edges += list(
zip(self.group_table_headers[::2], self.group_table_headers[1::2]))
for (start, end) in table_edges:
graph.add(end, start)
return list(graph.static_order())
@staticmethod
def get_colwidths(combined_cols, elem_cols):
hide_doc = True # Docs must be the first statement to show up
col_widths = dict()
# get whether each combined cell is in the group cols
is_member = list(map(lambda x: x in elem_cols, combined_cols))
log.debug(
textwrap.dedent("""
Generating Colspan widths:
Combined Cols:
%s
Elem Type Cols:
%s
Membership List:
%s
"""),
pformat(combined_cols),
pformat(elem_cols),
pformat(is_member),
)
for (ind, col) in enumerate(combined_cols):
if is_member[ind]:
# Get list of members that follow, grouped by whether they're
# within our elements of interest
next_elems = is_member[ind + 1:]
next_groups = itertools.groupby(next_elems)
# This overly tedious unpacking is because itertools.groupby
# returns a list of tuples with iterators in them. I'm just
# converting them into a more sane format. You could make this
# more efficient but I can't be arsed.
next_cols = []
for k, group in next_groups:
grp_list = []
for item in list(group):
grp_list += [item]
next_cols += [grp_list]
# every valid entry is 1 col wide
width = 1
# If we're followed by a bunch of entries not in our set
# then we can grow wide enough to fill those too.
if ((len(next_cols) != 0) and (len(next_cols[0]) != 0)
and (not next_cols[0][0])):
width += len(next_cols[0])
log.debug(
textwrap.dedent("""
For col '%s' at index '%s':
Next Members:
%s
Next Groups:
%s
Final Width:
%s
"""), col, ind, next_elems, next_cols, width)
col_widths[col] = width
else:
# invalid entries should be dropped
col_widths[col] = 0
return col_widths
recurse_entries = attr.ib(default=True, validator=valid.instance_of(bool))
other_vars = attr.ib(
factory=dict,
validator=valid.and_(
valid.instance_of(dict),
valid.deep_mapping(
key_validator=valid.instance_of(str),
value_validator=(lambda _a, _b, _c: True),
),
),
)
@property
def combined_group_col_widths(self):
hide_doc = True # Docs must be the first statement to show up
"""
The column widths for a group entry in a combined table
"""
widths = ConfigDocFormat.get_colwidths(self.combined_table_headers,
self.group_table_headers)
log.debug(
textwrap.dedent("""
Genrating Group Col Widths:
Combined Headers:
%s
Group Headers:
%s
Result Widths:
%s
"""),
pformat(self.combined_table_headers),
pformat(self.group_table_headers),
pformat(widths),
)
return widths
@property
def combined_val_col_widths(self):
hide_doc = True # Docs must be the first statement to show up
"""
The column widths for a value entry in a combined table
"""
widths = ConfigDocFormat.get_colwidths(self.combined_table_headers,
self.val_table_headers)
log.debug(
textwrap.dedent("""
Genrating Value Col Widths:
Combined Headers:
%s
Value Headers:
%s
Result Widths:
%s
"""),
pformat(self.combined_table_headers),
pformat(self.val_table_headers),
pformat(widths),
)
return widths
@classmethod
def get_val_data(cls, config_val):
hide_doc = True # Docs must be the first statement to show up
"""
Exact and format data for a ConfigValue
"""
return {
config_val.path_string(): {
'is_val': True,
'is_group': False,
'val': config_val.value,
'val_repr': repr(config_val.value),
'doc': config_val.doc,
'defined_in': config_val.ctxt.__qualname__,
'name': config_val.path[-1],
'path': config_val.path_string(),
}
}
@classmethod
def get_group_data(cls, config_group):
hide_doc = True # Docs must be the first statement to show up
"""
Extract and format data for a ConfigGroup
"""
return {
config_group.path_string(): {
'is_val': False,
'is_group': True,
'doc': config_group.doc,
'name': config_group.path[-1],
'path': config_group.path_string(),
}
}
@classmethod
def collect_group_data(self, config_group, recurse=False):
hide_doc = True # Docs must be the first statement to show up
"""
Run through the members of a ConfigGroup and gather all the template
data in a dictionary, where the keys are the path to each term.
This will recurse if necessary.
"""
data = dict()
for member in config_group.members.values():
if isinstance(member, ConfigValue):
data.update(ConfigDocFormat.get_val_data(member))
else:
data.update(ConfigDocFormat.get_group_data(member))
if recurse:
data.update(
ConfigDocFormat.collect_group_data(member, recurse))
return data
@classmethod
def format_data(self, format_dict, data):
hide_doc = True # Docs must be the first statement to show up
"""
Format all the values in a format dictionary given a particular
dataset.
"""
output = dict()
for (key, format_string) in format_dict.items():
output[key] = format_string.format_map(data)
for (key, value) in data.items():
if key not in output:
output[key] = value
log.debug(
textwrap.dedent("""
Formatting Data:
Format String:
%s
Data:
%s
Result:
%s
"""),
pformat(format_dict, depth=6, indent=2),
pformat(data, depth=6, indent=2),
pformat(output, depth=6, indent=2),
)
return output
def format_val_data(self, val_data):
hide_doc = True # Docs must be the first statement to show up
"""
format a value table entry if we have a formatter for it.
"""
if self.val_table_format != None:
return self.format_data(self.val_table_format, val_data)
else:
return dict()
def format_group_data(self, group_data):
hide_doc = True # Docs must be the first statement to show up
"""
Format a group table entry if we have a formatter for it.
"""
if self.group_table_format != None:
return self.format_data(self.group_table_format, group_data)
else:
return dict()
def format_member_data(self, data):
hide_doc = True # Docs must be the first statement to show up
log.debug("Formatting Member Data:\n\n%s", pformat(data))
output = None
if data['is_val']:
output = self.format_val_data(data)
else:
output = self.format_group_data(data)
log.debug("Formatted Member Data:\n\n%s", pformat(output))
return output
def process_config_group(self, config):
hide_doc = True # Docs must be the first statement to show up
"""
Run through a configuration group to produce all the variables that
will be fed to the template.
"""
# Process data from config group
rows = self.collect_group_data(config, self.recurse_entries)
formatted = dict()
for (path, data) in rows.items():
formatted[path] = self.format_member_data(data)
log.debug("Formatted `%s` data to get:\n\n%s", path,
pformat(formatted[path]))
vals = {
path: data
for (path, data) in formatted.items() if data['is_val']
}
groups = {
path: data
for (path, data) in formatted.items() if data['is_group']
}
log.debug(
textwrap.dedent("""
Control Group Raw Contents:
Rows:
%s
Vals:
%s
Groups :
%s
"""),
pformat(rows),
pformat(vals),
pformat(groups),
)
# Figure out which tables should exist in our output.
val_table_can_exist = ((len(vals) > 0)
and (self.val_table_format != None))
group_table_can_exist = ((len(groups) > 0)
and (self.group_table_format != None))
combined_table_exists = ((val_table_can_exist
and group_table_can_exist)
and self.combine_tables)
val_table_exists = (not combined_table_exists) and val_table_can_exist
group_table_exists = (
not combined_table_exists) and group_table_can_exist
log.debug(
textwrap.dedent("""
Table Existence Data:
Val Table Can Exist: %s
Grp Table Can Exist: %s
Val Table Exists: %s
Grp Table Exists: %s
Cmb Table Exists: %s
"""),
val_table_can_exist,
group_table_can_exist,
val_table_exists,
group_table_exists,
combined_table_exists,
)
# generate top level output data
output = dict()
output.update(self.other_vars)
output['doc'] = config.doc
output['val_table'] = dict()
output['group_table'] = dict()
output['combined_table'] = dict()
# generate val table data
output['val_table']['exists'] = val_table_exists
if val_table_exists:
output['val_table']['name'] = self.val_table_name
output['val_table']['headers'] = self.val_table_headers
output['val_table']['entries'] = [
vals[path] for path in sorted(vals.keys())
]
# generate group table data
output['group_table']['exists'] = group_table_exists
if group_table_exists:
output['group_table']['name'] = self.group_table_name
output['group_table']['headers'] = self.group_table_headers
output['group_table']['entries'] = [
groups[path] for path in sorted(groups.keys())
]
# generate group table data
output['combined_table']['exists'] = combined_table_exists
if combined_table_exists:
output['combined_table']['name'] = self.combined_table_name
output['combined_table']['headers'] = self.combined_table_headers
output['combined_table']['entries'] = [
formatted[path] for path in sorted(formatted.keys())
]
output['combined_table']['group_col_widths'] = (
self.combined_group_col_widths)
output['combined_table']['val_col_widths'] = (
self.combined_val_col_widths)
log.debug("Processed Doc Data:\n\n%s", pformat(output))
return output
@staticmethod
def render_docs(docs_format, config_group, empty_doc=""):
"""
Render the documentation for a config group using the provided
formatter.
"""
processed_data = docs_format.process_config_group(config_group)
if (processed_data['val_table']['exists']
or processed_data['group_table']['exists']
or processed_data['combined_table']['exists']):
return docs_format.template.render(processed_data)
else:
return empty_doc
