def _save_params_to_group(hdf_file: tables.File, group: tables.Group,
params_dict: dict, fitted: bool):
for param_name, param_value in params_dict.items():
if is_estimator(param_value):
param_group = hdf_file.create_group(group, param_name)
_save_estimator_to_group(hdf_file, param_group, param_value,
fitted)
elif is_list_of_named_estimators(param_value):
param_group = hdf_file.create_group(group, param_name)
_save_list_of_named_estimators(hdf_file, param_group, param_value,
fitted)
elif is_list_of_estimators(param_value):
param_group = hdf_file.create_group(group, param_name)
_save_list_of_estimators(hdf_file, param_group, param_value,
fitted)
else:
hdf_file.set_node_attr(group, param_name, param_value)
def _save_list_of_estimators(
hdf_file: tables.File,
group: tables.Group,
estimator_list: List[BaseEstimator],
fitted: bool,
):
hdf_file.set_node_attr(group, "__type__",
GroupType.LIST_OF_ESTIMATORS.name)
hdf_file.set_node_attr(group, "len", len(estimator_list))
for i, estimator in enumerate(estimator_list):
sub_group = hdf_file.create_group(group, f"item_{i}")
_save_estimator_to_group(hdf_file, sub_group, estimator, fitted)
def __init__(self, h5file: tables.File, parent_group: tables.Group,
exp_id: str, exp_title: str, variables: Mapping[str,
VarType]):
super(_ExperimentWriter, self).__init__()
self._id = exp_id
self._title = exp_title
self._file = h5file
try:
self._group = h5file.create_group(parent_group,
exp_id,
title=exp_title)
except tables.NodeError:
try:
node = h5file.get_node(parent_group, exp_id)
path = node._v_pathname
if isinstance(node, tables.Group):
raise ExperimentElementError('Experiment already exists at'
f'{path} in file {h5file}.')
elif isinstance(node, tables.Table):
raise ExperimentElementError('Name conflict: variable '
'table already exists at '
f'{path} in file {h5file}')
else:
raise ExperimentElementError(f'Conflict at {path} '
f'in file {h5file}')
except tables.NoSuchNodeError as e:
raise ExperimentElementError() from e
# metadata
self._group._v_attrs.created = datetime.datetime.now().isoformat()
self._group._v_attrs.finished = 'unfinished'
self._var_tables = dict()
for var_name, var_type in variables.items():
try:
tbl = h5file.create_table(self._group,
var_name,
description={
'record_time':
tables.Time64Col(),
'experiment_time':
tables.Time64Col(),
'value':
_vartype_columns[var_type]()
})
self._var_tables[var_name] = tbl
except KeyError:
raise UnsupportedVariableType(var_type)
self._sub_experiments = dict()
def _save_list_of_named_estimators(
hdf_file: tables.File,
group: tables.Group,
estimator_list: List[Tuple[str, BaseEstimator, Any]],
fitted: bool,
):
hdf_file.set_node_attr(group, "__type__",
GroupType.LIST_OF_NAMED_ESTIMATORS.name)
hdf_file.set_node_attr(group, "names",
[n for (n, e, *r) in estimator_list])
hdf_file.set_node_attr(group, "rests",
[r for (n, e, *r) in estimator_list])
for (name, estimator, *_rest) in estimator_list:
sub_group = hdf_file.create_group(group, name)
_save_estimator_to_group(hdf_file, sub_group, estimator, fitted)
def _save_estimator_to_group(hdf_file: tables.File, group: tables.Group,
estimator: BaseEstimator, fitted: bool):
# save estimator metadata
class_name = estimator.__class__.__module__ + "." + estimator.__class__.__name__
module_version = getattr(__import__(estimator.__class__.__module__),
"__version__")
hdf_file.set_node_attr(group, "__class_name__", class_name)
hdf_file.set_node_attr(group, "__module_version__", module_version)
hdf_file.set_node_attr(group, "__type__", GroupType.ESTIMATOR.name)
# save params
params_dict = get_params_dict(estimator)
# one would expect that those params are not fitted, and fitted can be set to Flase
# but some of them (for example pipeline.Pipeline.steps) do includes fitted estimators.
_save_params_to_group(hdf_file, group, params_dict, fitted=fitted)
if fitted:
# create fit group
fit_group = hdf_file.create_group(group, FIT_GROUP)
hdf_file.set_node_attr(fit_group, "__type__",
GroupType.FITTED_ATTRIBUTES.name)
# save fit params
fit_params_dict = get_fit_params_dict(estimator)
_save_params_to_group(hdf_file, fit_group, fit_params_dict, fitted)
def _add_game_for_agent(
agent_file: tables.File,
agent_name: str,
agent_version: int,
opponent_version: int,
game_result: GameResult,
) -> int:
"""
Create a complete record of a game played by an agent.
Parameters
----------
agent_file : tables.File
agent_name : str
agent_version : int
opponent_version : int
game_result : GameResult
Returns
-------
game_number : int
"""
ver_str = _version_string(agent_version)
gt: tables.Table
vg: tables.Group
try:
vg = agent_file.get_node("/", ver_str)
except tables.NoSuchNodeError:
vg = agent_file.create_group("/", ver_str)
try:
gt = agent_file.get_node(vg, "games")
except tables.NoSuchNodeError:
gt = agent_file.create_table(vg, "games", AgentGameOutcomeRow)
result_1 = game_result.result_1
result_2 = game_result.result_2
game_number = gt.nrows
moved_first = agent_name == result_1.name
if moved_first:
result_agent = result_1
result_opponent = result_2
else:
result_agent = result_2
result_opponent = result_1
opponent_name = result_opponent.name
gt_row = gt.row
gt_row["game_number"] = game_number
gt_row["opponent"] = opponent_name
gt_row["version"] = opponent_version
gt_row["moved_first"] = moved_first
gt_row["outcome"] = result_agent.outcome
gt_row["when/time_str"] = game_result.time_str
gt_row["when/time_sec"] = game_result.time_sec
gt_row.append()
game_number_str = _game_string(game_number)
gg = agent_file.create_group(vg, game_number_str)
for n in ("moves", "move_times"):
_add_array(agent_file, gg, n, getattr(result_1, n),
getattr(result_2, n))
_add_array(agent_file, gg, "state_size", result_agent.state_size)
_add_array(agent_file, gg, "seeds", result_agent.seeds)
for n in ("stdout", "stderr"):
_add_vlarray(agent_file, gg, n, getattr(result_agent, n))
agent_file.flush()
return game_number