def evaluate_reconstruction(
net: Network,
dataset: DataSet,
criterion_all: torch.nn.MSELoss,
) -> Dict[str, float]:
"""
return cost reconstructing the world from world input, e.g. "reconstruction cost".
"""
performance_name = f'cost-reconstruction_avg_{dataset.name}'
res = {performance_name: 0}
for event in dataset.get_events():
x = event.get_x(net.params.x_type)
y = event.get_y(net.params.y_type)
o = net.forward(x)
costs = criterion_all(o, y).detach().cpu().numpy()
res[performance_name] += np.mean(costs)
# divide sum by count
res[performance_name] /= len(dataset)
return res
def evaluate_classification(
net: Network,
dataset: DataSet,
criterion_all: torch.nn.BCEWithLogitsLoss,
) -> Dict[str, float]:
"""
return cost and accuracy for each feature, averaged across all samples in provided dataset.
"""
res = {}
for event in dataset.get_events():
x = event.get_x(net.params.x_type)
y = event.get_y(net.params.y_type)
o = net.forward(x)
costs_by_feature = criterion_all(o, y).detach().cpu().numpy()
o = o.detach().cpu().numpy()
# collect cost for each feature
for n, feature_label in enumerate(
event.feature_vector.get_feature_labels()):
performance_name = f'cost_{feature_label}_{dataset.name}'
res.setdefault(performance_name, 0.0)
res[performance_name] += costs_by_feature[n]
# collect accuracy for each feature_type
for feature_type, o_ids in event.feature_vector.feature_type2ids.items(
):
performance_name = f'acc_{feature_type}_{dataset.name}'
res.setdefault(performance_name, 0.0)
o_restricted = o[o_ids] # logits restricted to one feature_type
y_restricted = event.get_y(
net.params.y_type)[o_ids].detach().cpu().numpy()
if np.argmax(o_restricted) == np.argmax(y_restricted):
res[performance_name] += 1
# divide performance sum by count
num_events = len(dataset)
for performance_name in res:
res[performance_name] /= num_events
# add average cost and accuracy
res[f'cost_avg_{dataset.name}'] = np.mean([
performance for name, performance in res.items()
if name.startswith('cost')
])
res[f'acc_avg_{dataset.name}'] = np.mean([
performance for name, performance in res.items()
if name.startswith('acc')
])
if configs.Evaluation.means_only:
res = {k: v for k, v in res.items() if 'avg' in k}
return res
def main():
project_name = 'PolyominoWorld'
for param_path, label in gen_param_paths(
project_name,
param2requests,
param2default,
runs_path=None,
ludwig_data_path=None,
):
with (param_path / 'param2val.yaml').open('r') as f:
param2val = yaml.load(f, Loader=yaml.FullLoader)
params = Params.from_param2val(param2val)
for rep_id, path_to_net in enumerate(param_path.rglob('model.pt')):
pytorch_model = Network(params)
pytorch_model.load_state_dict(
torch.load(path_to_net, map_location=torch.device('cpu')))
pytorch_model.eval()
dummy_input = torch.zeros(
1, calc_world_vector_size(
params.add_grayscale)) # dummy input of expected shape
onnx_path_out = Path(
__file__
).parent.parent / 'onnx_models' / f'{param2val["param_name"]}_{rep_id}.onnx'
if not onnx_path_out.parent.exists():
onnx_path_out.parent.mkdir()
torch.onnx.export(pytorch_model,
dummy_input,
onnx_path_out,
verbose=True)
tf_rep = prepare(onnx.load(onnx_path_out))
tf_path_out = Path(
__file__
).parent.parent / 'tensorflow_models' / f'{param2val["param_name"]}_{rep_id}.pb'
if not tf_path_out.parent.exists():
tf_path_out.parent.mkdir()
tf_rep.export_graph(str(tf_path_out))
dataset = DataSet(world.generate_sequences(
leftout_colors=params.leftout_colors,
leftout_shapes=params.leftout_shapes,
leftout_variants=params.leftout_variants,
leftout_positions=get_leftout_positions(params.leftout_half),
),
params,
name='re-generated')
# multiple models may exist for the same hyper-parameter configuration
for path_to_net in list(param_path.rglob('model.pt')):
print(f'Loading net from {path_to_net}')
# load net
net = Network(params)
state_dict = torch.load(path_to_net,
map_location=torch.device('cpu'))
net.load_state_dict(state_dict)
net.requires_grad_(False)
net.eval()
header2column = {
'shape': [],
'size': [],
'color': [],
}
vectors = []
for event in dataset.get_events():
x = event.get_x(net.params.x_type)
),
params,
name='re-generated')
if RGB_ID is not None:
name_of_color_channel = {0: 'red', 1: 'green', 2: 'blue'}[RGB_ID]
else:
name_of_color_channel = None
# multiple models may exist for the same hyper-parameter configuration - use first only
for path_to_net in list(param_path.rglob('model.pt'))[:1]:
print(f'Loading net from {path_to_net}')
# load net
net = Network(params)
state_dict = torch.load(path_to_net,
map_location=torch.device('cpu'))
net.load_state_dict(state_dict)
net.eval()
h_x = net.h_xs[HIDDEN_LAYER_ID].weight.detach().numpy(
) # [num hidden, num world cells]
# get set of detectors for one or all color channel
detectors = []
for h_id, hi in enumerate(h_x):
# get weights to one color or all channel
if RGB_ID is not None:
hi_reshaped = hi.reshape(
(3, configs.World.max_x,
configs.World.max_y))[RGB_ID, :, :]
leftout_colors=params.leftout_colors,
leftout_shapes=params.leftout_shapes,
leftout_variants=params.leftout_variants,
leftout_positions=get_leftout_positions(params.leftout_half),
),
params,
name='re-generated')
# multiple models may exist for the same hyper-parameter configuration - iterate over each
paths_to_net = list(sorted(param_path.rglob(f'model_*.pt')))
if not paths_to_net:
raise FileNotFoundError('Did not find any model files')
rep_names = [p.name for p in param_path.glob('*') if p.is_dir()]
net = Network(params)
x_tick2ys = defaultdict(list)
for path_to_net in paths_to_net:
print(f'Loading net from {path_to_net}')
# load weights into net
state_dict = torch.load(path_to_net,
map_location=torch.device('cpu'))
net.load_state_dict(state_dict)
net.requires_grad_(False)
net.eval()
accuracy = evaluate_linear_readout(data, net, FEATURE_TYPE,
HIDDEN_LAYER_ID)
reduction='none') # for evaluation
elif params.criterion == 'bce':
criterion_avg = torch.nn.BCEWithLogitsLoss(
) # sigmoid function and then a binary cross entropy
criterion_all = torch.nn.BCEWithLogitsLoss(reduction='none')
else:
raise AttributeError(f'Invalid arg to criterion')
# multiple models may exist for the same hyper-parameter configuration - iterate over each
for path_to_net in param_path.rglob('model.pt'):
print()
print(f'Evaluating model={path_to_net}')
# load net
net = Network(params)
state_dict = torch.load(path_to_net,
map_location=torch.device('cpu'))
net.load_state_dict(state_dict)
net.eval()
cost_by_shape = np.zeros((len(configs.World.master_shapes), 1))
num_by_shape = np.zeros((len(configs.World.master_shapes), 1))
shape2id = {
s: n
for n, s in enumerate(configs.World.master_shapes)
}
# evaluate
for event in dataset.get_events():
def make_l_and_p(
data: DataSet,
net: Network,
feature_type: str,
h_layer_id: int,
h_ids: Optional[List[int]] = None,
state_is_random: bool = False,
state_is_input: bool = False,
) -> Tuple[np.array, np.array]:
"""
build data structures for evaluating linear readout
"""
lis = []
pis = []
for event in data.get_events():
# make l
if feature_type == 'shape':
li = [
1 if event.shape == s else 0
for s in configs.World.master_shapes
]
elif feature_type == 'color':
li = [
1 if event.color == s else 0
for s in configs.World.master_colors
]
elif feature_type == 'size':
li = [
1 if event.size == s else 0 for s in configs.World.master_sizes
]
else:
raise AttributeError('Invalid feature type')
lis.append(li)
# make p
x = event.world_vector.vector
if state_is_input:
state = x.numpy()
else:
if h_ids is None:
state = [h for h in net.compute_hs(x)][h_layer_id].numpy()
else:
state = x
for h_x in net.h_xs:
term1 = h_x.weight.numpy()[h_ids, :] @ state.numpy()
term2 = h_x.bias.numpy()[h_ids]
if net.params.hidden_activation_function == 'tanh':
state = np.tanh(term1 + term2)
else:
raise NotImplementedError
if state_is_random:
state = np.random.permutation(state)
pi = state.T
pis.append(pi)
l = np.array(lis).T # l has columns of localist target vectors
p = np.array(pis).T # p has columns of representation vectors
return l, p