def test_assign_from_struct():
a = Duck()
a[:] = [2, 3, 4, 5]
assert list(a.items()) == [(0, 2), (1, 3), (2, 4), (3, 5)]
a[:] = dict(a=1, b=2, c=3)
assert list(a.items()) == [('a', 1), ('b', 2), ('c', 3)]
def test_assign_from_struct():
a = Duck()
a[:] = [2, 3, 4, 5]
assert list(a.items()) == [(0, 2), (1, 3), (2, 4), (3, 5)]
a[:] = dict(a=1, b=2, c=3)
assert list(a.items()) == [('a', 1), ('b', 2), ('c', 3)]
def test_so_demo():
# Demo 1: Dynamic assignment
a = Duck()
a['a', 'aa1'] = 1
a['a', 'aa2'] = 2
a['b', 0, 'subfield1'] = 4
a['b', 0, 'subfield2'] = 5
a['b', 1, 'subfield1'] = 6
a['b', 1, 'subfield2'] = 7
assert list(a['b', 1, :]) == [6, 7]
assert a['b', :, 'subfield1'] == [4, 6]
assert a['a', 'aa2'] == 2
assert np.array_equal(a['b'].to_array(), [[4, 5], [6, 7]])
with pytest.raises(
KeyError
): # This should raise an error because key 'a' does not have subkeys 1, 'subfield1'
x = a[:, 1, 'subfield1']
# Demo 2: Sequential and Sliced Assignment
# Here we show another way to create the same structure as above
# 1) You can assign to a slice
# 2) You can use the "next" builtin like an index to append to the structure.
b = Duck()
b['a', :] = {
'aa1': 1,
'aa2': 2
} # Note: when assigning with dict, keys are sorted before insertion (OrderedDict order is kept though).
b['b', next, :] = {'subfield1': 4, 'subfield2': 5}
b['b', next, :] = {'subfield1': 6, 'subfield2': 7}
assert b == a
def demo_stochastic_approximator_tracking(step_sizer_dict, n_steps = 100, tc=5, noise=2, mag=10, seed = None):
t = np.arange(n_steps)
true_signals = {
'approach': mag*(1-np.exp(-t/tc)),
'delayed_sweep': mag/(1+np.exp(-(t-n_steps/2)/tc)),
'step': mag*(t>(n_steps/2)),
'bump': mag*((t>(n_steps/3))&(t<2*n_steps/3)),
}
# x_1_true = mag*(1-np.exp(-t/tc))
# x_2_true = mag/(1+np.exp(-(t-n_steps/2)/tc))
rng = np.random.RandomState(seed)
noise = noise*rng.randn(n_steps)
# x_1_noisy = x_1_true + noise
# x_2_noisy = x_2_true + noise
results = Duck()
fig = plt.figure(figsize=(14, 8))
# for i, (x_sig, x_true) in enumerate(zip([x_1_noisy, x_2_noisy], [x_1_true, x_2_true])):
for i, (name, x_true) in enumerate(true_signals.items()):
x_sig = x_true + noise
ax_sig = subplot_at(i, 0, fig=fig)
ax_err = subplot_at(i, 1, fig=fig)
ax_step = subplot_at(i, 2, fig=fig)
ax_sig.grid()
ax_err.grid()
ax_step.grid()
ax_sig.plot(x_true, color='k', linewidth=2)
ax_sig.plot(x_sig, color='gray', linewidth=2)
for filtername, step_sizer in step_sizer_dict.items():
average_signal, step_sizes = zip(*[(avg, a) for avg in [0] for st in [step_sizer] for x in x_sig for st, a in [st(x)] for avg in [(1-a) * avg + a * x]])
h, = ax_sig.plot(average_signal, label=filtername)
ax_sig.set_ylabel('Signal')
error = np.abs(np.array(average_signal) - x_true)
results[i, filtername, 'rmse'] = np.sqrt(np.mean(error**2))
ax_err.plot(error, color=h.get_color())
ax_err.set_ylabel('Error')
ax_step.plot(step_sizes, color=h.get_color(), label=filtername)
ax_step.set_ylabel('Step Size')
ax_step.legend()
print(results.description())
plt.show()
def test_arrayify_axis_demo():
a = Duck()
a[0, 'x'] = 1
a[0, 'y'] = 2
a[1, 'x'] = 3
a[1, 'y'] = 4
b = a.arrayify_axis(axis=0)
assert np.array_equal(b['x'], [1, 3])
assert np.array_equal(b['y'], [2, 4])
def test_arrayify_axis_demo():
a = Duck()
a[0, 'x'] = 1
a[0, 'y'] = 2
a[1, 'x'] = 3
a[1, 'y'] = 4
b = a.arrayify_axis(axis=0)
assert np.array_equal(b['x'], [1, 3])
assert np.array_equal(b['y'], [2, 4])
def test_open_next():
a = Duck()
a['a'] = Duck()
aa = a['a'].open(next)
aa['c'] = 4
aa['4'] = 5
aa = a['a'].open(next)
aa['c'] = 6
aa['4'] = 7
def test_arrayify_empty_stuct():
a = Duck.from_struct(OrderedDict([('a', [])]))
b = a.arrayify_axis(axis=1, subkeys='a')
assert isinstance(b['a'], np.ndarray)
assert np.array_equal(b['a'], [])
a = Duck.from_struct(OrderedDict([('a', [1, 2])]))
b = a.arrayify_axis(axis=1, subkeys='a')
assert isinstance(b['a'], np.ndarray)
assert np.array_equal(b['a'], [1, 2])
def test_duck_array_build():
a = Duck()
count = 0
for i in range(3):
for j in range(4):
a[i, j] = count
count += 1
desired_array = np.arange(12).reshape(3, 4)
assert a == desired_array
assert np.array_equal(a.to_array(), desired_array)
def test_open_key():
a = Duck()
a['a', :] = [3, 4]
with pytest.raises(KeyError):
a['b']
c = a.open('b')
c[next] = 5
c[next] = 6
assert a['b', 0]==5
assert a['b', 1]==6
def test_arrayify_empty_stuct():
a = Duck.from_struct(OrderedDict([('a', [])]))
b = a.arrayify_axis(axis=1, subkeys='a')
assert isinstance(b['a'], np.ndarray)
assert np.array_equal(b['a'], [])
a = Duck.from_struct(OrderedDict([('a', [1, 2])]))
b = a.arrayify_axis(axis=1, subkeys='a')
assert isinstance(b['a'], np.ndarray)
assert np.array_equal(b['a'], [1, 2])
def test_open_key():
a = Duck()
a['a', :] = [3, 4]
with pytest.raises(KeyError):
a['b']
c = a.open('b')
c[next] = 5
c[next] = 6
assert a['b', 0] == 5
assert a['b', 1] == 6
def test_duck_array_build():
a = Duck()
count = 0
for i in range(3):
for j in range(4):
a[i, j] = count
count+=1
desired_array = np.arange(12).reshape(3, 4)
assert a==desired_array
assert np.array_equal(a.to_array(), desired_array)
def test_assign_tuple_keys():
a = Duck()
a['training'] = Duck()
a['testing', next]=4
b = a.arrayify_axis(axis=1, subkeys='testing')
b = b.arrayify_axis(axis=1, subkeys='training', inplace=True).to_struct()
a['training', next, ...] = OrderedDict([('ops', OrderedDict([(('a', 'b'), 1), (('a', 'c'), 2)]))])
a['training', next, ...] = OrderedDict([('ops', OrderedDict([(('a', 'b'), 3), (('a', 'c'), 4)]))])
a.arrayify_axis(axis=1, subkeys='training').to_struct()
a['training', next, ...] = OrderedDict([('ops', OrderedDict([(('a', 'b'), 1), (('a', 'c'), 2)]))])
pass
def test_slice_assignment():
deep_struct = {'a': [1,{'c': 2, 'd': 3}], 'b': [1, 2]}
a = Duck()
a[:] = deep_struct
assert list(a.items(depth='full')) == [
(('a', ), [1,{'c': 2, 'd': 3}]),
(('b', ), [1, 2])
]
a = Duck()
a[:, :] = deep_struct
assert list(a.items(depth='full')) == [
(('a', 0), 1),
(('a', 1), {'c': 2, 'd': 3}),
(('b', 0), 1),
(('b', 1), 2)
]
a = Duck()
with raises(InitializerTooShallowError):
a[:, :, :] = deep_struct
a = Duck()
a[...] = deep_struct
assert list(a.items(depth='full')) == [
(('a', 0), 1),
(('a', 1, 'c'), 2),
(('a', 1, 'd'), 3),
(('b', 0), 1),
(('b', 1), 2)
]
a = Duck()
with raises(InvalidKeyError): # It would be nice to add this functionality, but maybe later.
a[..., next] = deep_struct
def test_reasonable_errors_on_wrong_keys():
a = Duck()
a['a'] = 1
with pytest.raises(InvalidKeyError):
a[1] = 'a'
with pytest.raises(InvalidKeyError):
a[next] = 'a'
a = Duck()
a[next] = 'a'
with pytest.raises(InvalidKeyError):
a['a'] = 1
with pytest.raises(InvalidKeyError):
a[10] = 'a' # Too high!
def test_key_get_on_set_bug():
a = Duck()
a['x', next] = {'a': 1, 'b': 2}
a['x', next] = {'a': 3, 'b': 4}
a['x', next, 'f'] = 4 # This once caused an error
a['x', next, 'g'] = 5
def test_break_in():
a = Duck()
a['x', next] = {'a': 1, 'b': 2}
a['x', next] = {'a': 3, 'b': 4}
with pytest.raises(KeyError) as err:
a['x', :, 'a']
b = a.break_in()
assert b['x', :, 'a'] == [1, 3]
with pytest.raises(KeyError) as err:
a['x', :, 'a']
a.break_in(inplace=True)
assert a['x', :, 'a'] == [1, 3]
def test_break_in():
a = Duck()
a['x', next] = {'a': 1, 'b': 2}
a['x', next] = {'a': 3, 'b': 4}
with pytest.raises(KeyError) as err:
a['x', :, 'a']
b = a.break_in()
assert b['x', :, 'a'] == [1, 3]
with pytest.raises(KeyError) as err:
a['x', :, 'a']
a.break_in(inplace=True)
assert a['x', :, 'a'] == [1, 3]
def _get_standard_test_duck():
a = Duck()
a['a', 'aa1'] = 1
a['a', 'aa2'] = 2
a['b', 0, 'subfield1'] = 4
a['b', 0, 'subfield2'] = 5
a['b', 1, 'subfield1'] = 6
a['b', 1, 'subfield2'] = 7
return a
def test_next_elipsis_assignment():
a = Duck()
a['a', next, ...] = OrderedDict([('b', [1, 2])])
a['a', next, ...] = OrderedDict([('b', [3, 4])])
# Should lead to form:
# 'a'
# 0
# 'b'
# 0: 1
# 1: 2
# 1
# 'b'
# 0: 3
# 1: 4
# If we arrayify axis, 1, we expect struct:
# Should lead to form:
# 'a'
# 'b'
# 0: [1, 3]
# 1: [2, 4]
assert np.array_equal(a.arrayify_axis(axis=1, subkeys='a')['a', 'b'], [[1, 3], [2, 4]])
a = Duck()
a['a', next, ...] = OrderedDict([('b', np.array([1, 2]))])
a['a', next, ...] = OrderedDict([('b', np.array([3, 4]))])
# Now, a is no longer broken into ... So we have to be careful with these elipses!
# 'a'
# 0
# 'b': [1, 2]
# 1
# 'b': [3, 4]
# If we arrayify axis, 1, we expect struct:
# Should lead to form:
# 'a'
# 'b'
# 0: [1, 2]
# 1: [3, 4]
assert np.array_equal(a.arrayify_axis(axis=1, subkeys='a')['a', 'b'], [[1, 2], [3, 4]])
def test_string_slices():
a = Duck()
for i, char in enumerate(
'abcdefghijklmnopqrstuvwxyz'[::-1]): # Note the reverse
a[char] = 26 - i
assert list(a['q':'m'].items()) == [('q', 17), ('p', 16), ('o', 15),
('n', 14)]
assert list(a['m':'q'].items()) == []
assert list(a['m':'q':-1].items()) == [('m', 13), ('n', 14), ('o', 15),
('p', 16)]
def test_occasional_value_filter():
a = Duck()
for i in range(20):
if i % 3 == 0:
a[next, ...] = {'a': i, 'b': i}
else:
a[next, ...] = {'a': i}
assert a[:, 'a'] == list(range(20))
with pytest.raises(KeyError):
print(a[:, 'b'])
assert a.filter[:, 'b'] == list(range(0, 20, 3))
# Test doc
a = Duck()
a[next, :] = {'a': 1, 'b': 2}
a[next, :] = {'a': 3}
a[next, :] = {'a': 4, 'b': 5}
with pytest.raises(KeyError):
assert a[:, 'b'] == [2, 5]
assert a.filter[:, 'b'] == [2, 5]
def test_boolean_indexing():
d = Duck()
d[next, :] = {'a': 1, 'b': 2, 'c': 7}
d[next, :] = {'a': 4, 'b': 3, 'c': 8}
d[next, :] = {'a': 3, 'b': 6, 'c': 9}
d[next, :] = {'a': 6, 'b': 2, 'c': 0}
assert d[[True, False, False, True], 'a'] == [1, 6]
assert d[d[:, 'b'].each_eq(2), 'a'] == [1, 6]
assert d[d[:, 'b'].each_in({3, 6}), 'a'] == [4, 3]
assert d[d[:, 'b'].each_eq(3) | d[:, 'b'].each_eq(6), 'a'] == [4, 3]
assert d[d[:, 'b'].each_in({3, 6}) & ~d[:, 'a'].each_in({3, 6})].only(
)['c'] == 8 # "Find the 'c' value of the only item in the duck where b is in {3, 6} and 'a' is not in {3, 6}
def test_split_get_assign():
a = Duck()
c = 0
for i in ['a', 'b']:
for j in [0, 1]:
for k in ['c', 'd']:
a[i, j, k] = c
c += 1
b = a[:, 1, :]
print(a[:, 1, :].description())
assert list(b.items(depth='full')) == [(('a', 'c'), 2), (('a', 'd'), 3),
(('b', 'c'), 6), (('b', 'd'), 7)]
with pytest.raises(InvalidKeyError):
a[:, 1, :] = b
def test_dict_assignment():
a = Duck(
) # When assigning with a dict we first sort keys. Here we just verify that keys remain sorted
a[next, :] = {
letter: number
for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz',
range(1, 27))
}
a[next, :] = {
letter: number
for letter, number in izip_equal('abcdefghijklmnopqrstuvwxyz',
range(27, 27 + 26))
}
assert list(a[0].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[0].values()) == list(range(1, 27))
assert list(a[1].keys()) == [char for char in 'abcdefghijklmnopqrstuvwxyz']
assert list(a[1].values()) == list(range(27, 27 + 26))
def do_test(test_subset_generators,
f_predict,
loss_dict,
n_test_iters,
collapse_loss='mean',
in_test_callback=None):
if callable(loss_dict):
loss_dict = dict(loss=loss_dict)
these_test_results = Duck()
losses = {}
pi = ProgressIndicator(n_test_iters, "Testing")
for subset_name, subset_generator in test_subset_generators.items():
start_time = time.time()
losses[subset_name] = []
n_tests = 0
for inputs, targets in subset_generator:
n_tests += 1
outputs = f_predict(inputs)
for loss_name, f_loss in loss_dict.items():
these_test_results[subset_name, Keys.LOSSES, loss_name,
next] = f_loss(outputs, targets)
pi.print_update()
if in_test_callback is not None:
in_test_callback(inputs=inputs,
targets=targets,
outputs=outputs)
assert n_tests > 0, "It appears that subset '{}' had no tests!".format(
subset_name)
these_test_results[subset_name, Keys.N_TESTS] = n_tests
if collapse_loss is not None:
collapse_func = {'mean': np.mean}[collapse_loss]
for loss_name, f_loss in loss_dict.items():
these_test_results[subset_name, Keys.LOSSES,
loss_name] = collapse_func(
these_test_results[subset_name,
Keys.LOSSES,
loss_name])
these_test_results[subset_name, Keys.TIME] = time.time() - start_time
return these_test_results
def test_next_elipsis_assignment():
a = Duck()
a['a', next, ...] = OrderedDict([('b', [1, 2])])
a['a', next, ...] = OrderedDict([('b', [3, 4])])
# Should lead to form:
# 'a'
# 0
# 'b'
# 0: 1
# 1: 2
# 1
# 'b'
# 0: 3
# 1: 4
# If we arrayify axis, 1, we expect struct:
# Should lead to form:
# 'a'
# 'b'
# 0: [1, 3]
# 1: [2, 4]
assert np.array_equal(
a.arrayify_axis(axis=1, subkeys='a')['a', 'b'], [[1, 3], [2, 4]])
a = Duck()
a['a', next, ...] = OrderedDict([('b', np.array([1, 2]))])
a['a', next, ...] = OrderedDict([('b', np.array([3, 4]))])
# Now, a is no longer broken into ... So we have to be careful with these elipses!
# 'a'
# 0
# 'b': [1, 2]
# 1
# 'b': [3, 4]
# If we arrayify axis, 1, we expect struct:
# Should lead to form:
# 'a'
# 'b'
# 0: [1, 2]
# 1: [3, 4]
assert np.array_equal(
a.arrayify_axis(axis=1, subkeys='a')['a', 'b'], [[1, 2], [3, 4]])
def test_reasonable_error_messages():
a = Duck()
a['x', next] = {'a': 1, 'b': 2}
a['x', next] = {'a': 3, 'b': 4}
with pytest.raises(KeyError) as err:
a['x', 0, 'a', 'b', 'c']
assert get_message(
err
) == "This Duck has no key: ['x',0,'a','b','c']. The deepest valid key was ['x',0]. If you want to break into leaves, use Duck.break_in()."
with pytest.raises(KeyError) as err:
a['x', 0, 'a', 'b', 'c', 'd']
assert get_message(
err
) == "This Duck has no key: ['x',0,'a','b','c','d']. The deepest valid key was ['x',0]. If you want to break into leaves, use Duck.break_in()."
with pytest.raises(KeyError) as err:
a['x', :, 'a']
assert get_message(
err
) == "This Duck has no key: ['x',:,'a']. The deepest valid key was ['x',:]. If you want to break into leaves, use Duck.break_in()."
with pytest.raises(KeyError) as err:
a['b', 0, 'a']
assert get_message(
err
) == "This Duck has no key: 'b', so it cannot read sub-key: ['b',0,'a']."
with pytest.raises(IndexError) as err:
a['x', 2, 'a']
assert get_message(
err
) == 'Your index "2" exceeds the range of this DynamicSequence of length 2'
a['x', next, 'f'] = 4
def test_assign_tuple_keys():
a = Duck()
a['training'] = Duck()
a['testing', next] = 4
b = a.arrayify_axis(axis=1, subkeys='testing')
b = b.arrayify_axis(axis=1, subkeys='training', inplace=True).to_struct()
a['training', next, ...] = OrderedDict([('ops',
OrderedDict([(('a', 'b'), 1),
(('a', 'c'), 2)]))])
a['training', next, ...] = OrderedDict([('ops',
OrderedDict([(('a', 'b'), 3),
(('a', 'c'), 4)]))])
a.arrayify_axis(axis=1, subkeys='training').to_struct()
a['training', next, ...] = OrderedDict([('ops',
OrderedDict([(('a', 'b'), 1),
(('a', 'c'), 2)]))])
pass
def demo_energy_based_initialize_eq_prop_fwd_energy(
n_epochs=25,
hidden_sizes=(500, ),
minibatch_size=20,
beta=0.5,
epsilon=0.5,
learning_rate=(0.1, .05),
n_negative_steps=20,
n_positive_steps=4,
initial_weight_scale=1.,
forward_deviation_cost=0.,
zero_deviation_cost=1,
epoch_checkpoint_period={
0: .25,
1: .5,
5: 1,
10: 2,
50: 4
},
n_test_samples=10000,
skip_zero_epoch_test=False,
train_with_forward='contrast',
forward_nonlinearity='rho(x)',
local_loss=True,
random_flip_beta=True,
seed=1234,
):
print('Params:\n' +
'\n'.join(list(f' {k} = {v}' for k, v in locals().items())))
assert train_with_forward in ('contrast', 'contrast+', 'energy', False)
rng = get_rng(seed)
n_in = 784
n_out = 10
dataset = get_mnist_dataset(flat=True, n_test_samples=None).to_onehot()
x_train, y_train = dataset.training_set.xy
x_test, y_test = dataset.test_set.xy # Their 'validation set' is our 'test set'
if is_test_mode():
x_train, y_train, x_test, y_test = x_train[:
100], y_train[:
100], x_test[:
100], y_test[:
100]
n_epochs = 1
layer_sizes = [n_in] + list(hidden_sizes) + [n_out]
eq_params = initialize_params(layer_sizes=layer_sizes,
initial_weight_scale=initial_weight_scale,
rng=rng)
forward_params = initialize_params(
layer_sizes=layer_sizes,
initial_weight_scale=initial_weight_scale,
rng=rng)
y_train = y_train.astype(np.float32)
sp = Speedometer(mode='last')
is_epoch_checkpoint = Checkpoints(epoch_checkpoint_period,
skip_first=skip_zero_epoch_test)
f_negative_eq_step = equilibriating_step.partial(
forward_deviation_cost=forward_deviation_cost,
zero_deviation_cost=zero_deviation_cost).compile()
f_inference_eq_step = equilibriating_step.partial(
forward_deviation_cost=forward_deviation_cost,
zero_deviation_cost=zero_deviation_cost).compile()
f_positive_eq_step = equilibriating_step.partial(
forward_deviation_cost=forward_deviation_cost,
zero_deviation_cost=zero_deviation_cost).compile()
f_parameter_update = update_eq_params.partial(
forward_deviation_cost=forward_deviation_cost,
zero_deviation_cost=zero_deviation_cost).compile()
f_forward_pass = forward_pass.partial(
nonlinearity=forward_nonlinearity).compile()
# f_forward_parameter_update = update_forward_params_with_energy.compile()
f_forward_parameter_contrast_update = update_forward_params_with_contrast.partial(
nonlinearity=forward_nonlinearity).compile()
def do_inference(forward_params_, eq_params_, x, n_steps):
states_ = forward_states_ = f_forward_pass(
x=x, params=forward_params_
) if train_with_forward else initialize_states(
n_samples=x.shape[0], noninput_layer_sizes=layer_sizes[1:])
for _ in range(n_steps):
states_ = f_inference_eq_step(params=eq_params_,
states=states_,
fwd_states=forward_states_,
x=x,
epsilon=epsilon)
return forward_states_[-1], states_[-1]
results = Duck()
# last_time, last_epoch = time(), -1
for i, (ixs, info) in enumerate(
minibatch_index_info_generator(n_samples=x_train.shape[0],
minibatch_size=minibatch_size,
n_epochs=n_epochs)):
epoch = i * minibatch_size / x_train.shape[0]
# print(f'Training Rate: {(time()-last_time)/(epoch-last_epoch):3g}s/ep')
# last_time, last_epoch = time(), epoch
if is_epoch_checkpoint(epoch):
n_samples = n_test_samples if n_test_samples is not None else len(
x_test)
(test_init_error,
test_neg_error), (train_init_error, train_neg_error) = [[
percent_argmax_incorrect(prediction, y[:n_test_samples])
for prediction in do_inference(forward_params_=forward_params,
eq_params_=eq_params,
x=x[:n_test_samples],
n_steps=n_negative_steps)
] for x, y in [(x_test, y_test), (x_train, y_train)]]
print(
f'Epoch: {epoch:.3g}, Iter: {i}, Test Init Error: {test_init_error:.3g}%, Test Neg Error: {test_neg_error:.3g}%, Train Init Error: {train_init_error:.3g}%, Train Neg Error: {train_neg_error:.3g}%, , Mean Rate: {sp(i):.3g}iter/s'
)
results[next, :] = dict(iter=i,
epoch=epoch,
test_init_error=test_init_error,
test_neg_error=test_neg_error,
train_init_error=train_init_error,
train_neg_error=train_neg_error)
yield results
if epoch > 2 and train_neg_error > 50:
return
# The Original training loop, just taken out here:
x_data_sample, y_data_sample = x_train[ixs], y_train[ixs]
states = forward_states = f_forward_pass(
x=x_data_sample, params=forward_params
) if train_with_forward else initialize_states(
n_samples=minibatch_size, noninput_layer_sizes=layer_sizes[1:])
for t in range(n_negative_steps):
states = f_negative_eq_step(params=eq_params,
states=states,
x=x_data_sample,
epsilon=epsilon,
fwd_states=forward_states)
negative_states = states
this_beta = rng.choice([-beta, beta]) if random_flip_beta else beta
for t in range(n_positive_steps):
states = f_positive_eq_step(params=eq_params,
states=states,
x=x_data_sample,
y=y_data_sample,
y_pressure=this_beta,
epsilon=epsilon,
fwd_states=forward_states)
positive_states = states
eq_params = f_parameter_update(x=x_data_sample,
params=eq_params,
negative_states=negative_states,
positive_states=positive_states,
fwd_states=forward_states,
learning_rates=learning_rate,
beta=this_beta)
if train_with_forward == 'contrast':
forward_params = f_forward_parameter_contrast_update(
x=x_data_sample,
forward_params=forward_params,
eq_states=negative_states,
learning_rates=learning_rate)
# forward_params = f_forward_parameter_contrast_update(x=x_data_sample, forward_params=forward_params, eq_states=negative_states, learning_rates=[lr/10 for lr in learning_rate])
elif train_with_forward == 'contrast+':
forward_params = f_forward_parameter_contrast_update(
x=x_data_sample,
forward_params=forward_params,
eq_states=positive_states,
learning_rates=learning_rate)
# elif train_with_forward == 'energy':
# forward_params = f_forward_parameter_update(x=x_data_sample, forward_params = forward_params, eq_params=eq_params, learning_rates=learning_rate)
else:
assert train_with_forward is False
def train_and_test_predictor(
f_train,
f_predict,
losses,
training_data_gen,
test_data_gen_constructors,
n_training_iters = None,
n_test_iters = None,
test_checkpoints = ('lin', 1000),
collapse_loss = 'mean',
progress_update_period = '5s',
in_test_callback = None,
post_test_callback = None,
post_train_callback = None,
save_train_return = False,
measures = None,
iterations_to_end = False
):
"""
:param f_train:
:param f_predict:
:param losses:
:param training_data_gen:
:param test_data_gen_constructors:
:param samples_count:
:param n_training_iters:
:param n_test_iters:
:param test_checkpoints:
:param collapse_loss:
:param progress_update_period:
:param in_test_callback:
:param post_test_callback:
:param post_train_callback:
:param save_train_return:
:param measures:
:return: If yield_array_data is False, an ArrayStruct with fields:
'training' Results recorded during training callbacks
training_iter The iteration of the training callback
'testing' Results recorded during tests
test_iter The index of the test
'iter' The number of training iterations finished at the time that the test is run
'time' The time at which the test is run
'samples' The number of samples seen so far
'results' A structure containing results
subset_name The name of the testing subset e.g. 'train', 'test' (you give subset names in test_data_gen_constructors)
'losses'
loss_name The name of the loss function (you provide this in losses)
n_tests The numner of tests that were run for this subset
'time' The time, in seconds, that it took to test on this subset.
Otherwise, if true, a structure the same object but training_iter and test_iter pushed to the leaf-position
"""
if measures is None:
measures = Duck()
if 'training' not in measures:
measures[Keys.TRAINING] = Duck()
if 'testing' not in measures:
measures[Keys.TESTING] = Duck()
is_test_time = Checkpoints(test_checkpoints) if not isinstance(test_checkpoints, Checkpoints) else test_checkpoints
pi = ProgressIndicator(n_training_iters, "Training", update_every=progress_update_period)
for inputs, targets in training_data_gen:
if is_test_time():
this_test_measures = measures[Keys.TESTING].open(next)
this_test_measures[Keys.ITER] = pi.get_iterations()
this_test_measures[Keys.TIME] = pi.get_elapsed()
this_test_measures[Keys.RESULTS] = do_test(
test_subset_generators={subset_name: constructor() for subset_name, constructor in test_data_gen_constructors.items()},
f_predict=f_predict,
loss_dict=losses,
n_test_iters=n_test_iters,
collapse_loss = collapse_loss,
in_test_callback = in_test_callback,
)
if post_test_callback is not None:
return_val = post_test_callback(this_test_measures)
if return_val is not None:
this_test_measures[Keys.CALLBACK, ...] = return_val
if iterations_to_end:
measures_to_yield = measures.arrayify_axis(axis=1, subkeys=Keys.TRAINING)
measures_to_yield = measures_to_yield.arrayify_axis(axis=1, subkeys=Keys.TESTING, inplace=True)
yield measures_to_yield.to_struct()
else:
yield measures
train_return = f_train(inputs, targets)
pi.print_update()
if save_train_return:
measures[Keys.TRAINING, Keys.RETURNS] = train_return
if post_train_callback:
return_val = post_train_callback(inputs=inputs, targets=targets, iter=pi.get_iterations())
if return_val is not None:
measures[Keys.TRAINING, Keys.CALLBACK, next, ...] = return_val
def test_dictarraylist():
# We build the same thing in four different ways
a = Duck()
a['a', 'aa1'] = 1
a['a', 'aa2'] = 2
a['b', 0, 'subfield1'] = 4
a['b', 0, 'subfield2'] = 5
a['b', 1, 'subfield1'] = 6
a['b', 1, 'subfield2'] = 7
a['b', 2, 'subfield2'] = 9 # Note: Out of order assignment here!
a['b', 2, 'subfield1'] = 8
b = Duck.from_struct(a.to_struct())
assert a.to_struct() == b.to_struct()
c = Duck()
c['a', :] = OrderedDict([('aa1', 1), ('aa2', 2)])
c['b', next, :] = OrderedDict([('subfield1', 4), ('subfield2', 5)])
c['b', next, :] = OrderedDict([('subfield1', 6), ('subfield2', 7)])
c['b', next, :] = OrderedDict([('subfield2', 9), ('subfield1', 8)])
d = Duck()
d['a', :] = OrderedDict([('aa1', 1), ('aa2', 2)])
d['b', :, :] = [OrderedDict([('subfield1', 4), ('subfield2', 5)]), OrderedDict([('subfield1', 6), ('subfield2', 7)]), OrderedDict([('subfield2', 9), ('subfield1', 8)])]
e = Duck()
e['a', ...] = OrderedDict([('aa1', 1), ('aa2', 2)])
e['b', ...] = [OrderedDict([('subfield1', 4), ('subfield2', 5)]), OrderedDict([('subfield1', 6), ('subfield2', 7)]), OrderedDict([('subfield2', 9), ('subfield1', 8)])]
f = a[:]
g = a[:, :]
for i, same_struct in enumerate([a, b, c, d, e, f, g]):
print ('Test {}'.format(i))
assert a.to_struct()==same_struct.to_struct()
assert a==same_struct
assert compute_fixed_hash(a.to_struct()) == compute_fixed_hash(b.to_struct())
assert same_struct['a', 'aa2'] == 2
assert list(same_struct['b', 1, :].values()) == [6, 7]
assert list(same_struct['b', :, 'subfield1'].values()) == [4, 6, 8]
assert same_struct['b', :, :].deepvalues() == [[4, 5], [6, 7], [9, 8]] # Note that the order-switching gets through.
assert same_struct['b', :, ['subfield1', 'subfield2']].deepvalues() == [[4, 5], [6, 7], [8, 9]]
assert same_struct['b', :, :].to_struct() == [OrderedDict([('subfield1', 4), ('subfield2', 5)]), OrderedDict([('subfield1', 6), ('subfield2', 7)]), OrderedDict([('subfield2', 9), ('subfield1', 8)])]
assert same_struct['b', 1:, :].to_struct() == [OrderedDict([('subfield1', 6), ('subfield2', 7)]), OrderedDict([('subfield2', 9), ('subfield1', 8)])]
with pytest.raises(KeyError):
x = same_struct['a', 'aa3']
with pytest.raises(KeyError):
x = same_struct['a', 1, 'subfield1']
with pytest.raises(IndexError):
x = same_struct['b', 4, 'subfield1']
with pytest.raises(KeyError): # This should raise an error because key 'a' does not have subkeys 1, 'subfield1'
x = same_struct[:, 1, 'subfield1']
assert np.array_equal(same_struct['b'].to_array(), [[4, 5], [6, 7], [8, 9]]) # Note... the order is corrected.
with pytest.raises(InvalidKeyError):
assert np.array_equal(same_struct.to_array()) # Note... the order is corrected.
new_struct = same_struct.arrayify_axis(axis=1, subkeys=('b',))
assert np.array_equal(new_struct['b', 'subfield1'], [4, 6, 8])
assert np.array_equal(new_struct['b', 'subfield2'], [5, 7, 9])
def experiment_mnist_eqprop_torch(
layer_constructor: Callable[[int, LayerParams], IDynamicLayer],
n_epochs=10,
hidden_sizes=(500, ),
minibatch_size=10, # update mini-batch size
batch_size=500, # total batch size
beta=.5,
random_flip_beta=True,
learning_rate=.05,
n_negative_steps=120,
n_positive_steps=80,
initial_weight_scale=1.,
online_checkpoints_period=None,
epoch_checkpoint_period=1.0, #'100s', #{0: .25, 1: .5, 5: 1, 10: 2, 50: 4},
skip_zero_epoch_test=False,
n_test_samples=10000,
prop_direction: Union[str, Tuple] = 'neutral',
bidirectional=True,
renew_activations=True,
do_fast_forward_pass=False,
rebuild_coders=True,
l2_loss=None,
splitstream=False,
seed=1234,
prediction_inp_size=17, ## prediction input size
delay=18, ## delay size for the clamped phase
pred=True, ## if you want to use the prediction
check_flg=False,
):
"""
Replicate the results of Scellier & Bengio:
Equilibrium Propagation: Bridging the Gap between Energy-Based Models and Backpropagation
https://www.frontiersin.org/articles/10.3389/fncom.2017.00024/full
Specifically, the train_model demo here:
https://github.com/bscellier/Towards-a-Biologically-Plausible-Backprop
Differences between our code and theirs:
- We do not keep persistent layer activations tied to data points over epochs. So our results should only really match for the first epoch.
- We evaluate training score periodically, rather than online average (however you can see online score by setting online_checkpoints_period to something that is not None)
"""
torch.manual_seed(seed)
device = 'cuda' if torch.cuda.is_available(
) and USE_CUDA_WHEN_AVAILABLE else 'cpu'
if device == 'cuda':
torch.set_default_tensor_type(torch.cuda.FloatTensor)
print(f'Using Device: {device}')
print('Params:\n' +
'\n'.join(list(f' {k} = {v}' for k, v in locals().items())))
rng = get_rng(seed)
n_in = 784
n_out = 10
dataset = input_data.read_data_sets('MNIST_data', one_hot=True)
x_train, y_train = torch.tensor(
dataset.train.images, dtype=torch.float32
).to(device), torch.tensor(dataset.train.labels, dtype=torch.float32).to(
device
) #(torch.tensor(a.astype(np.float32)).to(device) for a in dataset.mnist.train.images.xy)
x_test, y_test = torch.tensor(
dataset.test.images, dtype=torch.float32).to(device), torch.tensor(
dataset.test.labels, dtype=torch.float32).to(
device) # Their 'validation set' is our 'test set'
x_val, y_val = torch.tensor(
dataset.validation.images,
dtype=torch.float32).to(device), torch.tensor(
dataset.validation.labels, dtype=torch.float32).to(
device) # Their 'validation set' is our 'test set'
if is_test_mode():
x_train, y_train, x_test, y_test, x_val, y_val = x_train[:
100], y_train[:
100], x_test[:
100], y_test[:
100], x_val[:
100], y_val[:
100]
n_epochs = 1
n_negative_steps = 3
n_positive_steps = 3
layer_sizes = [n_in] + list(hidden_sizes) + [n_out]
ra = RunningAverage()
sp = Speedometer(mode='last')
is_online_checkpoint = Checkpoints(
online_checkpoints_period, skip_first=skip_zero_epoch_test
) if online_checkpoints_period is not None else lambda: False
is_epoch_checkpoint = Checkpoints(epoch_checkpoint_period,
skip_first=skip_zero_epoch_test)
training_states = initialize_states(
layer_constructor=layer_constructor,
#n_samples=minibatch_size,
n_samples=batch_size,
params=initialize_params(layer_sizes=layer_sizes,
initial_weight_scale=initial_weight_scale,
rng=rng))
# dbplot(training_states[0].params.w_fore[:10, :10], str(rng.randint(265)))
if isinstance(prop_direction, str):
fwd_prop_direction, backward_prop_direction = prop_direction, prop_direction
else:
fwd_prop_direction, backward_prop_direction = prop_direction
def do_test():
# n_samples = n_test_samples if n_test_samples is not None else len(x_test)
test_error, train_error, val_error = [
percent_argmax_incorrect(
run_inference(
x_data=x[:n_test_samples],
states=initialize_states(
layer_constructor=layer_constructor,
params=[s.params for s in training_states],
n_samples=n_samples),
n_steps=n_negative_steps,
prop_direction=fwd_prop_direction,
), y[:n_samples]).item()
for x, y in [(x_test, y_test), (x_train, y_train), (x_val, y_val)]
for n_samples in [
min(len(x), n_test_samples
) if n_test_samples is not None else len(x)
]
] # Not an actal loop... just hack for assignment in comprehensions
print(
f'Epoch: {epoch:.3g}, Iter: {i}, Test Error: {test_error:.3g}%, Train Error: {train_error:.3g}, Validation Error: {val_error:.3g}, Mean Rate: {sp(i):.3g}iter/s'
)
return dict(iter=i,
epoch=epoch,
train_error=train_error,
test_error=test_error,
val_error=val_error), train_error, test_error, val_error
results = Duck()
pi = ProgressIndicator(expected_iterations=n_epochs *
dataset.train.num_examples / minibatch_size,
update_every='10s')
dy_squared = []
dy_squared.append(None)
dy_squared.append(None)
for i, (ixs, info) in enumerate(
minibatch_index_info_generator(n_samples=x_train.size()[0],
minibatch_size=batch_size,
n_epochs=n_epochs)):
epoch = i * batch_size / x_train.shape[0]
if is_epoch_checkpoint(epoch):
check_flg = False
x_train, y_train = shuffle_data(x_train, y_train)
with pi.pause_measurement():
results[next, :], train_err, test_err, val_err = do_test()
## prepare for saving the parameters
ws, bs = zip(*((s.params.w_aft, s.params.b)
for s in training_states[1:]))
f = None
if os.path.isfile(directory + '/log.txt'):
f = open(directory + '/log.txt', 'a')
else:
os.mkdir(directory)
f = open(directory + '/log.txt', 'w')
f.write("Epoch: " + str(epoch) + '\n')
f.write("accuracy for training: " + str(train_err) + '\n')
f.write("accuracy for testing: " + str(test_err) + '\n')
f.write("accuracy for validation: " + str(val_err) + '\n')
f.close()
np.save(directory + '/w_epoch_' + str(epoch) + '.npy', ws)
np.save(directory + '/b_epoch_' + str(epoch) + '.npy', bs)
np.save(directory + '/dy_squared_epoch_' + str(epoch) + '.npy',
dy_squared)
yield results
if epoch > 100 and results[-1, 'train_error'] > 50:
return
# The Original training loop, just taken out here:
ixs = ixs.astype(np.int32) # this is for python version 3.7
x_data_sample, y_data_sample = x_train[ixs], y_train[ixs]
training_states, dy_squared = run_eqprop_training_update(
x_data=x_data_sample,
y_data=y_data_sample,
layer_states=training_states,
beta=beta,
random_flip_beta=random_flip_beta,
learning_rate=learning_rate,
layer_constructor=layer_constructor,
bidirectional=bidirectional,
l2_loss=l2_loss,
renew_activations=renew_activations,
n_negative_steps=n_negative_steps,
n_positive_steps=n_positive_steps,
prop_direction=prop_direction,
splitstream=splitstream,
rng=rng,
prediction_inp_size=prediction_inp_size,
delay=delay,
device=device,
epoch_check=check_flg,
epoch=epoch,
pred=pred,
batch_size=batch_size,
minibatch_size=minibatch_size,
dy_squared=dy_squared)
check_flg = False
this_train_score = ra(
percent_argmax_incorrect(output_from_state(training_states),
y_train[ixs]))
if is_online_checkpoint():
print(
f'Epoch {epoch:.3g}: Iter {i}: Score {this_train_score:.3g}%: Mean Rate: {sp(i):.2g}'
)
pi.print_update(info=f'Epoch: {epoch}')
results[next, :], train_err, test_err, val_err = do_test()
yield results
def train_and_test_predictor(f_train,
f_predict,
losses,
training_data_gen,
test_data_gen_constructors,
n_training_iters=None,
n_test_iters=None,
test_checkpoints=('lin', 1000),
collapse_loss='mean',
progress_update_period='5s',
in_test_callback=None,
post_test_callback=None,
post_train_callback=None,
save_train_return=False,
measures=None,
iterations_to_end=False):
"""
:param f_train:
:param f_predict:
:param losses:
:param training_data_gen:
:param test_data_gen_constructors:
:param samples_count:
:param n_training_iters:
:param n_test_iters:
:param test_checkpoints:
:param collapse_loss:
:param progress_update_period:
:param in_test_callback:
:param post_test_callback:
:param post_train_callback:
:param save_train_return:
:param measures:
:return: If yield_array_data is False, an ArrayStruct with fields:
'training' Results recorded during training callbacks
training_iter The iteration of the training callback
'testing' Results recorded during tests
test_iter The index of the test
'iter' The number of training iterations finished at the time that the test is run
'time' The time at which the test is run
'samples' The number of samples seen so far
'results' A structure containing results
subset_name The name of the testing subset e.g. 'train', 'test' (you give subset names in test_data_gen_constructors)
'losses'
loss_name The name of the loss function (you provide this in losses)
n_tests The numner of tests that were run for this subset
'time' The time, in seconds, that it took to test on this subset.
Otherwise, if true, a structure the same object but training_iter and test_iter pushed to the leaf-position
"""
if measures is None:
measures = Duck()
if 'training' not in measures:
measures['training'] = Duck()
if 'testing' not in measures:
measures['testing'] = Duck()
is_test_time = Checkpoints(test_checkpoints) if not isinstance(
test_checkpoints, Checkpoints) else test_checkpoints
pi = ProgressIndicator(n_training_iters,
"Training",
update_every=progress_update_period)
for inputs, targets in training_data_gen:
if is_test_time():
this_test_measures = measures['testing'].open(next)
this_test_measures['iter'] = pi.get_iterations()
this_test_measures['time'] = pi.get_elapsed()
this_test_measures['results'] = do_test(
test_subset_generators={
subset_name: constructor()
for subset_name, constructor in
test_data_gen_constructors.items()
},
f_predict=f_predict,
loss_dict=losses,
n_test_iters=n_test_iters,
collapse_loss=collapse_loss,
in_test_callback=in_test_callback,
)
if post_test_callback is not None:
post_test_callback(this_test_measures)
if iterations_to_end:
measures_to_yield = measures.arrayify_axis(axis=1,
subkeys='training')
measures_to_yield = measures_to_yield.arrayify_axis(
axis=1, subkeys='testing', inplace=True)
yield measures_to_yield.to_struct()
else:
yield measures
train_return = f_train(inputs, targets)
pi.print_update()
if save_train_return:
measures['training', 'returns'] = train_return
if post_train_callback:
return_val = post_train_callback(inputs=inputs,
targets=targets,
iter=pi.get_iterations())
if return_val is not None:
measures['training', 'callback', next, ...] = return_val
def train_online_network_checkpoints(model, dataset, checkpoint_generator = None, \
test_online=True, return_output = True, n_tests=0, offline_test_mode=None, is_cuda=False, \
online_test_reporter ='recent', error_func ='mse', batchify = False, print_every = 5):
"""
:param model: A TrainableStatefulModule
:param dataset: A 4-tuple of (x_train, y_train, x_test, y_test) where the first axis of each is the sample #
:param n_tests: Number of "splits", in the training set... (where we run a full test)
:return: train_test_errors: A tuple of (time_step, train_error, test_error)
"""
data = numpy_struct_to_torch_struct(dataset, cast_floats='float32')
if batchify:
data = [x[:, None] for x in data]
if len(data) == 4:
x_train, y_train, x_test, y_test = data
elif len(data) == 2:
x_train, y_train = data
x_test, y_test = [], []
else:
raise Exception(
'Expected data to be (x_train, y_train, x_test, y_test) or (x_test, y_test)'
)
assert len(y_train) == len(x_train)
assert len(x_test) == len(y_test)
if is_cuda:
x_train = x_train.cuda()
y_train = y_train.cuda()
x_test = x_test.cuda()
y_test = y_test.cuda()
if isinstance(checkpoint_generator, tuple):
distribution = checkpoint_generator[0]
if distribution == 'even':
interval, = checkpoint_generator[1:]
checkpoint_generator = (interval * i for i in itertools.count(1))
elif distribution == 'exp':
first, growth = checkpoint_generator[1:]
checkpoint_generator = (first * i * (1 + growth)**(i - 1)
for i in itertools.count(1))
else:
raise Exception("Can't make a checkpoint generator {}".format(
checkpoint_generator))
if isinstance(error_func, str):
error_func = create_loss_function(error_func)
n_training_samples = len(x_train)
test_iterations = [
int(n_training_samples * i / float(n_tests - 1))
for i in range(0, n_tests)
]
initial_state = model.get_state()
# results = SequentialStructBuilder()
results = Duck()
if test_online:
loss_accumulator = RunningAverage(
) if online_test_reporter == 'cum' else RecentRunningAverage(
) if online_test_reporter == 'recent' else lambda x: x if online_test_reporter is None else bad_value(
online_test_reporter)
t_start = time.time()
next_checkpoint = float('inf') if checkpoint_generator is None else next(
checkpoint_generator)
err = np.nan
pi = ProgressIndicator(
n_training_samples + 1,
update_every=(print_every, 'seconds'),
show_total=True,
post_info_callback=lambda: 'Iteration {} of {}. Online {} Error: {}'.
format(t, len(x_train), online_test_reporter, err))
for t in range(n_training_samples + 1):
if offline_test_mode is not None and t in test_iterations:
training_state = model.get_state()
model.set_state(initial_state)
if offline_test_mode == 'full_pass':
y_train_guess = torch_loop(model, is_cuda,
x_train[:t]) if t > 0 else None
if t < len(x_train) - 1:
y_middle_guess = torch_loop(model, is_cuda,
x_train[t:None])
y_test_guess = torch_loop(model, is_cuda, x_test)
if is_cuda:
y_train_guess = y_test_guess.cuda()
y_test_guess = y_test_guess.cuda()
if is_cuda:
train_err = error_func(
_flatten_first_2(y_train_guess),
_flatten_first_2(y_train[:t])).data.cpu().numpy(
) if y_train_guess is not None else np.nan
test_err = error_func(
_flatten_first_2(y_test_guess),
_flatten_first_2(y_test)).data.cpu().numpy()
else:
train_err = error_func(
_flatten_first_2(y_train_guess),
_flatten_first_2(y_train[:t])).data.numpy(
) if y_train_guess is not None else np.nan
test_err = error_func(
_flatten_first_2(y_test_guess),
_flatten_first_2(y_test)).data.numpy()
# train_err, test_err = tuple((y_guess - y_truth).abs().sum().data.numpy() for y_guess, y_truth in [(y_train_guess, y_train[:t] if t>0 else torch.zeros(2, 2, 2)/0), (y_test_guess, y_test)])
print('Iteration {} of {}: Training: {:.3g}, Test: {:.3g}'.
format(t, len(x_train), train_err, test_err))
results['offline_errors', next, :] = dict(t=t,
train=train_err,
test=test_err)
# results['offline_errors']['t'].next = t
# results['offline_errors']['train'].next = train_err
# results['offline_errors']['test'].next = test_err
elif offline_test_mode == 'cold_test':
y_test_guess = torch_loop(model, is_cuda, x_test)
y_test_guess = _flatten_first_2(y_test_guess)
if is_cuda:
y_test_guess = y_test_guess.cuda()
test_err = error_func(
y_test_guess,
_flatten_first_2(y_test)).data.cpu().numpy()
else:
test_err = error_func(
y_test_guess, _flatten_first_2(y_test)).data.numpy()
print('Iteration {} of {}: Test: {:.3g}'.format(
t, len(x_train), test_err))
results['offline_errors', next, :] = dict(t=t, test=test_err)
# results['offline_errors']['t'].next = t
# results['offline_errors']['test'].next = test_err
else:
raise Exception('No test_mode: {}'.format(offline_test_mode))
model.set_state(training_state)
if t < n_training_samples:
#model.set_state(initial_state)
#training_state = model.get_state()
#model.set_state(training_state)
out = model.train_it(x_train[t], y_train[t], is_cuda)
if return_output:
if is_cuda:
results['output', next] = out.data.cpu().numpy()[0]
else:
results['output', next] = out.data.numpy()[0]
if test_online:
# print 'Out: {}, Correct: {}'.format(np.argmax(out.data.numpy(), axis=1), torch_str(y_train[t]))
this_loss = error_func(out, y_train[t]).item()
err = loss_accumulator(this_loss)
results['online_errors', next] = this_loss
if online_test_reporter is not None:
results['smooth_online_errors', online_test_reporter,
next] = err
# if t in test_iterations:
# print('Iteration {} of {} Online {} Error: {}'.format(t, len(x_train), online_test_reporter, err))
pi()
if t >= next_checkpoint or t == n_training_samples:
results['checkpoints', next, :] = {
'iter': t,
'runtime': time.time() - t_start
}
yield results
next_checkpoint = next(checkpoint_generator)
# yield nested_map(lambda x: np.array(x), results, is_container_func=lambda x: isinstance(x, dict))
import pdb
pdb.set_trace()
def test_simple_growing():
a = Duck()
for i in range(10):
a[next] = i*2
assert np.array_equal(a.to_array(), np.arange(0, 20, 2))
def test_dictarraylist():
# We build the same thing in four different ways
a = Duck()
a['a', 'aa1'] = 1
a['a', 'aa2'] = 2
a['b', 0, 'subfield1'] = 4
a['b', 0, 'subfield2'] = 5
a['b', 1, 'subfield1'] = 6
a['b', 1, 'subfield2'] = 7
a['b', 2, 'subfield2'] = 9 # Note: Out of order assignment here!
a['b', 2, 'subfield1'] = 8
b = Duck.from_struct(a.to_struct())
assert a.to_struct() == b.to_struct()
c = Duck()
c['a', :] = OrderedDict([('aa1', 1), ('aa2', 2)])
c['b', next, :] = OrderedDict([('subfield1', 4), ('subfield2', 5)])
c['b', next, :] = OrderedDict([('subfield1', 6), ('subfield2', 7)])
c['b', next, :] = OrderedDict([('subfield2', 9), ('subfield1', 8)])
d = Duck()
d['a', :] = OrderedDict([('aa1', 1), ('aa2', 2)])
d['b', :, :] = [
OrderedDict([('subfield1', 4), ('subfield2', 5)]),
OrderedDict([('subfield1', 6), ('subfield2', 7)]),
OrderedDict([('subfield2', 9), ('subfield1', 8)])
]
e = Duck()
e['a', ...] = OrderedDict([('aa1', 1), ('aa2', 2)])
e['b', ...] = [
OrderedDict([('subfield1', 4), ('subfield2', 5)]),
OrderedDict([('subfield1', 6), ('subfield2', 7)]),
OrderedDict([('subfield2', 9), ('subfield1', 8)])
]
f = a[:]
g = a[:, :]
for i, same_struct in enumerate([a, b, c, d, e, f, g]):
print('Test {}'.format(i))
assert a.to_struct() == same_struct.to_struct()
assert a == same_struct
assert compute_fixed_hash(a.to_struct()) == compute_fixed_hash(
b.to_struct())
assert same_struct['a', 'aa2'] == 2
assert list(same_struct['b', 1, :].values()) == [6, 7]
assert list(same_struct['b', :, 'subfield1'].values()) == [4, 6, 8]
assert same_struct['b', :, :].deepvalues() == [[4, 5], [6, 7], [
9, 8
]] # Note that the order-switching gets through.
assert same_struct['b', :,
['subfield1', 'subfield2']].deepvalues() == [[4, 5],
[6, 7],
[8, 9]]
assert same_struct['b', :, :].to_struct() == [
OrderedDict([('subfield1', 4), ('subfield2', 5)]),
OrderedDict([('subfield1', 6), ('subfield2', 7)]),
OrderedDict([('subfield2', 9), ('subfield1', 8)])
]
assert same_struct['b', 1:, :].to_struct() == [
OrderedDict([('subfield1', 6), ('subfield2', 7)]),
OrderedDict([('subfield2', 9), ('subfield1', 8)])
]
with pytest.raises(KeyError):
x = same_struct['a', 'aa3']
with pytest.raises(KeyError):
x = same_struct['a', 1, 'subfield1']
with pytest.raises(IndexError):
x = same_struct['b', 4, 'subfield1']
with pytest.raises(
KeyError
): # This should raise an error because key 'a' does not have subkeys 1, 'subfield1'
x = same_struct[:, 1, 'subfield1']
assert np.array_equal(
same_struct['b'].to_array(),
[[4, 5], [6, 7], [8, 9]]) # Note... the order is corrected.
with pytest.raises(InvalidKeyError):
assert np.array_equal(
same_struct.to_array()) # Note... the order is corrected.
new_struct = same_struct.arrayify_axis(axis=1, subkeys=('b', ))
assert np.array_equal(new_struct['b', 'subfield1'], [4, 6, 8])
assert np.array_equal(new_struct['b', 'subfield2'], [5, 7, 9])
def test_slice_on_start():
a = Duck()
a['testing'] = Duck()
b = a.arrayify_axis(axis=1, subkeys='testing')
assert np.array_equal(b['testing'], [])
def test_slice_on_start():
a = Duck()
a['testing'] = Duck()
b = a.arrayify_axis(axis=1, subkeys='testing')
assert np.array_equal(b['testing'], [])
def test_slice_assignment():
deep_struct = {'a': [1, {'c': 2, 'd': 3}], 'b': [1, 2]}
a = Duck()
a[:] = deep_struct
assert list(a.items(depth='full')) == [(('a', ), [1, {
'c': 2,
'd': 3
}]), (('b', ), [1, 2])]
a = Duck()
a[:, :] = deep_struct
assert list(a.items(depth='full')) == [(('a', 0), 1),
(('a', 1), {
'c': 2,
'd': 3
}), (('b', 0), 1), (('b', 1), 2)]
a = Duck()
with raises(InitializerTooShallowError):
a[:, :, :] = deep_struct
a = Duck()
a[...] = deep_struct
assert list(a.items(depth='full')) == [(('a', 0), 1), (('a', 1, 'c'), 2),
(('a', 1, 'd'), 3), (('b', 0), 1),
(('b', 1), 2)]
a = Duck()
with raises(
InvalidKeyError
): # It would be nice to add this functionality, but maybe later.
a[..., next] = deep_struct
def experiment_mnist_eqprop(
layer_constructor,
n_epochs=10,
hidden_sizes=(500, ),
minibatch_size=20,
beta=.5,
random_flip_beta=True,
learning_rate=.05,
n_negative_steps=20,
n_positive_steps=4,
initial_weight_scale=1.,
online_checkpoints_period=None,
epoch_checkpoint_period=.25,
skip_zero_epoch_test=False,
n_test_samples=None,
prop_direction: Union[str, Tuple] = 'neutral',
bidirectional=True,
renew_activations=True,
do_fast_forward_pass=False,
rebuild_coders=True,
l2_loss=None,
splitstream=True,
seed=1234,
):
"""
Replicate the results of Scellier & Bengio:
Equilibrium Propagation: Bridging the Gap between Energy-Based Models and Backpropagation
https://www.frontiersin.org/articles/10.3389/fncom.2017.00024/full
Specifically, the train_model demo here:
https://github.com/bscellier/Towards-a-Biologically-Plausible-Backprop
Differences between our code and theirs:
- We do not keep persistent layer activations tied to data points over epochs. So our results should only really match for the first epoch.
- We evaluate training score periodically, rather than online average (however you can see online score by setting online_checkpoints_period to something that is not None)
"""
print('Params:\n' +
'\n'.join(list(f' {k} = {v}' for k, v in locals().items())))
rng = get_rng(seed)
n_in = 784
n_out = 10
dataset = get_mnist_dataset(flat=True, n_test_samples=None).to_onehot()
x_train, y_train = dataset.training_set.xy
x_test, y_test = dataset.test_set.xy # Their 'validation set' is our 'test set'
if is_test_mode():
x_train, y_train, x_test, y_test = x_train[:
100], y_train[:
100], x_test[:
100], y_test[:
100]
n_epochs = 1
layer_sizes = [n_in] + list(hidden_sizes) + [n_out]
rng = get_rng(rng)
y_train = y_train.astype(np.float32)
ra = RunningAverage()
sp = Speedometer(mode='last')
is_online_checkpoint = Checkpoints(
online_checkpoints_period, skip_first=skip_zero_epoch_test
) if online_checkpoints_period is not None else lambda: False
is_epoch_checkpoint = Checkpoints(epoch_checkpoint_period,
skip_first=skip_zero_epoch_test)
results = Duck()
training_states = initialize_states(
layer_constructor=layer_constructor,
n_samples=minibatch_size,
params=initialize_params(layer_sizes=layer_sizes,
initial_weight_scale=initial_weight_scale,
rng=rng))
if isinstance(prop_direction, str):
fwd_prop_direction, backward_prop_direction = prop_direction, prop_direction
else:
fwd_prop_direction, backward_prop_direction = prop_direction
for i, (ixs, info) in enumerate(
minibatch_index_info_generator(n_samples=x_train.shape[0],
minibatch_size=minibatch_size,
n_epochs=n_epochs)):
epoch = i * minibatch_size / x_train.shape[0]
if is_epoch_checkpoint(epoch):
n_samples = n_test_samples if n_test_samples is not None else len(
x_test)
y_pred_test, y_pred_train = [
run_inference(
x_data=x[:n_test_samples],
states=initialize_states(
layer_constructor=layer_constructor,
params=[s.params for s in training_states],
n_samples=min(len(x), n_test_samples)
if n_test_samples is not None else len(x)),
n_steps=n_negative_steps,
prop_direction=fwd_prop_direction,
) for x in (x_test, x_train)
]
# y_pred_train = run_inference(x_data=x_train[:n_test_samples], states=initialize_states(params=[s.params for s in training_states], n_samples=min(len(x_train), n_test_samples) if n_test_samples is not None else len(x_train)))
test_error = percent_argmax_incorrect(y_pred_test,
y_test[:n_test_samples])
train_error = percent_argmax_incorrect(y_pred_train,
y_train[:n_test_samples])
print(
f'Epoch: {epoch:.3g}, Iter: {i}, Test Error: {test_error:.3g}%, Train Error: {train_error:.3g}, Mean Rate: {sp(i):.3g}iter/s'
)
results[next, :] = dict(iter=i,
epoch=epoch,
train_error=train_error,
test_error=test_error)
yield results
if epoch > 2 and train_error > 50:
return
# The Original training loop, just taken out here:
x_data_sample, y_data_sample = x_train[ixs], y_train[ixs]
training_states = run_eqprop_training_update(
x_data=x_data_sample,
y_data=y_data_sample,
layer_states=training_states,
beta=beta,
random_flip_beta=random_flip_beta,
learning_rate=learning_rate,
layer_constructor=layer_constructor,
bidirectional=bidirectional,
l2_loss=l2_loss,
renew_activations=renew_activations,
n_negative_steps=n_negative_steps,
n_positive_steps=n_positive_steps,
prop_direction=prop_direction,
splitstream=splitstream,
rng=rng)
this_train_score = ra(
percent_argmax_correct(output_from_state(training_states),
y_train[ixs]))
if is_online_checkpoint():
print(
f'Epoch {epoch:.3g}: Iter {i}: Score {this_train_score:.3g}%: Mean Rate: {sp(i):.2g}'
)
def test_simple_growing():
a = Duck()
for i in range(10):
a[next] = i * 2
assert np.array_equal(a.to_array(), np.arange(0, 20, 2))
