def test_exponential_preprocessings():
data = np.array([ 1, 3, 5, 9, 7, -2])
exp_run_mean = np.array([ 0.200000, 0.160000, 0.328000, 0.262400, 0.409920, 0.327936])
exp_run_var = np.array([ 0.928000, 1.745920, 3.697869, 9.952428, 9.922424, 12.347100])
exp_standardized = np.array([ 0.830455, 1.695258, 1.763925, 1.874509, 0.993934, -1.336228])
run_mean = exponential_running_mean(np.array(data), factor_new=0.2,start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned, factor_new=0.2, start_var=1)
standardized = exponential_running_standardize(data, factor_new=0.2, start_mean=0, start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([ 2, 0, -5, -3, 0, 4])
exp_run_mean = np.array([ 0.400000, 0.320000, -0.744000, -1.195200, -0.956160, 0.035072])
exp_run_var = np.array([ 1.312000, 1.070080, 4.478771, 4.234478, 3.570431, 6.000475])
exp_standardized = np.array([ 1.396861, -0.309344, -2.011047, -0.877060, 0.506023, 1.618611])
run_mean = exponential_running_mean(np.array(data), factor_new=0.2,start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned, factor_new=0.2, start_var=1)
standardized = exponential_running_standardize(data, factor_new=0.2, start_mean=0, start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([ -3, 5, 8, 7, 4, -2])
exp_run_mean = np.array(np.array([ -0.600000, 0.520000, 2.016000, 3.012800, 3.210240, 2.168192]))
exp_run_var = np.array([ 1.952000, 5.575680, 11.622195, 12.477309, 10.106591, 11.560038])
exp_standardized = np.array([ -1.717795, 1.897270, 1.755284, 1.128775, 0.248424, -1.225937])
run_mean = exponential_running_mean(np.array(data), factor_new=0.2,start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned, factor_new=0.2, start_var=1)
standardized = exponential_running_standardize(data, factor_new=0.2, start_mean=0, start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([ 1, 0, 1, 0, 1, 0])
exp_run_mean = np.array([ 0.200000, 0.160000, 0.328000, 0.262400, 0.409920, 0.327936])
exp_run_var = np.array([ 0.928000, 0.747520, 0.688333, 0.564437, 0.521188, 0.438459])
exp_standardized = np.array([ 0.830455, -0.185058, 0.809972, -0.349266, 0.817360, -0.495250])
run_mean = exponential_running_mean(np.array(data), factor_new=0.2,start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned, factor_new=0.2, start_var=1)
standardized = exponential_running_standardize(data, factor_new=0.2, start_mean=0, start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
def exponential_standardize_cnt(cnt, init_block_size=1000, factor_new=1e-3,
eps=1e-4):
cnt_data = cnt.data
standardized_data = exponential_running_standardize(cnt_data,
factor_new=factor_new, init_block_size=init_block_size, axis=None,
eps=eps)
return cnt.copy(data=standardized_data)
def test_sum_prediction():
""" Test with a model that predicts sum over four samples """
rng = RandomState(3904890384)
n_samples_in_buffer = 1000
dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32)
markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32)
set_and_markers = np.concatenate((dataset, markers), axis=1)
factor_new=0.001
n_stride = 10
pred_freq = 11
standardized = exponential_running_standardize(dataset,
factor_new=factor_new, init_block_size=n_stride)
model = InputLayer([1,1,4,1])
model = GlobalPoolLayer(model,pool_function=T.sum)
expected = [np.sum(standardized[stop-4:stop], axis=0) for stop in xrange(11, dataset.shape[0], 11)]
expected = np.array(expected)
processor = StandardizeProcessor(factor_new=factor_new,
n_samples_in_buffer=n_samples_in_buffer)
online_model = OnlineModel(model)
online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq,
trainer=NoTrainer())
online_pred.initialize(n_chans=dataset.shape[1])
all_preds = []
for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride):
online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride])
if online_pred.has_new_prediction():
pred, _ = online_pred.pop_last_prediction_and_sample_ind()
all_preds.append(pred)
assert np.allclose(np.array(all_preds).squeeze(), expected, rtol=1e-3)
def test_online_predictor():
""" Test whether predictions are done at correct timepoints.
Model actually just returns input """
rng = RandomState(3904890384)
n_samples_in_buffer = 1000
dataset = rng.rand(n_samples_in_buffer*2,5).astype(np.float32)
markers = np.ones((n_samples_in_buffer*2,1)).astype(np.float32)
set_and_markers = np.concatenate((dataset, markers), axis=1)
factor_new=0.001
n_stride = 10
pred_freq = 11
standardized = exponential_running_standardize(dataset,
factor_new=factor_new, init_block_size=n_stride)
model = InputLayer([1,1,1,1])
processor = StandardizeProcessor(factor_new=factor_new,
n_samples_in_buffer=n_samples_in_buffer)
online_model = OnlineModel(model)
online_pred = OnlineCoordinator(processor, online_model, pred_freq=pred_freq,
trainer=NoTrainer())
online_pred.initialize(n_chans=dataset.shape[1])
all_preds = []
for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride):
online_pred.receive_samples(set_and_markers[i_start_sample:i_start_sample+n_stride])
if online_pred.has_new_prediction():
pred, _ = online_pred.pop_last_prediction_and_sample_ind()
all_preds.append(pred)
assert np.array_equal(np.array(all_preds).squeeze(), standardized[10::pred_freq])
def test_data_processor():
"""Compare standardized data to data standardized online
always giving blocks of 10 samples"""
rng = RandomState(3904890384)
n_samples_in_buffer = 1000
dataset = rng.rand(n_samples_in_buffer * 2, 5)
factor_new = 0.001
n_stride = 10
standardized = exponential_running_standardize(dataset,
factor_new=factor_new,
init_block_size=n_stride)
processor = StandardizeProcessor(factor_new=factor_new,
n_samples_in_buffer=n_samples_in_buffer)
processor.initialize(n_chans=dataset.shape[1])
for i_start_sample in xrange(0, dataset.shape[0] - n_stride + 1, n_stride):
processor.process_samples(dataset[i_start_sample:i_start_sample +
n_stride])
# compare all so far processed samples
assert np.allclose(
standardized[:i_start_sample + n_stride][-n_samples_in_buffer:],
processor.get_samples(-i_start_sample - n_stride, None),
rtol=1e-3,
atol=1e-5)
def test_exponential_multidimensional():
data = np.array([[[ 1, 3, 5, 9, 7, -2],
[ 2, 0, -5, -3, 0, 4]],
[[ -3, 5, 8, 7, 4, -2],
[ 1, 0, 1, 0, 1, 0]]])
exp_standardized = np.array([[[ 0.830455, 1.695258, 1.763925, 1.874509, 0.993934, -1.336228],
[ 1.396861, -0.309344, -2.011047, -0.877060, 0.506023, 1.618611]],
[[ -1.717795, 1.897270, 1.755284, 1.128775, 0.248424, -1.225937],
[ 0.830455, -0.185058, 0.809972, -0.349266, 0.817360, -0.495250]]])
standardized = exponential_running_standardize(data.swapaxes(0,2), factor_new=0.2, start_mean=0, start_var=1)
assert np.allclose(standardized.swapaxes(0,2), exp_standardized)
def test_exponential_multidimensional():
data = np.array([[[1, 3, 5, 9, 7, -2], [2, 0, -5, -3, 0, 4]],
[[-3, 5, 8, 7, 4, -2], [1, 0, 1, 0, 1, 0]]])
exp_standardized = np.array(
[[[0.830455, 1.695258, 1.763925, 1.874509, 0.993934, -1.336228],
[1.396861, -0.309344, -2.011047, -0.877060, 0.506023, 1.618611]],
[[-1.717795, 1.897270, 1.755284, 1.128775, 0.248424, -1.225937],
[0.830455, -0.185058, 0.809972, -0.349266, 0.817360, -0.495250]]])
standardized = exponential_running_standardize(data.swapaxes(0, 2),
factor_new=0.2,
start_mean=0,
start_var=1)
assert np.allclose(standardized.swapaxes(0, 2), exp_standardized)
def add_training_blocks_from_old_data(self, old_samples,
old_markers, data_processor):
# first standardize data
old_samples = exponential_running_standardize(old_samples,
factor_new=data_processor.factor_new, init_block_size=1000,
eps=data_processor.eps)
trial_starts, trial_stops = self.get_trial_start_stop_indices(
old_markers)
log.info("Adding {:d} trials".format(len(trial_starts)))
for trial_start, trial_stop in zip(trial_starts, trial_stops):
self.add_blocks(trial_start + self.trial_start_offset,
trial_stop, old_samples, old_markers)
# now lets add breaks
log.info("Adding {:d} breaks".format(len(trial_starts) - 1))
for break_start, break_stop in zip(trial_stops[:-1], trial_starts[1:]):
self.add_break(break_start, break_stop, old_samples, old_markers)
def add_training_blocks_from_old_data(self, old_samples, old_markers,
data_processor):
# first standardize data
old_samples = exponential_running_standardize(
old_samples,
factor_new=data_processor.factor_new,
init_block_size=1000,
eps=data_processor.eps)
trial_starts, trial_stops = self.get_trial_start_stop_indices(
old_markers)
log.info("Adding {:d} trials".format(len(trial_starts)))
for trial_start, trial_stop in zip(trial_starts, trial_stops):
self.add_trial(trial_start, trial_stop, old_samples, old_markers)
# now lets add breaks
log.info("Adding {:d} breaks".format(len(trial_starts) - 1))
for break_start, break_stop in zip(trial_stops[:-1], trial_starts[1:]):
self.add_break(break_start, break_stop, old_samples, old_markers)
def test_data_processor():
"""Compare standardized data to data standardized online
always giving blocks of 10 samples"""
rng = RandomState(3904890384)
n_samples_in_buffer = 1000
dataset = rng.rand(n_samples_in_buffer*2,5)
factor_new=0.001
n_stride = 10
standardized = exponential_running_standardize(dataset,
factor_new=factor_new, init_block_size=n_stride)
processor = StandardizeProcessor(factor_new=factor_new,
n_samples_in_buffer=n_samples_in_buffer)
processor.initialize(n_chans=dataset.shape[1])
for i_start_sample in xrange(0,dataset.shape[0]-n_stride+1,n_stride):
processor.process_samples(dataset[i_start_sample:i_start_sample+n_stride])
# compare all so far processed samples
assert np.allclose(standardized[:i_start_sample+n_stride][-n_samples_in_buffer:],
processor.get_samples(-i_start_sample-n_stride,None), rtol=1e-3, atol=1e-5)
def test_exponential_preprocessings():
data = np.array([1, 3, 5, 9, 7, -2])
exp_run_mean = np.array(
[0.200000, 0.160000, 0.328000, 0.262400, 0.409920, 0.327936])
exp_run_var = np.array(
[0.928000, 1.745920, 3.697869, 9.952428, 9.922424, 12.347100])
exp_standardized = np.array(
[0.830455, 1.695258, 1.763925, 1.874509, 0.993934, -1.336228])
run_mean = exponential_running_mean(np.array(data),
factor_new=0.2,
start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned,
factor_new=0.2,
start_var=1)
standardized = exponential_running_standardize(data,
factor_new=0.2,
start_mean=0,
start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([2, 0, -5, -3, 0, 4])
exp_run_mean = np.array(
[0.400000, 0.320000, -0.744000, -1.195200, -0.956160, 0.035072])
exp_run_var = np.array(
[1.312000, 1.070080, 4.478771, 4.234478, 3.570431, 6.000475])
exp_standardized = np.array(
[1.396861, -0.309344, -2.011047, -0.877060, 0.506023, 1.618611])
run_mean = exponential_running_mean(np.array(data),
factor_new=0.2,
start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned,
factor_new=0.2,
start_var=1)
standardized = exponential_running_standardize(data,
factor_new=0.2,
start_mean=0,
start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([-3, 5, 8, 7, 4, -2])
exp_run_mean = np.array(
np.array([-0.600000, 0.520000, 2.016000, 3.012800, 3.210240,
2.168192]))
exp_run_var = np.array(
[1.952000, 5.575680, 11.622195, 12.477309, 10.106591, 11.560038])
exp_standardized = np.array(
[-1.717795, 1.897270, 1.755284, 1.128775, 0.248424, -1.225937])
run_mean = exponential_running_mean(np.array(data),
factor_new=0.2,
start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned,
factor_new=0.2,
start_var=1)
standardized = exponential_running_standardize(data,
factor_new=0.2,
start_mean=0,
start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
data = np.array([1, 0, 1, 0, 1, 0])
exp_run_mean = np.array(
[0.200000, 0.160000, 0.328000, 0.262400, 0.409920, 0.327936])
exp_run_var = np.array(
[0.928000, 0.747520, 0.688333, 0.564437, 0.521188, 0.438459])
exp_standardized = np.array(
[0.830455, -0.185058, 0.809972, -0.349266, 0.817360, -0.495250])
run_mean = exponential_running_mean(np.array(data),
factor_new=0.2,
start_mean=0)
demeaned = data - run_mean
run_var = exponential_running_var_from_demeaned(demeaned,
factor_new=0.2,
start_var=1)
standardized = exponential_running_standardize(data,
factor_new=0.2,
start_mean=0,
start_var=1)
assert np.allclose(exp_run_mean, run_mean)
assert np.allclose(exp_run_var, run_var)
assert np.allclose(exp_standardized, standardized)
