def batch(sem_kwargs, gibbs_kwargs, epsilon_e, batch_number=0):
x_list_no_switch, x_list_switch = generate_task()
n, d = np.concatenate(x_list_switch).shape
# run through with the switch condition
sem_switch = SEM(**sem_kwargs)
sem_switch.run_w_boundaries(list_events=x_list_switch, leave_progress_bar=False)
# create the corrupted memory traces
y_mem_switch = list() # these are list, not sets, for hashability
y_mem_noswitch = list() # these are list, not sets, for hashability
for t in range(n):
# n.b. python uses stdev, not var
x_mem = x_list_switch[t / 10][t % 10, :] + np.random.normal(scale=gibbs_kwargs['tau'] ** 0.5, size=d)
e_mem = [None, sem_switch.event_models.keys()[t / (n / 2)]][np.random.rand() < epsilon_e]
t_mem = t + np.random.randint(-gibbs_kwargs['b'], gibbs_kwargs['b'] + 1)
y_mem_switch.append([x_mem, e_mem, t_mem])
# do the no-switch condition ahead of time
e_mem = [None, 0][np.random.rand() < epsilon_e]
y_mem_noswitch.append([x_mem, e_mem, t_mem])
# sample from memory
gibbs_kwargs['y_mem'] = y_mem_switch
gibbs_kwargs['sem_model'] = sem_switch
y_samples, e_samples, _ = gibbs_memory_sampler(**gibbs_kwargs)
results = pd.DataFrame({
'Condition': 'Shift',
'r2': adjusted_rand_score(sem_switch.results.e_hat, np.array([0, 1])),
'Reconstruction Segementation': evaluate_seg(e_samples, np.concatenate([[e0] * 10 for e0 in sem_switch.event_models])),
'Overall Acc': reconstruction_accuracy(y_samples, y_mem_switch).mean(),
'Non-boundary Acc': evaluate_bound_acc(y_samples, y_mem_switch),
'Boundary Acc': evaluate_non_bound_acc(y_samples, y_mem_switch),
'Batch': [batch_number],
}, index=[batch_number])
clear_sem(sem_switch)
sem_switch = None
# run through with the no-switch condition
sem_no_switch = SEM(**sem_kwargs)
sem_no_switch.run_w_boundaries(list_events=x_list_no_switch, leave_progress_bar=False)
gibbs_kwargs['y_mem'] = y_mem_noswitch
gibbs_kwargs['sem_model'] = sem_no_switch
y_samples, e_samples, x_samples = gibbs_memory_sampler(**gibbs_kwargs)
results = pd.concat([results, pd.DataFrame({
'Condition': 'No-Shift',
'Overall Acc': reconstruction_accuracy(y_samples, y_mem_noswitch).mean(),
'Non-boundary Acc': evaluate_bound_acc(y_samples, y_mem_noswitch),
'Boundary Acc': evaluate_non_bound_acc(y_samples, y_mem_noswitch),
'Batch': [batch_number],
}, index=[batch_number])], sort=True)
clear_sem(sem_no_switch)
sem_no_switch = None
return results
def batch_switch_only(sem_kwargs, gibbs_kwargs, epsilon_e, batch_number=0):
_, x_list_switch = generate_task()
n, d = np.concatenate(x_list_switch).shape
sem_switch = SEM(**sem_kwargs)
sem_switch.run_w_boundaries(list_events=x_list_switch, leave_progress_bar=False)
# create the corrupted memory traces
y_mem_switch = list() # these are list, not sets, for hashability
for t in range(n):
x_mem = x_list_switch[t / 10][t % 10, :] + np.random.randn(d) * gibbs_kwargs['tau']
e_mem = [None, sem_switch.event_models.keys()[t / (n / 2)]][np.random.rand() < epsilon_e]
t_mem = t + np.random.randint(-gibbs_kwargs['b'], gibbs_kwargs['b'] + 1)
y_mem_switch.append([x_mem, e_mem, t_mem])
# sample from memory
gibbs_kwargs['y_mem'] = y_mem_switch
gibbs_kwargs['sem'] = sem_switch
y_samples, e_samples, x_samples = gibbs_memory_sampler(**gibbs_kwargs)
results = pd.DataFrame({
'Condition': 'Shift',
'r2': adjusted_rand_score(sem_switch.results.e_hat, np.array([0, 1])),
'Reconstruction Segementation': evaluate_seg(e_samples,
np.concatenate([[e0] * 10 for e0 in sem_switch.event_models])),
'Overall Acc': eval_acc(y_samples, y_mem_switch),
'Non-boundary Acc': evaluate_bound_acc(y_samples, y_mem_switch),
'Boundary Acc': evaluate_non_bound_acc(y_samples, y_mem_switch),
'Batch': [batch_number],
}, index=[batch_number])
return results
def run_block(sem_kwargs, gibbs_kwargs, epsilon_e, block_number=0):
# generate an experiment
x_list_items, e_tokens = generate_experiment()
n, d = np.concatenate(x_list_items).shape
pre_locs = [
ii for ii in range(len(e_tokens) - 1)
if e_tokens[ii] != e_tokens[ii + 1]
]
pst_locs = [
ii for ii in range(1, len(e_tokens))
if e_tokens[ii] != e_tokens[ii - 1]
]
# Train SEM on the stimuli
sem = SEM(**sem_kwargs)
sem.run_w_boundaries(list_events=x_list_items, progress_bar=False)
e_seg = np.reshape([[ii] * np.sum(e_tokens == t, dtype=int)
for t, ii in enumerate(sem.results.e_hat)], -1)
# create the corrupted memory trace
y_mem = list() # these are list, not sets, for hashability
for t in range(n):
# n.b. python uses stdev, not var
x_mem = np.concatenate(x_list_items)[t, :] + np.random.normal(
scale=gibbs_kwargs['tau']**0.5, size=d)
e_mem = [None, e_seg[t]][np.random.rand() < epsilon_e]
t_mem = t + np.random.randint(-gibbs_kwargs['b'],
gibbs_kwargs['b'] + 1)
y_mem.append([x_mem, e_mem, t_mem])
# add the models to the kwargs
y_samples, e_samples, x_samples = gibbs_memory_sampler(
y_mem, sem, **gibbs_kwargs)
results = pd.DataFrame({
'Block': [block_number],
'Adj-r2':
[adjusted_rand_score(sem.results.e_hat, np.array([0, 1, 0, 1, 0]))],
'Recon Segment':
evaluate_seg(e_samples, e_seg),
'Overall Acc':
eval_acc(y_samples, y_mem),
'Pre-Boundary':
np.mean([
evaluate_item_position_acc(y_samples, y_mem, t) for t in pre_locs
]),
'Boundary':
np.mean([
evaluate_item_position_acc(y_samples, y_mem, t) for t in pst_locs
]),
'Transitions Pre-Boundary':
np.mean([score_transitions(y_samples, y_mem, t) for t in pre_locs]),
'Transitions Boundary':
np.mean([score_transitions(y_samples, y_mem, t) for t in pst_locs]),
'Pre-boundary Acc':
eval_item_acc(y_samples, y_mem, pre_locs),
'Boundary Acc':
eval_item_acc(y_samples, y_mem, pst_locs),
})
clear_sem(sem)
sem = None
return results
def batch(sem_kwargs,
gibbs_kwargs,
epsilon_e_switch=0.25,
epsilon_e_noswitch=0.75,
gamma=2.5,
n_rooms=25,
progress_bar=True):
sem_model = SEM(**sem_kwargs)
_gibbs_kwargs = {
k: v
for k, v in gibbs_kwargs.iteritems() if k != 'e_true'
}
acc = []
list_events, list_objects = make_task(n_rooms=n_rooms)
sem_model.init_for_boundaries(list_events)
if progress_bar:
def my_it(iterator):
return tqdm(iterator,
desc='Run SEM',
leave=False,
total=len(list_events))
else:
def my_it(iterator):
return iterator
y_mem_switch = list()
for itt, x in my_it(enumerate(list_events)):
sem_model.update_single_event(x)
n_items, d = np.shape(x)
e_list = np.concatenate([[sem_model.results.e_hat[itt]] * n_items
for t in range(n_rooms)])
# create a corrupted memory trace for the switch condition
y_mem_noswitch = [yi for yi in y_mem_switch]
for t in range(n_items):
x_mem = x[t, :] + np.random.normal(
scale=_gibbs_kwargs['tau']**0.5,
size=d) # note, python uses stdev, not var
e_mem = [None, sem_model.results.e_hat[-1]
][np.random.rand() < epsilon_e_switch]
t_mem = t + np.random.randint(-_gibbs_kwargs['b'],
_gibbs_kwargs['b'] + 1)
y_mem_switch.append([x_mem, e_mem, t_mem])
# for the no-switch condition
e_mem = [None, sem_model.results.e_hat[-1]
][np.random.rand() < epsilon_e_noswitch]
y_mem_noswitch.append([x_mem, e_mem, t_mem])
# for speed, just reconstruct the past 3 events at max
if len(y_mem_switch) > 3 * 2:
y_mem_switch = y_mem_switch[-6:]
y_mem_noswitch = y_mem_noswitch[-6:]
e_list = e_list[-6:]
# reconstruct (Switch)
_gibbs_kwargs['y_mem'] = y_mem_switch
_gibbs_kwargs['sem_model'] = sem_model
y_samples, e_samples, x_samples = gibbs_memory_sampler(**_gibbs_kwargs)
x_samples = np.array(x_samples)
item_acc = reconstruction_accuracy(y_samples=y_samples,
y_mem=y_mem_switch)
# evaluate the probability of the associated vs dissociated items
obj_a, obj_b = list_objects[itt]
x_samples_ii = np.reshape(x_samples[:, -2:, :], (-1, d))
p_a_greater_than_b = \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_a, axis=1) * gamma) < \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_b, axis=1) * gamma)
# use the correct scoring method
acc.append({
'Room Number':
itt,
'Condition':
'Switch',
'Reconstruction Accuracy':
item_acc.mean(),
'Last Room Reconstruction Acc':
item_acc[-2:].mean(),
'Pr(A > B)':
p_a_greater_than_b,
'Reconstruction Segementation':
evaluate_seg(e_samples, e_list),
})
# clear things from memory
y_samples, e_samples, x_samples = None, None, None
# reconstruct (No-Switch)
_gibbs_kwargs['y_mem'] = y_mem_noswitch
y_samples, e_samples, x_samples = gibbs_memory_sampler(**_gibbs_kwargs)
x_samples = np.array(x_samples)
item_acc = reconstruction_accuracy(y_samples=y_samples,
y_mem=y_mem_noswitch)
# evaluate the probability of the associated vs dissociated items
obj_a, obj_b = list_objects[itt]
x_samples_ii = np.reshape(x_samples[:, -2:, :], (-1, d))
p_a_greater_than_b = \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_a, axis=1) * gamma) < \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_b, axis=1) * gamma)
# use the correct scoring method
acc.append({
'Room Number':
itt,
'Condition':
'No-Switch',
'Last Room Reconstruction Acc':
item_acc[-2:].mean(),
'Reconstruction Accuracy':
item_acc.mean(),
'Pr(A > B)':
p_a_greater_than_b,
'Reconstruction Segementation':
evaluate_seg(e_samples, e_list),
})
# clear things from memory
y_samples, e_samples, x_samples = None, None, None
# clear SEM from memory
clear_sem(sem_model)
sem_model = None
return pd.DataFrame(acc)
def batch(sem,
gibbs_kwargs,
epsilon_e_switch=0.25,
epsilon_e_noswitch=0.75,
gamma=2.5,
n_rooms=25):
gibbs_kwargs = {k: v for k, v in gibbs_kwargs.iteritems() if k != 'e_true'}
acc = []
list_events, list_objects = make_task(n_rooms=n_rooms)
sem.init_for_boundaries(list_events)
y_mem_switch = list()
for itt, x in tqdm(enumerate(list_events),
leave=False,
total=len(list_events)):
sem.update_single_event(x)
n_items, d = np.shape(x)
# create a corrupted memory trace for the switch condition
y_mem_noswitch = [yi for yi in y_mem_switch]
for t in range(n_items):
x_mem = x[t, :] + np.random.randn(d) * gibbs_kwargs['tau']
e_mem = [None, sem.event_models.keys()[-1]
][np.random.rand() < epsilon_e_switch]
t_mem = t + np.random.randint(-gibbs_kwargs['b'],
gibbs_kwargs['b'] + 1)
y_mem_switch.append([x_mem, e_mem, t_mem])
# for the no-switch condition
e_mem = [None, sem.event_models.keys()[-1]
][np.random.rand() < epsilon_e_noswitch]
y_mem_noswitch.append([x_mem, e_mem, t_mem])
# for speed, just reconstruct the past 3 events at max
if len(y_mem_switch) > 2 * 4:
y_mem_switch = y_mem_switch[-8:]
y_mem_noswitch = y_mem_noswitch[-8:]
# reconstruct (Switch)
gibbs_kwargs['y_mem'] = y_mem_switch
gibbs_kwargs['sem'] = sem
y_samples, e_samples, x_samples = gibbs_memory_sampler(**gibbs_kwargs)
x_samples = np.array(x_samples)
item_acc = eval_acc(y_samples=y_samples, y_mem=y_mem_switch)
# evaluate the probability of the associated vs dissociated items
obj_a, obj_b = list_objects[itt]
x_samples_ii = np.reshape(x_samples[:, -4:, :], (-1, d))
p_a_greater_than_b = \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_a, axis=1) * gamma) < \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_b, axis=1) * gamma)
# use the correct scoring method
acc.append({
'Condition': 'Switch',
'Accuracy': item_acc.mean(),
'Pr(A > B)': p_a_greater_than_b,
})
# reconstruct (No-Switch)
gibbs_kwargs['y_mem'] = y_mem_noswitch
y_samples, e_samples, x_samples = gibbs_memory_sampler(**gibbs_kwargs)
x_samples = np.array(x_samples)
item_acc = eval_acc(y_samples=y_samples, y_mem=y_mem_noswitch)
# evaluate the probability of the associated vs dissociated items
obj_a, obj_b = list_objects[itt]
x_samples_ii = np.reshape(x_samples[:, -4:, :], (-1, d))
p_a_greater_than_b = \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_a, axis=1) * gamma) < \
-logsumexp(-np.linalg.norm(x_samples_ii - obj_b, axis=1) * gamma)
# use the correct scoring method
acc.append({
'Condition': 'No-Switch',
'Accuracy': item_acc.mean(),
'Pr(A > B)': p_a_greater_than_b,
})
# clear SEM from memory
sem.clear_event_models()
sem = None
return pd.DataFrame(acc)