def test_event_transfer(): es = EventSegment(2) sample_data = np.asarray([[1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1, 1]]) with pytest.raises(NotFittedError, message="Should need to set variance"): seg = es.find_events(sample_data.T)[0] with pytest.raises(NotFittedError, message="Should need to set patterns"): seg = es.find_events(sample_data.T, np.asarray([1, 1]))[0] es.set_event_patterns(np.asarray([[1, 0], [0, 1]])) seg = es.find_events(sample_data.T, np.asarray([1, 1]))[0] events = np.argmax(seg, axis=1) assert np.array_equal(events, [0, 0, 0, 1, 1, 1, 1]),\ "Failed to correctly transfer two events to new data"
def test_fit_shapes(): K = 5 V = 3 T = 10 es = EventSegment(K, n_iter=2) sample_data = np.random.rand(V, T) es.fit(sample_data.T) assert es.segments_[0].shape == (T, K), "Segmentation from fit " \ "has incorrect shape" assert np.isclose(np.sum(es.segments_[0], axis=1), np.ones(T)).all(), \ "Segmentation from learn_events not correctly normalized" T2 = 15 sample_data2 = np.random.rand(V, T2) test_segments, test_ll = es.find_events(sample_data2.T) assert test_segments.shape == (T2, K), "Segmentation from find_events " \ "has incorrect shape" assert np.isclose(np.sum(test_segments, axis=1), np.ones(T2)).all(), \ "Segmentation from find_events not correctly normalized" es_invalid = EventSegment(K) with pytest.raises(ValueError, message="T < K should cause error"): es_invalid.model_prior(K-1) with pytest.raises(ValueError, message="#Events < K should cause error"): es_invalid.set_event_patterns(np.zeros((V, K-1)))
def test_fit_shapes(): K = 5 V = 3 T = 10 es = EventSegment(K, n_iter=2) sample_data = np.random.rand(V, T) es.fit(sample_data.T) assert es.segments_[0].shape == (T, K), "Segmentation from fit " \ "has incorrect shape" assert np.isclose(np.sum(es.segments_[0], axis=1), np.ones(T)).all(), \ "Segmentation from learn_events not correctly normalized" T2 = 15 sample_data2 = np.random.rand(V, T2) test_segments, test_ll = es.find_events(sample_data2.T) assert test_segments.shape == (T2, K), "Segmentation from find_events " \ "has incorrect shape" assert np.isclose(np.sum(test_segments, axis=1), np.ones(T2)).all(), \ "Segmentation from find_events not correctly normalized" es_invalid = EventSegment(K) with pytest.raises(ValueError, message="T < K should cause error"): es_invalid.model_prior(K - 1) with pytest.raises(ValueError, message="#Events < K should cause error"): es_invalid.set_event_patterns(np.zeros((V, K - 1)))
def test_event_transfer(): es = EventSegment(2) sample_data = np.asarray([[1, 1, 1, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1, 1]]) with pytest.raises(NotFittedError, message="Should need to set variance"): seg = es.find_events(sample_data.T)[0] with pytest.raises(NotFittedError, message="Should need to set patterns"): seg = es.find_events(sample_data.T, np.asarray([1, 1]))[0] es.set_event_patterns(np.asarray([[1, 0], [0, 1]])) seg = es.find_events(sample_data.T, np.asarray([1, 1]))[0] events = np.argmax(seg, axis=1) assert np.array_equal(events, [0, 0, 0, 1, 1, 1, 1]),\ "Failed to correctly transfer two events to new data"
def test_chains(): es = EventSegment(5, event_chains=np.array(['A', 'A', 'B', 'B', 'B'])) es.set_event_patterns(np.array([[1, 1, 0, 0, 0], [0, 0, 1, 1, 1]])) sample_data = np.array([[0, 0, 0], [1, 1, 1]]) seg = es.find_events(sample_data.T, 0.1)[0] ev = np.nonzero(seg > 0.99)[1] assert np.array_equal(ev, [2, 3, 4]),\ "Failed to fit with multiple chains"
def test_sym_ll(): ev = np.array([0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2]) random_state = np.random.RandomState(0) ev_pat = random_state.rand(3, 10) D_forward = np.zeros((len(ev), 10)) for t in range(len(ev)): D_forward[t, :] = ev_pat[ev[t], :] + 0.1 * random_state.rand(10) D_backward = np.flip(D_forward, axis=0) hmm_forward = EventSegment(3) hmm_forward.set_event_patterns(ev_pat.T) _, ll_forward = hmm_forward.find_events(D_forward, var=1) hmm_backward = EventSegment(3) hmm_backward.set_event_patterns(np.flip(ev_pat.T, axis=1)) _, ll_backward = hmm_backward.find_events(D_backward, var=1) assert (ll_forward == ll_backward),\ "Log-likelihood not symmetric forward/backward"
def test_chains(): es = EventSegment(5, event_chains=np.array(['A', 'A', 'B', 'B', 'B'])) es.set_event_patterns(np.array([[1, 1, 0, 0, 0], [0, 0, 1, 1, 1]])) sample_data = np.array([[0, 0, 0], [1, 1, 1]]) seg = es.find_events(sample_data.T, 0.1)[0] ev = np.nonzero(seg > 0.99)[1] assert np.array_equal(ev, [2, 3, 4]),\ "Failed to fit with multiple chains"
def test_sym(): es = EventSegment(4) evpat = np.repeat(np.arange(10).reshape(-1, 1), 4, axis=1) es.set_event_patterns(evpat) D = np.repeat(np.arange(10).reshape(1, -1), 20, axis=0) ev = es.find_events(D, var=1)[0] # Check that events 1-4 and 2-3 are symmetric assert np.all(np.isclose(ev[:, :2], np.fliplr(np.flipud(ev[:, 2:])))),\ "Fit with constant data is not symmetric"
def test_sym(): es = EventSegment(4) evpat = np.repeat(np.arange(10).reshape(-1, 1), 4, axis=1) es.set_event_patterns(evpat) D = np.repeat(np.arange(10).reshape(1, -1), 20, axis=0) ev = es.find_events(D, var=1)[0] # Check that events 1-4 and 2-3 are symmetric assert np.all(np.isclose(ev[:, :2], np.fliplr(np.flipud(ev[:, 2:])))),\ "Fit with constant data is not symmetric"
def heldout_ll(data, n_events, split): """Compute log-likelihood on heldout subjects Fits an event segmentation model with n_events to half of the subjects, then measures the log-likelihood of this model on the other half. The returned log-likehood averages across both choices of which half is used for training and which is used for testing. The boolean array split defines which subjects are in each half. Parameters ---------- data : ndarray subj x TR x Voxels data array n_events : int Number of events for event segmentation model split : ndarray Boolean vector, subj in one group are True and in the other are False Returns ------- float Average of log-likelihoods on testing groups """ d = deepcopy(data) # Remove nan voxels nan_idxs = np.where(np.isnan(d)) nan_idxs = list(set(nan_idxs[2])) d = np.delete(np.asarray(d), nan_idxs, axis=2) # Train and test event segmentation across groups group1 = d[split].mean(0) group2 = d[~split].mean(0) es = EventSegment(n_events).fit(group1) _, ll12 = es.find_events(group2) es = EventSegment(n_events).fit(group2) _, ll21 = es.find_events(group1) return (ll12 + ll21) / 2
def test_chains(): es = EventSegment(5, event_chains=np.array(['A', 'A', 'B', 'B', 'B'])) sample_data = np.array([[0, 0, 0], [1, 1, 1]]) with pytest.raises(RuntimeError): seg = es.fit(sample_data.T)[0] pytest.fail("Can't use fit() with event chains") es.set_event_patterns(np.array([[1, 1, 0, 0, 0], [0, 0, 1, 1, 1]])) seg = es.find_events(sample_data.T, 0.1)[0] ev = np.nonzero(seg > 0.99)[1] assert np.array_equal(ev, [2, 3, 4]),\ "Failed to fit with multiple chains"
def compute_fits_hmm(data: np.ndarray, k: int, mindist: int, type='HMM', y=None, t1=None, ind1=None, zs=False): if type == 'HMM': hmm = HMM(k) elif type == 'HMMsplit': hmm = HMM(k, split_merge=True) if zs == True: data = zscore(data, axis=0, ddof=1) hmm.fit(data) if y is None: tdata = data else: if zs == True: y = zscore(y, axis=0, ddof=1) tdata = y _, LL_HMM = hmm.find_events(tdata) hmm_bounds = np.insert(np.diff(np.argmax(hmm.segments_[0], axis=1)), 0, 0).astype(int) if t1 is None and ind1 is None: ind = np.triu(np.ones(tdata.shape[0], bool), mindist) z = GSBS._zscore(tdata) t = np.cov(z)[ind] else: ind = ind1 t = t1 stateseq = deltas_states(deltas=hmm_bounds)[:, None] diff, same, alldiff = (lambda c: (c == 1, c == 0, c > 0))(cdist( stateseq, stateseq, "cityblock")[ind]) WAC_HMM = np.mean(t[same]) - np.mean(t[alldiff]) tdist_HMM = 0 if sum(same) < 2 else ttest_ind( t[same], t[diff], equal_var=False)[0] return LL_HMM, WAC_HMM, tdist_HMM, hmm_bounds, t, ind
subj_id_train_inner = subj_id_all_inner[subj_id_train_inner] subj_id_test_inner = subj_id_all_inner[subj_id_test_inner] print("-Train:", subj_id_train_inner,"Validate:", subj_id_test_inner, ', now try different k...') D_train = D[:,:,subj_id_train_inner] #14 subjects to be trained D_val = D[:,:,subj_id_test_inner] #5 subjects for validation D_test = D[:,:,subj_id_test_outer] #held-out subject for knum in range(len(ks)): #loop through all possible K values #ev = brainiak.eventseg.event.EventSegment(ks[knum]) #set up HMM if splitm: #if running split_merge ev = EventSegment(ks[knum],split_merge=True) else: #if no split_merge ev = EventSegment(ks[knum]) ev.fit(D_train.mean(2).T) #fit to average pattern of training set segments,ll=ev.find_events(D_val.mean(2).T)#grab segments + log lik on average of validation set #print('log likelihood for k=%s is %s' %(ks[knum],ll)) #uncomment to see ll values loglik[subj_id_test_outer,jj,knum] = ll #store loglik value for this K all_ll[jj,:]=loglik[subj_id_test_outer,jj,:] best_ll[subj_id_test_outer,jj] = np.max(loglik[subj_id_test_outer,jj,:]) #best loglik! jj=jj+1 mean_all_ll=np.mean(all_ll,axis=0) #grab mean log lik fill3=mean_all_ll.argsort() #sort by log likelihood value fill3=fill3[len(ks)-1] #find best K print('Best K = %s' %fill3) #print best K best_k_subj[subj_id_test_outer]=ks[fill3] #store best k to assign for this sub if splitm: #if running split_merge ev = brainiak.eventseg.event.EventSegment(ks[fill3],split_merge=True) #re-train model w/ this best K else: #if no split_merge ev = brainiak.eventseg.event.EventSegment(ks[fill3]) #re-train model w/ this best K #ev.fit(D[:,:,np.arange(nS) != subj_id_test_outer].mean(2).T) #fit
What if we don't want to prespecify the number of events, but instead want to determine the number of events from the data? One way to determine the best number of events is to fit the model on a training set and then test the model fit on independent subjects. k_array = np.arange(20, 61, 10) test_ll = np.zeros(len(k_array)) for i, k in enumerate(k_array): print('Trying %d events' % k) print(' Fitting model on training subjects...') movie_train = np.mean(movie[:8], axis = 0) movie_HMM = EventSegment(k) movie_HMM.fit(movie_train) print(' Testing model fit on held-out subjects...') movie_test = np.mean(movie[8:], axis = 0) _, test_ll[i] = movie_HMM.find_events(movie_test) plt.plot(k_array, test_ll) plt.xlabel('Number of events') plt.ylabel('Log-likelihood') movie_dur = nTRs * 1.5 # Data acquired every 1.5 seconds secax = plt.gca().secondary_xaxis('top', functions=(lambda x: movie_dur / x, lambda x: movie_dur / x)) secax.set_xlabel('Average event length (sec)') #### 1.3 Optimal segmentation with the HMM Since 40 events maximized the test log-likelihood, we'll generate two versions of HMM boundaries using 40 events. In addition to the "vanilla" HMM, we'll run an HMM with more flexibility during fitting (allowing for split-merge operations). This is slower (and so should usually only be used for generating a final segmentation), but can produce better fits if events are very uneven in duration. We will use these segmentations below for comparison with an alternative event segmentation method (GSBS) and with human labeled event boundaries. print('Fitting HMM with 40 events...') HMM40 = EventSegment(n_events = 40) HMM40.fit(movie_group)