def train_from_samples(self, patches): #from pnet.latent_bernoulli_mm import LatentBernoulliMM from pnet.bernoullimm import BernoulliMM min_prob = self._settings.get('min_prob', 0.01) support_mask = self._settings.get('support_mask') if support_mask is not None: kp_patches = patches[:,support_mask] else: kp_patches = patches.reshape((patches.shape[0], -1, patches.shape[-1])) if self._settings.get('kp') == 'funky': patches = patches[:,::2,::2] # Only patches above the threshold print('patches', patches.shape) patches = patches[np.apply_over_axes(np.sum, patches.astype(np.int64), [1, 2, 3]).ravel() >= self._settings['threshold']] print('patches', patches.shape) flatpatches = kp_patches.reshape((kp_patches.shape[0], -1)) mm = BernoulliMM(n_components=self._num_parts, n_iter=10, tol=1e-15, n_init=1, random_state=self._settings.get('em_seed', 0), min_prob=min_prob, verbose=False) mm.fit(flatpatches) logprob, resp = mm.eval(flatpatches) comps = resp.argmax(-1) Hall = (mm.means_ * np.log(mm.means_) + (1 - mm.means_) * np.log(1 - mm.means_)) H = -Hall.mean(-1) if support_mask is not None: self._parts = 0.5 * np.ones((self._num_parts,) + patches.shape[1:]) self._parts[:,support_mask] = mm.means_.reshape((self._parts.shape[0], -1, self._parts.shape[-1])) else: self._parts = mm.means_.reshape((self._num_parts,)+patches.shape[1:]) self._weights = mm.weights_ # Calculate entropy of parts # Sort by entropy II = np.argsort(H) self._parts = self._parts[II] self._num_parts = II.shape[0] #self._train_info['entropy'] = H[II] return comps
def train_from_samples(self, patches,num_parts): #from pnet.latent_bernoulli_mm import LatentBernoulliMM from pnet.bernoullimm import BernoulliMM min_prob = self._settings.get('min_prob', 0.01) flatpatches = patches.reshape((patches.shape[0], -1)) parts = np.ones((num_parts,) + patches.shape[1:]) if 0: mm = BernoulliMM(n_components=num_parts, n_iter=20, tol=1e-15,n_init=2, random_state=0, min_prob=min_prob, verbose=False) print(mm.fit(flatpatches)) print('AIC', mm.aic(flatpatches)) print('BIC', mm.bic(flatpatches)) #import pdb; pdb.set_trace() parts = mm.means_.reshape((num_parts,)+patches.shape[1:]) #self._weights = mm.weights_ else: from pnet.bernoulli import em ret = em(flatpatches, num_parts,20,numpy_rng=self._settings.get('em_seed',0),verbose=True) parts = ret[1].reshape((num_parts,) + patches.shape[1:]) self._weights = np.arange(self._num_parts) #self._weights = mm.weights # Calculate entropy of parts Hall = (parts * np.log(parts) + (1 - parts) * np.log(1 - parts)) H = -np.apply_over_axes(np.mean, Hall, [1, 2, 3])[:,0,0,0] # Sort by entropy II = np.argsort(H) parts[:] = parts[II] #self._train_info['entropy'] = H[II] return parts
def train(self, X, Y): K = Y.max() + 1 mm_models = [] for k in xrange(K): Xk = X[Y == k] Xk = Xk.reshape((Xk.shape[0], -1)) mm = BernoulliMM(n_components=self._n_components, n_iter=10, n_init=1, random_state=0, min_prob=self._min_prob, blocksize=200) mm.fit(Xk) mm_models.append(mm.means_.reshape((self._n_components,)+X.shape[1:])) self._models = np.asarray(mm_models)
def train(self, X, Y, OriginalX = None): K = Y.max() + 1 mm_models = [] self._modelinstance = [] for k in range(K): Xk = X[Y == k] #import pdb ; pdb.set_trace() #print(Xk.shape) #print(X.shape) #print(Y.shape) Xk = Xk.reshape((Xk.shape[0], -1)) if self._mixture_type == "bernoullimm": mm = BernoulliMM(n_components=self._n_components, n_iter=10, n_init=1, random_state=0, min_prob=self._min_prob,blocksize = self._block_size) # else will assume it as gaussian mixture model else: from sklearn.mixture import GMM mm = GMM(n_components = self._n_components, n_iter = 20, n_init = 1, random_state=0, covariance_type = self._settings.get('covariance_type', 'diag'),) mm.fit(Xk) mm_models.append(mm.means_.reshape((self._n_components,)+X.shape[1:])) self._modelinstance.append(mm) self._models = np.asarray(mm_models)
def train(self, X, Y = None, OriginalX = None): if self._weights_file is not None and self._grouping_type!= 'KLDistance': weights = np.load(self._weights_file) self._getPoolMatrix(weights,X.shape[1:], X) elif self._grouping_type == 'rbm': print("running RBM") n_hidden = self._settings.get('n_hidden', 200) training_epochs = self._settings.get('epochs', 50) learning_rate = self._settings.get('rate', 0.1) seed = self._settings.get('rbm_seed',123) rbmModel = testRBM(X.reshape(X.shape[0],-1), learning_rate = learning_rate, training_epochs = training_epochs, n_hidden = n_hidden, randomSeed = seed) weights = rbmModel.W.get_value(borrow=True) if self._save_weights_file is not None: np.save(self._save_weights_file,weights) self._getPoolMatrix(weights,X.shape[1:], X) elif self._grouping_type == 'logReg': print("running Logistic Regression") numTraining = 9 * X.shape[0]//10 trainingDataSet = X.reshape(X.shape[0],-1)[0:numTraining] trainingDataLabel = Y[0:numTraining] validationDataSet = X.reshape(X.shape[0],-1)[numTraining:] validationDataLabel = Y[numTraining:] dataSize = trainingDataSet.shape[1] classifier = sgd_optimization_mnist(dataSize, trainingDataSet,trainingDataLabel,validationDataSet, validationDataLabel) weights = classifier.W.get_value(borrow=True) if self._save_weights_file is not None: np.save(self._save_weights_file, weights) self._getPoolMatrix(weights, X.shape[1:], X) elif self._grouping_type == 'mixture_model': #pass mixtureModel = BernoulliMM(n_components = 1000, n_iter = 100, n_init = 2, random_state = self._settings.get('em_seed', 0), min_prob = 0.005, joint=False,blocksize = 100) mixtureModel.fit(X.reshape((X.shape[0],-1))) print("mixtureModelweights") print(mixtureModel.weights_.shape) print(np.sum(mixtureModel.weights_)) weights = mixtureModel.means_.reshape((mixtureModel.n_components,-1)) #weights = np.log(weights/(1-weights)) weights = np.swapaxes(weights,0,1) #joint_probability = mixtureModel.joint_probability component_weights = mixtureModel.weights_ posterior = mixtureModel.posterior #print(np.sum(posterior,axis = 1)) #print("posterior") #print(posterior.shape) #print(joint_probability.shape) if self._save_weights_file is not None: #np.save(self._save_weights_file,weights) np.save(self._save_weights_file,weights) np.save("./modelWeights.npy",mixtureModel.weights_) np.save("./modelMeans.npy",mixtureModel.means_) np.save("./modelPosterior.npy", posterior) #plt.hist(mixtureModel.weights_) #plt.show() #weights = posterior #self._getPoolMatrix(weights, X.shape[1:], X) self._getPoolMatrixByKLDistance(weights, X.shape[1:], X) #self._getPoolMatrixByMutual(weights, X.shape[1:],joint_probability,component_weights) #TODO: write mixture model weights generating. elif self._grouping_type == 'KLDistance': distanceFile = np.load(self._weights_file) self._getPoolMatrixByDistance(distanceFile, X.shape[1:], X)
def train_from_samples(self, patches, original_patches): # from pnet.latent_bernoulli_mm import LatentBernoulliMM from pnet.bernoullimm import BernoulliMM print(patches.shape) min_prob = self._settings.get("min_prob", 0.01) # num_permutation = self._shifting_shape[0] * self._shifting_shape[1] # parts = np.ones((self._num_true_parts * num_permutation ,) + patches.shape[2:]) parts = np.ones((self._num_parts,) + patches[0].shape) d = np.prod(patches.shape[1:]) # print(d,num_permutation) if 0: # \permutation = np.empty((num_permutation, num_permutation * d),dtype = np.int_) for a in range(num_permutation): if a == 0: permutation[a] = np.arange(num_permutation * d) else: permutation[a] = np.roll(permutation[a - 1], d) flatpatches = patches.reshape((patches.shape[0], -1)) print(flatpatches.shape) if 0: mm = BernoulliMM( n_components=num_parts, n_iter=20, tol=1e-15, n_init=2, random_state=0, min_prob=min_prob, verbose=False ) print(mm.fit(flatpatches)) print("AIC", mm.aic(flatpatches)) print("BIC", mm.bic(flatpatches)) # import pdb; pdb.set_trace() parts = mm.means_.reshape((num_parts,) + patches.shape[1:]) # self._weights = mm.weights_ elif 0: from pnet.bernoulli import em print("before EM") ret = em( flatpatches, self._num_true_parts, 10, mu_truncation=min_prob, permutation=permutation, numpy_rng=self._settings.get("em_seed", 0), verbose=True, ) comps = ret[3] parts = ret[1].reshape((self._num_true_parts * num_permutation,) + patches.shape[2:]) self._weights = np.arange(self._num_parts) else: rng = np.random.RandomState(self._settings.get("em_seed", 0)) from pnet.latentShiftEM import LatentShiftEM # from latentShiftEM import latentShiftEM result = LatentShiftEM( flatpatches, num_mixture_component=self._num_parts, parts_shape=(self._part_shape[0], self._part_shape[1], 8), region_shape=(self._sample_shape[1], self._sample_shape[1], 8), shifting_shape=self._shifting_shape, max_num_iteration=25, loglike_tolerance=1e-3, mu_truncation=(1, 1), additional_mu=None, permutation=None, numpy_rng=rng, verbose=True, ) comps = result[3] print(comps.shape) print(original_patches.shape) print(result[1].shape) parts = result[1].reshape((self._num_parts, self._part_shape[0], self._part_shape[1], 8)) self._bkg_probability = result[4] self._parts = parts print(comps[:50, 0]) print(comps[:50, 1]) self._visparts = np.asarray( [original_patches[comps[:, 0] == k, comps[comps[:, 0] == k][:, 1]].mean(0) for k in range(self._num_parts)] ) print(self._visparts.shape) import amitgroup.plot as gr gr.images(self._visparts, zero_to_one=False, show=False, vmin=0, vmax=1, fileName="moduleShiftingParts1.png") return parts
def _train_inner(self, flatpatches, shape, hier, depth=0): min_prob = self._settings.get('min_prob', 0.01) #if len(flatpatches) < 5: ## Put all of them in both #flatpatches.mean() mm = BernoulliMM(n_components=self._num_parts_per_layer, n_iter=20, tol=1e-15, n_init=5, random_state=self._settings.get('em_seed', 0), #params='m', min_prob=min_prob) mm.fit(flatpatches) logprob, resp = mm.eval(flatpatches) comps = resp.argmax(-1) counts = np.bincount(comps, minlength=self._num_parts_per_layer) if 0: from scipy.stats import scoreatpercentile lower0 = scoreatpercentile(resp[...,0].ravel(), 50) lower1 = scoreatpercentile(resp[...,1].ravel(), 50) lows = [lower0, lower1] hier_flatpatches = [flatpatches[resp[...,m] >= lows[m]] for m in xrange(self._num_parts_per_layer)] # Reset the means for m in xrange(self._num_parts_per_layer): mm.means_[m] = np.clip(hier_flatpatches[m].mean(0), min_prob, 1 - min_prob) else: hier_flatpatches = [flatpatches[comps == m] for m in xrange(self._num_parts_per_layer)] #if depth == 0: #print('counts', counts) #print(depth, 'hier_flatpatches', map(np.shape, hier_flatpatches)) all_parts = [] all_weights = [] all_counts = [] all_hierarchies = [] pp = mm.means_.reshape((self._num_parts_per_layer,) + shape) hier[depth].append(pp) if depth+1 < self._depth: # Iterate for m in xrange(self._num_parts_per_layer): parts, weights, counts0 = self._train_inner(hier_flatpatches[m], shape, hier, depth=depth+1) all_parts.append(parts) all_weights.append(weights) all_counts.append(counts0) #all_hierarchies.append(hierarchy_parts) flat_parts = np.concatenate(all_parts, axis=0) all_weights = np.concatenate(all_weights, axis=0) all_counts = np.concatenate(all_counts, axis=0) return flat_parts, all_weights, all_counts#, [pp] + all_hierarchies else: parts = mm.means_ weights = mm.weights_ #all_parts.reshape((self._num_parts,)+patches.shape[1:]) return parts, weights, counts
def train_from_samples(self, patches): #from pnet.latent_bernoulli_mm import LatentBernoulliMM min_prob = self._settings.get('min_prob', 0.01) kp_patches = patches.reshape((patches.shape[0], -1, patches.shape[-1])) flatpatches = kp_patches.reshape((kp_patches.shape[0], -1)) hier = [[] for _ in xrange(self._depth)] all_parts, all_weights, counts = self._train_inner(flatpatches, patches.shape[1:], hier) if 0: # Post-training # Warm-up the EM with the leaves F = self._num_parts_per_layer ** self._depth mm = BernoulliMM(n_components=F, n_iter=20, tol=1e-15, n_init=1, random_state=100+self._settings.get('em_seed', 0), init_params='', #params='m', min_prob=min_prob) mm.means_ = all_parts mm.weights_ = counts / counts.sum() mm.fit(flatpatches) logprob, resp = mm.eval(flatpatches) comps = resp.argmax(-1) #for d in xrange(self._depth): #import pdb; pdb.set_trace() for d in reversed(xrange(len(hier))): for j in xrange(len(hier[d])): hier_dj = hier[d][j] for k in xrange(len(hier_dj)): if d == len(hier) - 1: hier[d][j][k] = mm.means_[j*self._num_parts_per_layer + k].reshape(patches.shape[1:]) else: hier[d][j][k] = hier[d+1][j*self._num_parts_per_layer + k].mean(0) #hier[d][j][k] = [flatpatches[comps == f].mean(0).reshape((-1,) + patches.shape[1:]) #else: #hier[d][j][k] += # Now, update the entire tree with these new results if 0: def pprint(x): if isinstance(x, list): return "[" + ", ".join(map(pprint, x)) + "]" elif isinstance(x, np.ndarray): #return "array{}".format(x.shape) return 'A' #print(pprint(parts_hierarchy)) print(pprint(hier)) #import pdb; pdb.set_trace() print(all_parts.shape) self._parts = all_parts.reshape((self._num_parts,)+patches.shape[1:]) self._weights = all_weights self._counts = counts #self._parts_hierarchy = parts_hierarchy self._hier = hier self._flathier = np.asarray(sum(hier, [])) from scipy.special import logit if self._num_parts_per_layer == 2: self._w = logit(self._flathier[:,1]) - logit(self._flathier[:,0]) self._constant_terms = np.apply_over_axes(np.sum, np.log(1 - self._flathier[:,1]) - np.log(1 - self._flathier[:,0]), [1, 2, 3])[:,0,0,0] if 0: # Train it again by initializing with this mm = BernoulliMM(n_components=self._num_parts, n_iter=20, tol=1e-15, n_init=1, init_params='w', random_state=self._settings.get('em_seed', 0), min_prob=min_prob) mm.means_ = self._parts.reshape((self._num_parts, -1)) mm.fit(flatpatches) self._parts = mm.means_.reshape((self._num_parts,)+patches.shape[1:]) self._weights = mm.weights_ Hall = (self._parts * np.log(self._parts) + (1 - self._parts) * np.log(1 - self._parts)) H = -np.apply_over_axes(np.mean, Hall, [1, 2, 3]).ravel() self._train_info['entropy'] = H
def train_from_samples(self, patches,patches_original): #from pnet.latent_bernoulli_mm import LatentBernoulliMM from pnet.bernoullimm import BernoulliMM min_prob = self._settings.get('min_prob', 0.01) flatpatches = patches.reshape((patches.shape[0], -1)) if 1: mm = BernoulliMM(n_components=self._num_parts, n_iter=20, tol=1e-15,n_init=2, random_state=self._settings.get('em_seed',0), min_prob=min_prob, verbose=False) mm.fit(flatpatches) print(mm.fit(flatpatches)) #print('AIC', mm.aic(flatpatches)) #print('BIC', mm.bic(flatpatches)) if 0: # Draw samples #size = (20, 20) import gv import os N = 10000 D = np.prod(self._part_shape) size = (20, 20) grid = gv.plot.ImageGrid(size[0], size[1], self._part_shape) samp = mm.sample(n_samples=np.prod(size)).reshape((-1,) + self._part_shape) samples = mm.sample(n_samples=N).reshape((-1,) + self._part_shape) print('samples', samples.shape) types = np.asarray(list(gv.multirange(*[2]*D))).reshape((-1,) + self._part_shape) th = np.clip(types, 0.01, 0.99) t = th[:,np.newaxis] x = samples[np.newaxis] llh0 = x * np.log(t) + (1 - x) * np.log(1 - t) counts0 = np.bincount(np.argmax(llh0.sum(-1).sum(-1), 0), minlength=th.shape[0]) x1 = patches[np.newaxis,...,0] llh1 = x1 * np.log(t) + (1 - x1) * np.log(1 - t) counts1 = np.bincount(np.argmax(llh1.sum(-1).sum(-1), 0), minlength=th.shape[0]) #import pdb; pdb.set_trace() w0 = counts0 / counts0.sum() w1 = counts1 / counts1.sum() print('w0', w0) print('w1', w1) #import pdb; pdb.set_trace() #import pdb; pdb.set_trace() self._parts = mm.means_.reshape((self._num_parts,)+patches.shape[1:]) self._weights = mm.weights_ else: #mm = ag.stats.BernoulliMixture(self._num_parts, flatpatches, max_iter=2000) #mm.run_EM(1e-6, min_probability=min_prob) #self._parts = mm.templates.reshape((self._num_parts,)+patches.shape[1:]) #self._weights = mm.weights from pnet.bernoulli import em ret = em(flatpatches, self._num_parts,20,numpy_rng=self._settings.get('em_seed',0),verbose=True) self._parts = ret[1].reshape((self._num_parts,) + patches.shape[1:]) self._weights = np.arange(self._num_parts) if 0: predictedGroups = mm.predict(flatpatches) # Calculate entropy of parts Hall = (self._parts * np.log(self._parts) + (1 - self._parts) * np.log(1 - self._parts)) H = -np.apply_over_axes(np.mean, Hall, [1, 2, 3])[:,0,0,0]
def train_from_samples(self, patches): #from pnet.latent_bernoulli_mm import LatentBernoulliMM min_prob = self._settings.get('min_prob', 0.01) kp_patches = patches.reshape((patches.shape[0], -1, patches.shape[-1])) flatpatches = kp_patches.reshape((kp_patches.shape[0], -1)) q = [] models = [] constant_terms = [] constant_terms_unsummed = [] tree = []# -123*np.ones((1000, 2), dtype=np.int64) cur_pos = 0 s = 0 cur_part_id = 0 q.insert(0, (cur_pos, 0, flatpatches)) s += 1 #cur_pos += 1 while q: p, depth, x = q.pop() model = np.clip(np.mean(x, 0), min_prob, 1 - min_prob) def entropy(x): return -(x * np.log2(x) + (1 - x) * np.log2(1 - x)) H = np.mean(entropy(model)) sc = self._settings.get('split_criterion', 'H') if len(x) < self._settings.get('min_samples_per_part', 20) or depth >= self._max_depth or \ (sc == 'H' and H < self._settings.get('split_entropy', 0.30)): #tree[p,0] = -1 #tree[p,1] = cur_part_id tree.append((-1, cur_part_id)) cur_part_id += 1 else: mm = BernoulliMM(n_components=self._num_parts_per_layer, n_iter=self._settings.get('n_iter', 8), tol=1e-15, n_init=self._settings.get('n_init', 1), # May improve a bit to increase this random_state=self._settings.get('em_seed', 0), #params='m', min_prob=min_prob) mm.fit(x[:self._settings.get('traing_limit')]) logprob, resp = mm.eval(x) comps = resp.argmax(-1) w = logit(mm.means_[1]) - logit(mm.means_[0]) Hafter = np.mean(entropy(mm.means_[0])) * mm.weights_[0] + np.mean(entropy(mm.means_[1])) * mm.weights_[1] IG = H - Hafter if sc == 'IG' and IG < self._settings.get('min_information_gain', 0.05): tree.append((-1, cur_part_id)) cur_part_id += 1 else: tree.append((len(models), s)) K_unsummed = np.log((1 - mm.means_[1]) / (1 - mm.means_[0])) K = np.sum(K_unsummed) models.append(w) constant_terms.append(K) constant_terms_unsummed.append(K_unsummed) #tree[p,1] = s q.insert(0, (s, depth+1, x[comps == 0])) #cur_pos += 1 q.insert(0, (s+1, depth+1, x[comps == 1])) #cur_pos += 1 s += 2 shape = (len(models),) + patches.shape[1:] weights = np.asarray(models).reshape(shape) constant_terms = np.asarray(constant_terms) constant_terms_unsummed = np.asarray(constant_terms_unsummed).reshape(shape) tree = np.asarray(tree, dtype=np.int64) self._tree = tree self._num_parts = cur_part_id #print('num_parts', self._num_parts) self._w = weights self._constant_terms = constant_terms supp_radius = self._settings.get('keypoint_suppress_radius', 0) if supp_radius > 0: NW = self._w.shape[0] max_indices = self._settings.get('keypoint_max', 1000) keypoints = np.zeros((NW, max_indices, 3), dtype=np.int64) kp_constant_terms = np.zeros(NW) num_keypoints = np.zeros(NW, dtype=np.int64) from gv.keypoints import get_key_points for k in xrange(NW): kps = get_key_points(self._w[k], suppress_radius=supp_radius, max_indices=max_indices) NK = len(kps) num_keypoints[k] = NK keypoints[k,:NK] = kps for kp in kps: kp_constant_terms[k] += constant_terms_unsummed[k,kp[0],kp[1],kp[2]] self._keypoints = keypoints self._num_keypoints = num_keypoints self._keypoint_constant_terms = kp_constant_terms