def dHdt(ps,Mut=None,dt=10**-6): """Return instantaneous rate of change of entropy""" L = int(log(len(ps),4)) if Mut is None: Mut = mutation_matrix_ref(mu=1,w=L) psp = mutate(ps,dt,Mut) return (h(psp)-h(ps))/dt
def flow_to_h(hf,p0,tol=10**-2,eta=10**-6): """Given initial point p0, pursue gradient flow until reaching final entropy hf""" p = np.copy(p0) hp = h(p0) iterations = 0 while abs(hp-hf) > tol: g = grad(p) p += g*(hf-hp) * eta hp = h(p) # if iterations % 1000 == 0: # print "p:",p,"g:",g*eta,hp,np.linalg.norm(g*eta) iterations += 1 if np.any(np.isnan(p)): return None return p
def plot_h_vs_ic(L,sigmas=interpolate(0.1,10,100),max_h=None,M=None,xfunc=lambda ps:2*L): if max_h is None: print "generating samples" pss = [simplexify_sample(4**L,sigma=sigma) for sigma in tqdm(sigmas)] else: pss = [] while len(pss) < trials: ps = sample(L) if h(ps) < max_h: pss.append(ps) print len(pss) print "computing M" if M is None: M = marginalization_matrix(L) icq_s = map(lambda ps:ic(ps,M),tqdm(pss)) print "computing entropy" icp_s = map(lambda ps:2*L - h_np(ps),tqdm(pss)) # print "computing total mi" # mis = map(lambda ps:total_mi(ps,M),tqdm(pss)) # print "computing columnwise entropies" # hqs = map(lambda ps:psfm_entropy(ps,M),tqdm(pss)) # plt.scatter(hs,hqs) plt.scatter(icp_s,icq_s) #plt.plot([0,2*L],[2*L,0]) #plt.plot([0,2*L],[0,2*L]) # plt.plot([0,2],[0,4]) # plt.plot([0,2],[0,2*L]) # print pearsonr(ics,hs) # print spearmanr(ics,hs) plt.plot([0,2*L],[0,2*L]) plt.plot(*pl(lambda icp:L*icp+2*(L-L**2),[2*(L-1),2*L]),color='b') plt.xlabel("Distribution IC") plt.ylabel("PSFM IC") plt.title("Distribution vs. Columnwise IC, Length=%s" % L)
def _perform_error_estimation(self): # print('Performing error estimation...', end='\r') error_estimation_indices = communication.receive_list(self.cqc, self.sender_pkey) sender_key_part = communication.receive_binary_list(self.cqc, self.sender_pkey) num_errors = 0.0 key_part = [] for i in range(0, len(error_estimation_indices)): key_part.append(self.sifted_key[error_estimation_indices[i]]) if sender_key_part[i] != key_part[i]: num_errors += 1.0 communication.send_binary_list(self.cqc, self.sender, self.skey, key_part) self.error_estimation = num_errors / len(key_part) # print('B Performing error estimation... Done!') # print('B Error rate = {}'.format(self.error_estimation)) error_estimation_indices.sort() self.sifted_key = utils.remove_indices(self.sifted_key, error_estimation_indices) remaining_bits = len(self.sifted_key) min_entropy = remaining_bits * (1 - utils.h(self.error_estimation)) max_key = min_entropy - 2 * utils.log(1/self.security_param, 2) - 1 return self.n <= max_key
def hdist(desired_ent,n): """Return distribution on n outcomes with entropy <= h (bits)""" ent = n while(ent > desired_ent): ps = simplex_sample(n) ent = h(ps) return ps
def _perform_error_estimation(self): print('Performing error estimation...', end='\r') error_estimation_indices = [] key_part = [] for i in range(0, self.n): r = random.randint(0, len(self.sifted_key) - 1) while r in error_estimation_indices: r = random.randint(0, len(self.sifted_key) - 1) error_estimation_indices.append(r) key_part.append(self.sifted_key[r]) communication.send_message(self.cqc, self.receiver, self.skey, error_estimation_indices) communication.send_binary_list(self.cqc, self.receiver, self.skey, key_part) receiver_key_part = communication.receive_binary_list( self.cqc, self.receiver_pkey) num_errors = 0.0 for i in range(0, len(key_part)): if receiver_key_part[i] != key_part[i]: num_errors += 1.0 self.error_estimation = num_errors / len(key_part) print('Performing error estimation... Done!') print('Error rate = {}'.format(self.error_estimation)) error_estimation_indices.sort() self.sifted_key = utils.remove_indices(self.sifted_key, error_estimation_indices) remaining_bits = len(self.sifted_key) min_entropy = remaining_bits * (1 - utils.h(self.error_estimation)) max_key = min_entropy - 2 * utils.log(1 / self.security_param, 2) - 1 return self.n <= max_key
def stationary_stat_neglect_fg(matrix,n,Ne,T,samples=1000): acc = 0 Z = 0 ws = [] for sample in trange(samples): motif = sample_motif_neglect_fg(matrix,n,Ne) log_f = log_fitness(matrix,motif,G) log_q = dsample_motif_neglect_fg(matrix,motif,Ne) t = T(motif) w = exp(log_f - log_q) acc += t * w Z += w print acc/Z ws.append(w) print "entropy of samples:",h(normalize(ws)), log(samples) - h(normalize(ws)) return acc/Z
def entropy_from_ps(ps, N): K = len(ps) ns = [0] * K xs = range(K) for i in xrange(N): j = inverse_cdf_sample(xs, ps) ns[j] += 1 return h([n/float(N) for n in ns])
def rvector(beta, K=4): #trials = 0 while True: #trials += 1 p = simplex_sample(K) if random.random() < exp(-beta*h(p)): #print "acceptance rate:",1/float(trials) return p
def rvector(beta, K=4): #trials = 0 while True: #trials += 1 p = simplex_sample(K) if random.random() < exp(-beta * h(p)): #print "acceptance rate:",1/float(trials) return p
def mean_ic_from_eps(cs,n,L): """compute approximate information content of motif with c mismatches in each site""" #"""Should depend only on permutations, not on substitutions""" # cs counts number of MISMATCHES in each site p_mismatch = 1.0/(n*L)*sum(cs) p_match = 1 - p_mismatch col_ent = h([p_match,p_mismatch/3.0,p_mismatch/3.0,p_mismatch/3.0]) return L*(2 - col_ent)
def entropy_from_ps(ps, N): K = len(ps) ns = [0] * K xs = range(K) for i in xrange(N): j = inverse_cdf_sample(xs, ps) ns[j] += 1 return h([n / float(N) for n in ns])
def astar(draw, grid, start, end): count = 0 frontier = PriorityQueue() frontier.put((0, count, start)) frontier_hash = {start} g_score = {spot: float("inf") for row in grid for spot in row} g_score[start] = 0 f_score = {spot: float("inf") for row in grid for spot in row} f_score[start] = utils.h(start.get_pos(), end.get_pos()) came_from = {} while not frontier.empty(): for event in pygame.event.get(): if utils.is_quit(event): pygame.quit() current = frontier.get()[2] frontier_hash.remove(current) if current == end: utils.reconstruct_path(came_from, end, draw) start.make_start() end.make_end() return True for neighbor in current.neighbors: temp_g_score = g_score[current] + 1 if temp_g_score < g_score[neighbor]: came_from[neighbor] = current g_score[neighbor] = temp_g_score f_score[neighbor] = temp_g_score + utils.h( neighbor.get_pos(), end.get_pos()) if neighbor not in frontier_hash: count += 1 frontier.put((f_score[neighbor], count, neighbor)) frontier_hash.add(neighbor) neighbor.make_visited() draw() if current != start: current.make_expanded() return False
def mean_ic_from_eps(cs, n, L): """compute approximate information content of motif with c mismatches in each site""" #"""Should depend only on permutations, not on substitutions""" # cs counts number of MISMATCHES in each site p_mismatch = 1.0 / (n * L) * sum(cs) p_match = 1 - p_mismatch col_ent = h( [p_match, p_mismatch / 3.0, p_mismatch / 3.0, p_mismatch / 3.0]) return L * (2 - col_ent)
def minimize_dHdt_test(): L = 4 K = int(4**L) for i in range(10): ps = np.array(simplex_sample(K)) print "marginalizing" qs = qs_from_psfm(marginalize(ps)) print "sampling wtih given entropy" rs = sample_with_given_entropy(K,h(qs),tol_factor=10**-6) print "minimizing" qsp = minimize_dHdt(qs) rsp = minimize_dHdt(rs) print "qs:",dHdt(qs),dHdt(qsp) print "rs:",dHdt(rs),dHdt(rsp)
def minimize_dHdt(ps): converged = False eta = 10**-10 dt = 10**-6 hist = [] while not converged: print dHdt(ps),sum(ps),h(ps),min(ps) bvs = entropic_isocontour(ps) dHp = dHdt(ps) grad = [(dHdt(ps + bv*dt)-dHp)/dt for bv in bvs] dp = sum(bv*g for (bv,g) in zip(bvs,grad)) print "sum dp:",sum(dp) ps = ps + -dp*eta if abs(sum(dp)) < 10**-100: converged = True return ps
def plot_h_vs_ic(num_cols,trials,max_h=None): if max_h is None: pss = [sample(num_cols) for i in tqdm(range(trials))] else: pss = [] while len(pss) < trials: ps = sample(num_cols) if h(ps) < max_h: pss.append(ps) print len(pss) ics = map(ic,tqdm(pss)) hs = map(h,tqdm(pss)) plt.scatter(hs,ics) plt.plot([0,2*num_cols],[2*num_cols,0]) print pearsonr(ics,hs) plt.xlabel("Entropy") plt.ylabel("IC")
def predict_ic(matrix, mu, Ne, N=100): nu = Ne - 1 ep_min, ep_max, L = sum(map(min, matrix)), sum(map(max, matrix)), len(matrix) site_sigma = site_sigma_from_matrix(matrix) density = lambda ep: (1 / (1 + exp(ep - mu)))**(Ne - 1) * dnorm( ep, 0, site_sigma) * (ep_min <= ep <= ep_max) d_density = lambda ep: ep / site_sigma**2 + nu / (1 + exp(mu - ep)) mode = bisect_interval(d_density, -100, 100) if mode < ep_min: mode = ep_min dmode = density(mode) # calculate mean epsilon via rejection sampling eps = [] while len(eps) < N: ep = random.random() * (ep_max - ep_min) + ep_min if random.random() < density(ep) / dmode: eps.append(ep) #return eps des_mean_ep = mean(eps) des_mean_ep_analytic = integrate.quad(lambda ep: ep * density(ep), ep_min, ep_max) # print "des_means:", des_mean_ep, des_mean_ep_analytic # print "min ep: %s max_ep: %s des_mean_ep: %s" % (ep_min, ep_max, des_mean_ep) def mean_ep(lamb): try: psfm = psfm_from_matrix(matrix, lamb=lamb) return sum([ ep * p for (mat_row, psfm_row) in zip(matrix, psfm) for (ep, p) in zip(mat_row, psfm_row) ]) except: print matrix, lamb raise Exception try: lamb = bisect_interval(lambda l: mean_ep(l) - des_mean_ep, -20, 20) except: print matrix, mu, Ne raise Exception tilted_psfm = psfm_from_matrix(matrix, lamb) return sum([2 - h(col) for col in tilted_psfm])
def plot_ic_vs_pairwise_mi(L,sigmas=interpolate(0.01,10,100),max_h=None,M=None): if max_h is None: print "generating samples" pss = [simplexify_sample(4**L,sigma=sigma) for sigma in sigmas] else: pss = [] while len(pss) < trials: ps = sample(L) if h(ps) < max_h: pss.append(ps) print len(pss) print "computing M" if M is None: M = marginalization_matrix(L) print "computing ic" total_ics = map(lambda ps:total_ic(ps),tqdm(pss)) psfm_ics = map(lambda ps:2*L-psfm_entropy(ps,M),tqdm(pss)) print "computing pairwise mi" pair_mis = map(lambda ps:pairwise_mi_ref(ps,M),tqdm(pss)) print "computing total mi" total_mis = map(lambda ps:total_mi(ps,M),tqdm(pss)) # print "computing columnwise entropies" # plt.scatter(hs,hqs) plt.scatter(psfm_ics,pair_mis,color='g',label='pair MI') plt.scatter(psfm_ics,total_mis,label='total MI') plt.scatter(psfm_ics,total_ics,color='r',label='Total IC') #plt.plot([0,2*L],[2*L,0]) #plt.plot([0,2*L],[0,2*L]) # plt.plot([0,2],[0,4]) # plt.plot([0,2],[0,2*L]) # print pearsonr(ics,hs) # print spearmanr(ics,hs) # plt.plot([0,2*L],[0,2*L]) # plt.plot(*pl(lambda icp:L*icp+2*(L-L**2),[2*(L-1),2*L])) plt.xlabel("PSFM IC") plt.ylabel("Bits") plt.title("Length=%s" % L) plt.legend()
def entropy_hessian_experiment(): L = 3 K = int(4**L) eps = 10**-6 ps = np.array(simplex_sample(K)) qs = qs_from_psfm(marginalize(ps)) rs = sample_with_given_entropy(K,h(qs),tol_factor=10**-2) bvs_p = entropic_isocontour(ps) bvs_q = entropic_isocontour(qs) bvs_r = entropic_isocontour(rs) dHp = dHdt(ps) dHps = [dHdt(normalize1(ps + bvp*eps)) for bvp in bvs_p] dHq = dHdt(qs) dHqs = [dHdt(normalize1(qs + bvq*eps)) for bvq in bvs_q] dHr = dHdt(rs) dHrs = [dHdt(normalize1(rs + bvr*eps)) for bvr in bvs_r] plt.plot(dHps,color='r') plt.plot([dHp]*(K-2),color='r',linestyle='--') plt.plot(dHqs,color='g') plt.plot([dHq]*(K-2),color='g',linestyle='--') plt.plot(dHrs,color='b') plt.plot([dHr]*(K-2),color='b',linestyle='--') plt.show()
def predict_ic(matrix, mu, Ne, N=100): nu = Ne - 1 ep_min, ep_max, L = sum(map(min,matrix)), sum(map(max,matrix)), len(matrix) site_sigma = site_sigma_from_matrix(matrix) density = lambda ep:(1/(1+exp(ep-mu)))**(Ne-1) * dnorm(ep,0,site_sigma)*(ep_min <= ep <= ep_max) d_density = lambda ep:ep/site_sigma**2 + nu/(1+exp(mu-ep)) mode = bisect_interval(d_density, -100, 100) if mode < ep_min: mode = ep_min dmode = density(mode) # calculate mean epsilon via rejection sampling eps = [] while len(eps) < N: ep = random.random() * (ep_max - ep_min) + ep_min if random.random() < density(ep)/dmode: eps.append(ep) #return eps des_mean_ep = mean(eps) des_mean_ep_analytic = integrate.quad(lambda ep:ep*density(ep), ep_min, ep_max) # print "des_means:", des_mean_ep, des_mean_ep_analytic # print "min ep: %s max_ep: %s des_mean_ep: %s" % (ep_min, ep_max, des_mean_ep) def mean_ep(lamb): try: psfm = psfm_from_matrix(matrix, lamb=lamb) return sum([ep * p for (mat_row, psfm_row) in zip(matrix, psfm) for (ep, p) in zip(mat_row, psfm_row)]) except: print matrix, lamb raise Exception try: lamb = bisect_interval(lambda l:mean_ep(l) - des_mean_ep, -20, 20) except: print matrix, mu, Ne raise Exception tilted_psfm = psfm_from_matrix(matrix, lamb) return sum([2 - h(col) for col in tilted_psfm])
def f(beta): return (2 - (h(rvector(beta)) + K))*L
def resample_pop(): re_obs = Counter(bs(sample)).values() return 2**h([v/N for v in re_obs])
def ic(ps,M=None): psfm = marginalize(ps,M) return sum(2-h(col) for col in psfm)
def psfm_entropy(ps,M=None): psfm = marginalize(ps,M) return sum(h(col) for col in psfm)
def ic(ps): psfm = marginalize(ps) return sum(2-h(col) for col in psfm)
def rec_h(rec): p = np.sum(rec)/float(len(rec)) return h([p,1-p])
def dHdt2(ps): L = num_cols_from_vector(ps) fs = ps - norm_lap(ps) term1 = cross_h(fs,ps) term2 = h(ps) return 3*L*(term1 - term2)
def back_propagate(theta1, theta2, train_images, train_labels, nclass, alpha=0.001, lambdaa=0.0007, max_iter=50, act='sig', batch_size=32, logging=1): """ Method of updating the weights in NN Model by taking gradients of theta using cost function thata = theta - f('theta) Mini-batch gradient descent, applied to get the gradient of the theta. Here updation of weights use momentum factor(gamma) so as to approach global minima faster Core of ANN, BackProp.. @ Parameters: ------------- test_images: np.array Contains the test_images whose labels need to be predicted test_labels: np.array Contains the labels(1/0) corresponding to selected images theta1: np.array Contains the trained theta weights corresponding to input->hidden layer theta2: np.array Contains the trained theta weights corresponding to hidden->output layer train_images: np.array Contains the train_images used to learn the weights of networks train_labels: np.array Contains the labels(1/0) corresponding to train_images nclass: int No of unique class present in the training dataset alpha: float Learning rate, rate at which each gradient update take place lambdaa: float Regularization term which penalizes the cost function max_iter: int No of epochs to be performed on data to learn the weights act: str Activation function which is applied to the neurons in forward propagation batch_size: int No of images,labels to be fetched from overall data at each iterations for updation of weights logging: int Steps at which logs are displayed or recorded @ Returns: ---------- parameters: dict trained theta1,theta2 and per epoch Loss values """ # Used to store theta1 & theta2 parameters = {} # Momentum Factor gamma = 0.9 # Intial dtheta values used for # momentum dtheta1, dtheta2 = 0.0, 0.0 # One-Hot labelling the labels of data one_hot = output_encoding(train_labels, nclass) # Used to store best theta1 and theta2 values # whose error was least in whole epochs best_theta1, best_theta2 = (np.zeros((theta1.shape[0], theta1.shape[1])), np.zeros((theta2.shape[0], theta2.shape[1]))) # Store the value of cost in each epochs cost_list = [] # Global Min Error term err = 100.0 for epoch in np.arange(0, max_iter): # Used to print results of result summary k = 0 print print('\nOverall Min. Error rate : ' + str(err)) print # Softmax in Final Layer for batchX, batchY in get_batch(train_images, one_hot, batch_size): m, n = batchX.shape a2 = h(theta1, batchX, act) a2 = np.insert(a2, 0, 1, axis=0) a3 = h(theta2, a2.T, func='softmax') eps = alpha / float(m) # Error in Hidden and Output Layer delta3 = (a3 - batchY) * derivative(a3, 'none') delta2 = ((theta2.T).dot(delta3)) * derivative(a2, act) # Gradient of Theta Matrices ktheta1 = np.dot(delta2[1:, :], batchX) ktheta2 = np.dot(delta3, a2.T) # Momemtum Part to Accelerate the Learning Rate dtheta1 = eps * (ktheta1 + lambdaa * theta1) + gamma * dtheta1 dtheta2 = eps * (ktheta2 + lambdaa * theta2) + gamma * dtheta2 theta1 = theta1 - dtheta1 theta2 = theta2 - dtheta2 # Cost Per Batch iteration cost_epoch = cost(a3, batchY, { 'Theta1': theta1, 'Theta2': theta2 }, lambdaa) cost_list.append(cost_epoch) # Summary of Back Prop if (k % LOGGING_STEPS == 0): accuracy = model_score({ 'Theta1': theta1, 'Theta2': theta2 }, train_images, train_labels, act) error = 100.0 - accuracy # Error Updation if LEss Error is Discovered if (error < err): err = error # Store the best theta of least error best_theta1 = theta1 best_theta2 = theta2 # Info of Learning of NN print("Epoch " + str(epoch + 1) + " in " + str(k + 1) + " iter" + " | " "Train Error rate: " + str(error) + "%" + " | Batch loss: " + str(cost_epoch)) k = k + 1 parameters = { 'Theta1': best_theta1, 'Theta2': best_theta2, 'Loss': cost_list } return parameters
def entropy_from_partition(part): Z = float(sum(part)) ps = [c/Z for c in part] return h(ps)
def f(beta): return (2 - (h(rvector(beta)) + K)) * L
def mh_sample(K,iterations=50000): p0 = np.array(simplex_sample(K)) f = lambda p:1/h(p)**K proposal = lambda p:propose(p,sigma=1) return mh(f,proposal,p0,iterations=iterations)
def entropy_from_counts(counts): N = float(sum(counts)) ps = [c / N for c in counts] return h(ps)
def ic_from_matrix(matrix): psfm = psfm_from_matrix(matrix) return sum(2 - h(col) for col in psfm)