def find_convex_hull(X, num_iter, num_points=None): """ if num_points is set to None, find_convex_hull will return all the points in the convex hull (that have been found) sorted according to their sharpness. Otherwise, it will return the N-sharpest points. """ (N, D) = X.shape if (num_points == None): num_points = N # randomly choose 'num_iter' direction on the unit sphere. # find the maximal point in the chosen direction, and add 1 to its counter. # only points on the convex hull will be hit, and 'sharp' corners will # have more hits than 'smooth' corners. hits = p.zeros((N, 1)) for j in xrange(num_iter): a = p.randn(D) a = a / p.norm(a) i = p.dot(X, a).argmax() hits[i] += 1 # don't take points with 0 hits num_points = min(num_points, sum(p.find(hits))) # the indices of the n-best points o = list(p.argsort(hits, 0)[xrange(-1, -(num_points + 1), -1)].flat) return X[o, :]
def shiftgrid(lon0,datain,lonsin,start=True): """ shift global lat/lon grid east or west. assumes wraparound (or cyclic point) is included. lon0: starting longitude for shifted grid (ending longitude if start=False). lon0 must be on input grid (with the range of lonsin). datain: original data. lonsin: original longitudes. start[True]: if True, lon0 represents he starting longitude of the new grid. if False, lon0 is the ending longitude. returns dataout,lonsout (data and longitudes on shifted grid). """ if pylab.fabs(lonsin[-1]-lonsin[0]-360.) > 1.e-4: raise ValueError, 'cyclic point not included' if lon0 < lonsin[0] or lon0 > lonsin[-1]: raise ValueError, 'lon0 outside of range of lonsin' i0 = pylab.argsort(pylab.fabs(lonsin-lon0))[0] dataout = pylab.zeros(datain.shape,datain.typecode()) lonsout = pylab.zeros(lonsin.shape,lonsin.typecode()) if start: lonsout[0:len(lonsin)-i0] = lonsin[i0:] else: lonsout[0:len(lonsin)-i0] = lonsin[i0:]-360. dataout[:,0:len(lonsin)-i0] = datain[:,i0:] if start: lonsout[len(lonsin)-i0:] = lonsin[1:i0+1]+360. else: lonsout[len(lonsin)-i0:] = lonsin[1:i0+1] dataout[:,len(lonsin)-i0:] = datain[:,1:i0+1] return dataout,lonsout
def classify(self, x, k): d = self.X - tile(x.reshape(self.n, 1), self.N) dsq = sum(d * d, 0) minindex = argmin(dsq) temp = argsort(dsq) ### Custom code starting here ### # Save the data for each class around the point in a array score = [0, 0, 0] # With the help of k surrounding points score each class for x in range(0, k): if ((self.c[temp[x]]) == 1.0): score[0] += 1 elif ((self.c[temp[x]]) == 2.0): score[1] += 1 elif ((self.c[temp[x]]) == 3.0): score[2] += 1 # Check to which class the point is classified if (score[0] > score[1] and score[0] > score[2]): return 1.0 elif (score[1] > score[2]): return 2.0 # If there are points with the same value, assign the class of the nearest neighbour. elif (score[0] == score[1] and score[0] == score[2]): return self.c[minindex] else: return 3.0
def rank_by_distance_bhatt(self, qkeys, ikeys, rkeys, dists): """ :: Reduce timbre-channel distances to ranks list by ground-truth key indices Bhattacharyya distance on timbre-channel probabilities and Kullback distances """ # timbre-channel search using pre-computed distances ranks_list = [] t_keys, t_lens = self.get_adb_lists(0) rdists=pylab.ones(len(t_keys))*float('inf') qk = self._get_probs_tc(qkeys) for i in range(len(ikeys[0])): # number of include keys ikey=[] dk = pylab.zeros(self.timbre_channels) for t_chan in range(self.timbre_channels): # timbre channels ikey.append(ikeys[t_chan][i]) try: # find dist of key i for query i_idx = rkeys[t_chan].index( ikey[t_chan] ) # dataset include-key match # the reduced distance function in include_keys order # distance is Bhattacharyya distance on probs and dists dk[t_chan] = dists[t_chan][i_idx] except: print("Key not found in result list: ", ikey, "for query:", qkeys[t_chan]) raise error.BregmanError() rk = self._get_probs_tc(ikey) a_idx = t_keys.index( ikey[0] ) # audiodb include-key index rdists[a_idx] = distance.bhatt(pylab.sqrt(pylab.absolute(dk)), pylab.sqrt(pylab.absolute(qk*rk))) #search for the index of the relevant keys rdists = pylab.absolute(rdists) sort_idx = pylab.argsort(rdists) # Sort fields into database order for r in self.ground_truth: # relevant keys ranks_list.append(pylab.where(sort_idx==r)[0][0]) # Rank of the relevant key return ranks_list, rdists
def plot_elecs_and_neurons(neuron_dict, ext_sim_dict, neural_sim_dict): pl.close('all') fig_all = pl.figure(figsize=[15,15]) ax_all = fig_all.add_axes([0.1, 0.1, 0.8, 0.8], frameon=False) for elec in xrange(len(ext_sim_dict['elec_z'])): ax_all.plot(ext_sim_dict['elec_z'][elec], ext_sim_dict['elec_y'][elec], color='b',\ marker='$E%i$'%elec, markersize=20 ) legends = [] for i, neur in enumerate(neuron_dict): folder = os.path.join(neural_sim_dict['output_folder'], neuron_dict[neur]['name']) coor = np.load(os.path.join(folder,'coor.npy')) x,y,z = coor n_compartments = len(x) fig = pl.figure(figsize=[10, 10]) ax = fig.add_axes([0.1, 0.1, 0.8, 0.8], frameon=False) # Plot the electrodes for elec in xrange(len(ext_sim_dict['elec_z'])): ax.plot(ext_sim_dict['elec_z'][elec], ext_sim_dict['elec_y'][elec], color='b',\ marker='$%i$'%elec, markersize=20 ) # Plot the neuron xmid, ymid, zmid = np.load(folder + '/coor.npy') xstart, ystart,zstart = np.load(folder + '/coor_start.npy') xend, yend, zend = np.load(folder + '/coor_end.npy') diam = np.load(folder + '/diam.npy') length = np.load(folder + '/length.npy') n_compartments = len(diam) for comp in xrange(n_compartments): if comp == 0: xcoords = pl.array([xmid[comp]]) ycoords = pl.array([ymid[comp]]) zcoords = pl.array([zmid[comp]]) diams = pl.array([diam[comp]]) else: if zmid[comp] < 0.400 and zmid[comp] > -.400: xcoords = pl.r_[xcoords, pl.linspace(xstart[comp], xend[comp], length[comp]*3*1000)] ycoords = pl.r_[ycoords, pl.linspace(ystart[comp], yend[comp], length[comp]*3*1000)] zcoords = pl.r_[zcoords, pl.linspace(zstart[comp], zend[comp], length[comp]*3*1000)] diams = pl.r_[diams, pl.linspace(diam[comp], diam[comp], length[comp]*3*1000)] argsort = pl.argsort(-xcoords) ax.scatter(zcoords[argsort], ycoords[argsort], s=20*(diams[argsort]*1000)**2, c=xcoords[argsort], edgecolors='none', cmap='gray') ax_all.plot(zmid[0], ymid[0], marker='$%i$'%i, markersize=20, label='%i: %s' %(i, neur)) #legends.append('%i: %s' %(i, neur)) ax.axis(ext_sim_dict['plot_range']) ax.axis('equal') ax.axis(ext_sim_dict['plot_range']) ax.set_xlabel('z [mm]') ax.set_ylabel('y [mm]') fig.savefig(os.path.join(neural_sim_dict['output_folder'],\ 'neuron_figs', '%s.png' % neur)) ax_all.axis('equal') ax.axis(ext_sim_dict['plot_range']) ax_all.set_xlabel('z [mm]') ax_all.set_ylabel('y [mm]') ax_all.legend() fig_all.savefig(os.path.join(neural_sim_dict['output_folder'], 'fig.png'))
def rank_by_distance_avg(self, qkeys, ikeys, rkeys, dists): """ :: Reduce timbre-channel distances to ranks list by ground-truth key indices Kullback distances """ # timbre-channel search using pre-computed distances ranks_list = [] t_keys, t_lens = self.get_adb_lists(0) rdists=pylab.ones(len(t_keys))*float('inf') for t_chan in range(self.timbre_channels): # timbre channels t_keys, t_lens = self.get_adb_lists(t_chan) for i, ikey in enumerate(ikeys[t_chan]): # include keys, results try: # find dist of key i for query i_idx = rkeys[t_chan].index( ikey ) # lower_bounded include-key index a_idx = t_keys.index( ikey ) # audiodb include-key index # the reduced distance function in include_keys order # distance is the sum for now if t_chan: rdists[a_idx] += dists[t_chan][i_idx] else: rdists[a_idx] = dists[t_chan][i_idx] except: print("Key not found in result list: ", ikey, "for query:", qkeys[t_chan]) raise error.BregmanError() #search for the index of the relevant keys rdists = pylab.absolute(rdists) sort_idx = pylab.argsort(rdists) # Sort fields into database order for r in self.ground_truth: # relevant keys ranks_list.append(pylab.where(sort_idx==r)[0][0]) # Rank of the relevant key return ranks_list, rdists
def main(): plt.ion() fil = FletcherFilter() Niter = 12 logp = plt.zeros((Niter,2)) for k in range(Niter): while True: #print k p = plt.rand(2) if not fil.dominated(p): break logp[k] = p fil.add(p, 0.0, 0.0) ff = fil.values[fil.valid] ff = plt.r_[[[1e-6,1]], ff[plt.argsort(ff[:,0])], [[1,1e-6]]] ww = plt.zeros((ff.shape[0] * 2 - 1, 2)) ww[::2] = ff ww[1::2,0] = ff[1:,0] ww[1::2,1] = ff[:-1,1] plt.loglog(ww[:,0], ww[:,1], '-') plt.loglog(logp[:,0], logp[:,1], 'ys-', lw=2) plt.axis([0,1,0,1]) plt.axis('equal') plt.grid() code.interact()
def plot_uhecrs(phi, theta, values, cmap='jet', **kwargs): f = kwargs.get("figure", pylab.figure(figsize=(10, 10 * 0.75))) s = f.add_subplot(111, projection='hammer') pylab.grid(lw=1, color='0.5', alpha=0.5) # s.invert_xaxis() if values is None: pylab.scatter(-phi, theta, edgecolors=kwargs.get('edge', 'none'), marker='.', label="a", s=kwargs.get('size', 10)) else: srt = pylab.argsort(values) pylab.scatter(-phi[srt], theta[srt], c=values[srt], vmin=min(values), vmax=max(values), edgecolors=kwargs.get('edge', '0.7'), lw=0.1, marker='.', label="a", s=kwargs.get('size', 10), cmap=cmap) cb = pylab.colorbar(format='%g', orientation='horizontal', aspect=30, shrink=0.8, pad=0.1, label=kwargs.get('valuelabel', None)) cb.solids.set_edgecolor("face") pylab.gca().set_xticklabels([]) pylab.tight_layout(pad=0.1) return f, s
def rank_by_distance_bhatt(self, qkeys, ikeys, rkeys, dists): """ :: Reduce timbre-channel distances to ranks list by ground-truth key indices Bhattacharyya distance on timbre-channel probabilities and Kullback distances """ # timbre-channel search using pre-computed distances ranks_list = [] t_keys, t_lens = self.get_adb_lists(0) rdists=pylab.ones(len(t_keys))*float('inf') qk = self._get_probs_tc(qkeys) for i in range(len(ikeys[0])): # number of include keys ikey=[] dk = pylab.zeros(self.timbre_channels) for t_chan in range(self.timbre_channels): # timbre channels ikey.append(ikeys[t_chan][i]) try: # find dist of key i for query i_idx = rkeys[t_chan].index( ikey[t_chan] ) # dataset include-key match # the reduced distance function in include_keys order # distance is Bhattacharyya distance on probs and dists dk[t_chan] = dists[t_chan][i_idx] except: print "Key not found in result list: ", ikey, "for query:", qkeys[t_chan] raise error.BregmanError() rk = self._get_probs_tc(ikey) a_idx = t_keys.index( ikey[0] ) # audiodb include-key index rdists[a_idx] = distance.bhatt(pylab.sqrt(pylab.absolute(dk)), pylab.sqrt(pylab.absolute(qk*rk))) #search for the index of the relevant keys rdists = pylab.absolute(rdists) sort_idx = pylab.argsort(rdists) # Sort fields into database order for r in self.ground_truth: # relevant keys ranks_list.append(pylab.where(sort_idx==r)[0][0]) # Rank of the relevant key return ranks_list, rdists
def find_convex_hull(X, num_iter, num_points=None): """ if num_points is set to None, find_convex_hull will return all the points in the convex hull (that have been found) sorted according to their sharpness. Otherwise, it will return the N-sharpest points. """ (N, D) = X.shape if (num_points == None): num_points = N # randomly choose 'num_iter' direction on the unit sphere. # find the maximal point in the chosen direction, and add 1 to its counter. # only points on the convex hull will be hit, and 'sharp' corners will # have more hits than 'smooth' corners. hits = p.zeros((N, 1)) for j in xrange(num_iter): a = p.randn(D) a = a / p.norm(a) i = p.dot(X, a).argmax() hits[i] += 1 # don't take points with 0 hits num_points = min(num_points, sum(p.find(hits))) # the indices of the n-best points o = list(p.argsort(hits, 0)[xrange(-1, -(num_points+1), -1)].flat) return X[o, :]
def rank_by_distance_avg(self, qkeys, ikeys, rkeys, dists): """ :: Reduce timbre-channel distances to ranks list by ground-truth key indices Kullback distances """ # timbre-channel search using pre-computed distances ranks_list = [] t_keys, t_lens = self.get_adb_lists(0) rdists=pylab.ones(len(t_keys))*float('inf') for t_chan in range(self.timbre_channels): # timbre channels t_keys, t_lens = self.get_adb_lists(t_chan) for i, ikey in enumerate(ikeys[t_chan]): # include keys, results try: # find dist of key i for query i_idx = rkeys[t_chan].index( ikey ) # lower_bounded include-key index a_idx = t_keys.index( ikey ) # audiodb include-key index # the reduced distance function in include_keys order # distance is the sum for now if t_chan: rdists[a_idx] += dists[t_chan][i_idx] else: rdists[a_idx] = dists[t_chan][i_idx] except: print "Key not found in result list: ", ikey, "for query:", qkeys[t_chan] raise error.BregmanError() #search for the index of the relevant keys rdists = pylab.absolute(rdists) sort_idx = pylab.argsort(rdists) # Sort fields into database order for r in self.ground_truth: # relevant keys ranks_list.append(pylab.where(sort_idx==r)[0][0]) # Rank of the relevant key return ranks_list, rdists
def sorted_points(df, col1, col2): x = pylab.array(df[col1]) y = pylab.array(df[col2]) order = pylab.argsort(x) x = x[order] y = y[order] return x, y
def FDR_BH(self, p): """Benjamini-Hochberg p-value correction for multiple hypothesis testing.""" p = np.asfarray(p) by_descend = p.argsort()[::-1] by_orig = by_descend.argsort() steps = float(len(p)) / np.arange(len(p), 0, -1) q = np.minimum(1, np.minimum.accumulate(steps * p[by_descend])) return q[by_orig]
def gen_topic(result, t, W0, word_list, k0): if t in result.topics: return result.topics[t] word_weight = W0[:,t] word_ind = argsort(word_weight)[::-1] topic_node = [(word_list[w], W0[w,t]) for w in word_ind[:k0]] result.topics[t] = topic_node return topic_node
def gen_group(result, g, W1, W0, word_list, k1, k0): if g in result.groups: return result.groups[g] topic_weight = W1[:,g] topic_ind = argsort(topic_weight)[::-1] for t in topic_ind[:k1]: gen_topic(result, t, W0, word_list, k0) group_node = [(t, W1[t,g]) for t in topic_ind[:k1]] result.groups[g] = group_node return group_node
def gen_super(result, s, W2, W1, W0, word_list, k2, k1, k0): if s in result.supers: return result.supers[s] group_weight = W2[:,s] group_ind = argsort(group_weight)[::-1] for g in group_ind[:k2]: gen_group(result, g, W1, W0, word_list, k1, k0) super_node = [(g, W2[g,s]) for g in group_ind[:k2]] result.supers[s] = super_node return super_node
def __init__(self, logger=None, debug=True): hostname = socket.gethostname() self.port = '/dev/arduinoElAxis' self.baudrate = 57600 self.debug = debug self.lock = threading.Lock() self.ser = serial.Serial() self.setLogger(logger) #prefix = datetime.datetime.now().strftime('%Y%m%d_%H%M%S') #if logger == None: # self.lw = logWriter.logWriter(prefix, verbose=False) #else: # self.lw = logger self.posD = -9999.999 stepPole = array([ 0., 100., 200., 300., 400., 500., 600., 700., 800., 900., 1000., 1100., 1200., 1300., 1400., 1500., 1600., 1700., 1800., 1900., 2000., 2100., 2200., 2300., 2400., 2500., 2600., 2700., 2800., 2900., 3000., 3100., 3200., 3300., 3400. ]) anglePole = array([ 93.75, 90.45, 87.8, 84.95, 82.15, 79.45, 76.85, 74.3, 71.85, 69.4, 67., 64.65, 62.25, 60.05, 57.65, 55.35, 53.15, 50.85, 48.55, 46.2, 43.95, 41.7, 39.3, 36.95, 34.5, 32.2, 29.6, 27.05, 24.6, 21.8, 19.2, 16.55, 13.8, 11.15, 8.94 ]) stepSummit = array([ 0., 100., 200., 300., 400., 500., 600., 700., 800., 900., 1000., 1100., 1200., 1300., 1400., 1500., 1600., 1700., 1800., 1900., 2000., 2100., 2200., 2300., 2400., 2500., 2600., 2700., 2800., 2900., 3000., 3100., 3200. ]) angleSummit = array([ 92.6, 88.85, 85.35, 82.2, 79.15, 76.35, 73.7, 70.7, 67.95, 65.25, 62.85, 60.35, 57.7, 55.25, 52.65, 50.25, 47.6, 45.2, 42.7, 40.15, 37.5, 35.0, 32.5, 29.55, 27.2, 24.3, 21.25, 18.5, 15.6, 12.5, 9.45, 6.0, 2.65 ]) if 'wvr1' in hostname: angle = anglePole step = stepPole elif 'wvr2' in hostname: angle = angleSummit step = stepSummit q = argsort(angle) self.step2Angle = scipy.interpolate.interp1d(step, angle, kind='linear') self.angle2Step = scipy.interpolate.interp1d(angle[q], step[q], kind='linear')
def plot_frequencies_hyperedges(self): """Plot frequencies of the hyperedges amonsgt all models .. plot:: >>> from cellnopt.optimiser import ASPBool >>> from cellnopt.data import cnodata >>> a = ASPBool(cnodata("PKN-ToyMMB.sif"), cnodata("MD-ToyMMB.csv")) >>> a.run(fit=2, gtts=False, size=2) >>> a.plot_frequencies_hyperedges() .. todo:: refine plotting to cope with xlabels being too long. """ from pylab import clf, plot, xlim, grid, xlabel, ylabel, vlines, title from pylab import xticks, arange, argsort _freq = list(self.family.frequencies) values = [x[1] for x in _freq] names = [x[0] for x in _freq] # let us sort them (reversed) and sort the names accordingly values = sorted(values, reverse=True) sorted_names = [names[i] for i in argsort(names)[::-1]] N = len(values) Nzeros = values.count(0) Nones = values.count(1) clf(); plot(range(1, N+1), values, 'o-'); xlim([1, N]); grid() vlines(Nones, 0, 1, linestyles="--") vlines(Nones+(N-Nzeros), 0,1, linestyles="--") xlabel("Hyperedges") xticks(arange(1,1+len(names)), sorted_names, rotation=90, fontsize=8) ylabel("Frequency") title("Frequencies of hyperedges over %s sub-optimal models \nwithin %s %% tolerance" % (len(self.models), self.fit*100))
def val_distribution(df, val, categories, plot_func, plot_descriptor,\ outdir=FIGS_DIR, fig_suffix=None, category_subset=None): for category in categories: subset_cols = ['neuron name', 'neuron type', 'alpha'] if category != 'neuron type': subset_cols.append(category) df2 = df.drop_duplicates(subset=subset_cols) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] else: df2 = remove_small_counts(df2, category,\ min_count=CATEGORY_MIN_COUNTS[category]) cat_vals = [] medians = [] for name, group in df2.groupby(category): cat_vals.append(name) medians.append(pylab.median(group[val])) cat_vals = pylab.array(cat_vals) mean = pylab.array(medians) order = pylab.argsort(medians) order = cat_vals[order] pylab.figure() sns.set() dist_plot = plot_func(x=val, y=category, data=df2, orient='h', order=order) dist_plot.tick_params(axis='y', labelsize=20) pylab.tight_layout() pylab.xlabel(val, fontsize=20) pylab.ylabel(category, fontsize=20) fname = '%s_%ss_%s' % (category.replace(' ', '_'),\ val.replace(' ', '_'), plot_descriptor) if fig_suffix != None: fname += '_%s' % fig_suffix pylab.savefig('%s/%s.pdf' % (outdir, fname), format='pdf') pylab.close()
def category_dists(df, categories, outdir=FIGS_DIR, fig_suffix=None,\ category_subset=None): for category in categories: df2 = df.drop_duplicates(subset=list(set(['neuron name', 'neuron type', category]))) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] else: df2 = remove_small_counts(df2, category,\ min_count=CATEGORY_MIN_COUNTS[category]) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] df2['dist'] = pylab.log10(df2['dist']) cat_vals = [] cat_means = [] for cat_val, group in df2.groupby(category): cat_vals.append(cat_val) cat_mean = pylab.mean(group['dist']) cat_means.append(cat_mean) order = pylab.argsort(cat_means) cat_vals = pylab.array(cat_vals) sorted_vals = cat_vals[order] pylab.figure() sns.set() dist_plot = sns.barplot(x=category, y='dist', data=df2, order=sorted_vals) pylab.xticks(rotation='vertical', size=20) pylab.xlabel(category, size=20) pylab.ylabel('log-distance to Pareto front', size=20) #pylab.tight_layout() fname = 'pareto_dists_%s' % category.replace(' ', '_') if fig_suffix != None: fname += '_%s' % fig_suffix pylab.savefig('%s/%s.pdf' % (outdir, fname), bbox_inches='tight')
def scatter_dists(df): df2 = df.drop_duplicates(subset='name') neural_dist = df2['neural_dist'] centroid_dist = df2['centroid_dist'] random_dist = df2['random_dist'] assert len(neural_dist) == len(centroid_dist) == len(random_dist) order = pylab.argsort(neural_dist) neural_dist = neural_dist[order] centroid_dist = centroid_dist[order] random_dist = random_dist[order] x = range(len(neural_dist)) pylab.figure() pylab.scatter(x, random_dist, c='m', label='random') pylab.scatter(x, centroid_dist, c='g', label='centroid') pylab.scatter(x, neural_dist, c='r', label='neural') pylab.ylabel('distance') pylab.title('Distance to Pareto Front') pylab.legend() pylab.savefig('%s/pareto_dists.pdf' % OUTDIR, format='pdf') pylab.close()
def zipf_law(df): # Plot of absolute frequency from pylab import arange, argsort, loglog, logspace, log10, text counts = df.total tokens = df.index ranks = arange(1, len(counts)+1) indices = argsort(-counts) frequencies = counts[indices] fig, ax = plt.subplots(figsize=(8,6)) ax.set_ylim(1,10**6) ax.set_xlim(1,10**6) loglog(ranks, frequencies, marker=".") ax.plot([1,frequencies[0]],[frequencies[0],1],color='r') #ax.set_title("Zipf plot for phrases tokens") ax.set_xlabel("Frequency rank of token") ax.set_ylabel("Absolute frequency of token") ax.grid(True) for n in list(logspace(-0.5, log10(len(counts)-2), 15).astype(int)): dummy = text(ranks[n], frequencies[n], " " + tokens[indices[n]], verticalalignment="bottom", horizontalalignment="left") ax.figure.savefig(figOutputPath / '2_zipf_law.png', format='png') print('Exported 2_zipf_law.png')
def get_density_stats(dirname="sample"): img_stats = [] chr_density_sort = [] tree = os.walk(dirname) for subfol in tree: img_path = subfol[0] img_files = subfol[2] for img in img_files: img_char = img.split(".")[0] img_sum = sum(imread(os.path.join(img_path,img))) # print (os.path.join(img_path,img)) img_stats.append(img_sum) # print img_sum, img_char chr_density_sort.append(int(img_char)) chr_density_sort = array(chr_density_sort) chr_density_sort = chr_density_sort[argsort(img_stats)] # print chr_density_sort, len(chr_density_sort) print "Visual density sort obtained." return chr_density_sort
def save_density_stats(dirname="sample"): img_stats = [] chr_density_sort = [] tree = os.walk(dirname) for subfol in tree: img_path = subfol[0] img_files = subfol[2] for img in img_files: img_char = img.split(".")[0] img_sum = sum(imread(os.path.join(img_path,img))) img_stats.append(img_sum) chr_density_sort.append(int(img_char)) chr_density_sort = array(chr_density_sort) chr_density_sort = list(chr_density_sort[argsort(img_stats)]) # print chr_density_sort, len(chr_density_sort) print "Visual density sort obtained." f = open('density_stats.txt', 'w') f.writelines("".join(map(chr, chr_density_sort))) f.close()
def plot_uhecrs(phi, theta, values, cmap='jet', **kwargs): f = kwargs.get("figure", pylab.figure(figsize=(10, 10 * 0.75))) s = f.add_subplot(111, projection='hammer') pylab.grid(lw=1, color='0.5', alpha=0.5) # s.invert_xaxis() if values is None: pylab.scatter(-phi, theta, edgecolors=kwargs.get('edge', 'none'), marker='.', label="a", s=kwargs.get('size', 10)) else: srt = pylab.argsort(values) pylab.scatter(-phi[srt], theta[srt], c=values[srt], vmin=min(values), vmax=max(values), edgecolors=kwargs.get('edge', '0.7'), lw=0.1, marker='.', label="a", s=kwargs.get('size', 10), cmap=cmap) cb = pylab.colorbar( format='%g', orientation='horizontal', aspect=30, shrink=0.8, pad=0.1, label=kwargs.get('valuelabel', None)) cb.solids.set_edgecolor("face") pylab.gca().set_xticklabels([]) pylab.tight_layout(pad=0.1) return f, s
def raster_tuning(ax): fullbehaviorDir = behaviorDir+subject+'/' behavName = subject+'_tuning_curve_'+tuningBehavior+'.h5' tuningBehavFileName=os.path.join(fullbehaviorDir, behavName) tuning_bdata = loadbehavior.BehaviorData(tuningBehavFileName,readmode='full') freqEachTrial = tuning_bdata['currentFreq'] possibleFreq = np.unique(freqEachTrial) numberOfTrials = len(freqEachTrial) # -- The old way of sorting (useful for plotting sorted raster) -- sortedTrials = [] numTrialsEachFreq = [] #Used to plot lines after each group of sorted trials for indf,oneFreq in enumerate(possibleFreq): #indf is index of this freq and oneFreq is the frequency indsThisFreq = np.flatnonzero(freqEachTrial==oneFreq) #this gives indices of this frequency sortedTrials = np.concatenate((sortedTrials,indsThisFreq)) #adds all indices to a list called sortedTrials numTrialsEachFreq.append(len(indsThisFreq)) #finds number of trials each frequency has sortingInds = argsort(sortedTrials) #gives array of indices that would sort the sortedTrials # -- Load event data and convert event timestamps to ms -- tuning_ephysDir = os.path.join(settings.EPHYS_PATH, subject,tuningEphys) tuning_eventFilename=os.path.join(tuning_ephysDir, 'all_channels.events') tuning_ev=loadopenephys.Events(tuning_eventFilename) #load ephys data (like bdata structure) tuning_eventTimes=np.array(tuning_ev.timestamps)/SAMPLING_RATE #get array of timestamps for each event and convert to seconds by dividing by sampling rate (Hz). matches with eventID and tuning_evID=np.array(tuning_ev.eventID) #loads the onset times of events (matches up with eventID to say if event 1 went on (1) or off (0) tuning_eventOnsetTimes=tuning_eventTimes[tuning_evID==1] #array that is a time stamp for when the chosen event happens. #ev.eventChannel woul load array of events like trial start and sound start and finish times (sound event is 0 and trial start is 1 for example). There is only one event though and its sound start while (numberOfTrials < len(tuning_eventOnsetTimes)): tuning_eventOnsetTimes = tuning_eventOnsetTimes[:-1] ####################################################################################################### ###################THIS IS SUCH A HACK TO GET SPKDATA FROM EPHYSCORE################################### ####################################################################################################### thisCell = celldatabase.CellInfo(animalName=subject,############################################ ephysSession = tuningEphys, tuningSession = 'DO NOT NEED THIS', tetrode = tetrode, cluster = cluster, quality = 1, depth = 0, tuningBehavior = 'DO NOT NEED THIS', behavSession = tuningBehavior) tuning_spkData = ephyscore.CellData(thisCell) tuning_spkTimeStamps = tuning_spkData.spikes.timestamps (tuning_spikeTimesFromEventOnset,tuning_trialIndexForEachSpike,tuning_indexLimitsEachTrial) = spikesanalysis.eventlocked_spiketimes(tuning_spkTimeStamps,tuning_eventOnsetTimes,tuning_timeRange) #print 'numTrials ',max(tuning_trialIndexForEachSpike)##################################### ''' Create a vector with the spike timestamps w.r.t. events onset. (spikeTimesFromEventOnset,trialIndexForEachSpike,indexLimitsEachTrial) = eventlocked_spiketimes(timeStamps,eventOnsetTimes,timeRange) timeStamps: (np.array) the time of each spike. eventOnsetTimes: (np.array) the time of each instance of the event to lock to. timeRange: (list or np.array) two-element array specifying time-range to extract around event. spikeTimesFromEventOnset: 1D array with time of spikes locked to event. o trialIndexForEachSpike: 1D array with the trial corresponding to each spike. The first spike index is 0. indexLimitsEachTrial: [2,nTrials] range of spikes for each trial. Note that the range is from firstSpike to lastSpike+1 (like in python slices) spikeIndices ''' tuning_sortedIndexForEachSpike = sortingInds[tuning_trialIndexForEachSpike] #Takes values of trialIndexForEachSpike and finds value of sortingInds at that index and makes array. This array gives an array with the sorted index of each trial for each spike # -- Calculate tuning -- #nSpikes = spikesanalysis.spiketimes_to_spikecounts(spikeTimesFromEventOnset,indexLimitsEachTrial,responseRange) #array of the number of spikes in range for each trial '''Count number of spikes on each trial in a given time range. spikeTimesFromEventOnset: vector of spikes timestamps with respect to the onset of the event. indexLimitsEachTrial: each column contains [firstInd,lastInd+1] of the spikes on a trial. timeRange: time range to evaluate. Spike times exactly at the limits are not counted. returns nSpikes ''' ''' meanSpikesEachFrequency = np.empty(len(possibleFreq)) #make empty array of same size as possibleFreq # -- This part will be replace by something like behavioranalysis.find_trials_each_type -- trialsEachFreq = [] for indf,oneFreq in enumerate(possibleFreq): trialsEachFreq.append(np.flatnonzero(freqEachTrial==oneFreq)) #finds indices of each frequency. Appends them to get an array of indices of trials sorted by freq # -- Calculate average firing for each freq -- for indf,oneFreq in enumerate(possibleFreq): meanSpikesEachFrequency[indf] = np.mean(nSpikes[trialsEachFreq[indf]]) ''' #clf() #if (len(tuning_spkTimeStamps)>0): #ax1 = plt.subplot2grid((4,4), (3, 0), colspan=1) #spikesorting.plot_isi_loghist(spkData.spikes.timestamps) #ax3 = plt.subplot2grid((4,4), (3, 3), colspan=1) #spikesorting.plot_events_in_time(tuning_spkTimeStamps) #samples = tuning_spkData.spikes.samples.astype(float)-2**15 #samples = (1000.0/tuning_spkData.spikes.gain[0,0]) *samples #ax2 = plt.subplot2grid((4,4), (3, 1), colspan=2) #spikesorting.plot_waveforms(samples) #ax4 = plt.subplot2grid((4,4), (0, 0), colspan=3,rowspan = 3) plot(tuning_spikeTimesFromEventOnset, tuning_sortedIndexForEachSpike, '.', ms=3) #axvline(x=0, ymin=0, ymax=1, color='r') #The cumulative sum of the list of specific frequency presentations, #used below for plotting the lines across the figure. numTrials = cumsum(numTrialsEachFreq) #Plot the lines across the figure in between each group of sorted trials for indf, num in enumerate(numTrials): ax.axhline(y = num, xmin = 0, xmax = 1, color = '0.90', zorder = 0) tickPositions = numTrials - mean(numTrialsEachFreq)/2 tickLabels = ["%0.2f" % (possibleFreq[indf]/1000) for indf in range(len(possibleFreq))] ax.set_yticks(tickPositions) ax.set_yticklabels(tickLabels) ax.set_ylim([-1,numberOfTrials]) ylabel('Frequency Presented (kHz), {} total trials'.format(numTrials[-1])) #title(ephysSession+' T{}c{}'.format(tetrodeID,clusterID)) xlabel('Time (sec)') ''' ax5 = plt.subplot2grid((4,4), (0, 3), colspan=1,rowspan=3) ax5.set_xscale('log') plot(possibleFreq,meanSpikesEachFrequency,'o-') ylabel('Avg spikes in window {0}-{1} sec'.format(*responseRange)) xlabel('Frequency') ''' #show() '''
def main(): import optparse from numpy import sum # Parse command line parser = optparse.OptionParser(usage=USAGE) parser.add_option("-p", "--plot", action="store_true", help="Generate pdf with IR-spectrum, broadened with Lorentzian") parser.add_option("-i", "--info", action="store_true", help="Set up/ Calculate vibrations & quit") parser.add_option("-s", "--suffix", action="store", help="Call suffix for binary e.g. 'mpirun -n 4 '", default='') parser.add_option("-r", "--run", action="store", help="path to FHI-aims binary",default='') parser.add_option("-x", "--relax", action="store_true", help="Relax initial geometry") parser.add_option("-m", "--molden", action="store_true", help="Output in molden format") parser.add_option("-w", "--distort", action="store_true", help="Output geometry distorted along imaginary modes") parser.add_option("-t", "--submit", action="store", help="""\ Path to submission script, string <jobname> will be replaced by name + counter, string <outfile> will be replaced by filename""") parser.add_option("-d", "--delta", action="store", type="float", help="Displacement", default=0.0025) parser.add_option("-b", "--broadening", action="store", type="float", help="Broadening for IR-spectrum in cm^{-1}", default=5) options, args = parser.parse_args() if options.info: print __doc__ sys.exit(0) if len(args) != 2: parser.error("Need exactly two arguments") AIMS_CALL=options.suffix+' '+options.run hessian_thresh = -1 name=args[0] mode=args[1] delta=options.delta broadening=options.broadening run_aims=False if options.run!='': run_aims=True submit_script = options.submit is not None if options.plot: import matplotlib as mpl mpl.use('Agg') from pylab import figure if options.plot or mode=='1' or mode=='2': from pylab import savetxt, transpose, eig, argsort, sort,\ sign, pi, dot, sum, linspace, argmin, r_, convolve # Constant from scipy.constants bohr=constants.value('Bohr radius')*1.e10 hartree=constants.value('Hartree energy in eV') at_u=constants.value('atomic mass unit-kilogram relationship') eV=constants.value('electron volt-joule relationship') c=constants.value('speed of light in vacuum') Ang=1.0e-10 hbar=constants.value('Planck constant over 2 pi') Avo=constants.value('Avogadro constant') kb=constants.value('Boltzmann constant in eV/K') pi=constants.pi hessian_factor = eV/(at_u*Ang*Ang) grad_dipole_factor=(eV/(1./(10*c)))/Ang #(eV/Ang -> D/Ang) ir_factor = 1 # Asign all filenames inputgeomerty = 'geometry.in.'+name inputcontrol = 'control.in.'+name atomicmasses = 'masses.'+name+'.dat'; xyzfile = name+'.xyz'; moldenname =name+'.molden'; hessianname = 'hessian.'+name+'.dat'; graddipolename = 'grad_dipole.'+name+'.dat'; irname = 'ir.'+name+'.dat'; deltas=array([-delta,delta]) coeff=array([-1,1]) c_zero = - 1. / (2. * delta) f=open('control.in','r') # read control.in template template_control=f.read() f.close if submit_script: f=open(options.submit,'r') # read submission script template template_job=f.read() f.close folder='' # Dummy ########### Central Point ################################################## if options.relax and (mode=='0' or mode=='2'): # First relax input geometry filename=name+'.out' folder=name+'_relaxation' if not os.path.exists(folder): os.mkdir(folder) # Create folder shutil.copy('geometry.in', folder+'/geometry.in') # Copy geometry new_control=open(folder+'/control.in','w') new_control.write(template_control+'relax_geometry trm 1E-3\n') # Relax! new_control.close() os.chdir(folder) # Change directoy print 'Central Point' if run_aims: os.system(AIMS_CALL+' > '+filename) # Run aims and pipe the output # into a file named 'filename' if submit_script: replace_submission(template_job, name, 0, filename) os.chdir('..') ############################################################################ # Check for relaxed geometry if os.path.exists(folder+'/geometry.in.next_step'): geometry=open(folder+'/geometry.in.next_step','r') else: geometry=open('geometry.in','r') # Read input geometry n_line=0 struc=structure() lines=geometry.readlines() for line in lines: n_line= n_line+1 if line.rfind('set_vacuum_level')!=-1: # Vacuum Level struc.vacuum_level=float(split_line(line)[-1]) if line.rfind('lattice_vector')!=-1: # Lattice vectors and periodic lat=split_line(line)[1:] struc.lattice_vector=append(struc.lattice_vector,float64(array(lat)) [newaxis,:],axis=0) struc.periodic=True if line.rfind('atom')!=-1: # Set atoms line_vals=split_line(line) at=Atom(line_vals[-1],line_vals[1:-1]) if n_line<len(lines): nextline=lines[n_line] if nextline.rfind('constrain_relaxation')!=-1: # constrained? at=Atom(line_vals[-1],line_vals[1:-1],True) else: at=Atom(line_vals[-1],line_vals[1:-1]) struc.join(at) geometry.close() n_atoms= struc.n() n_constrained=n_atoms-sum(struc.constrained) # Atomic mass file mass_file=open(atomicmasses,'w') mass_vector=zeros([0]) for at_unconstrained in struc.atoms[struc.constrained==False]: mass_vector=append(mass_vector,ones(3)*1./sqrt(at_unconstrained.mass())) line='{0:10.5f}'.format(at_unconstrained.mass()) for i in range(3): line=line+'{0:11.4f}'.format(at_unconstrained.coord[i]) line=line+'{0:}\n'.format(at_unconstrained.kind) mass_file.writelines(line) mass_file.close() # Init dip = zeros([n_constrained*3,3]) hessian = zeros([n_constrained*3,n_constrained*3]) index=0 counter=1 # Set up / Read folders for displaced atoms for atom in arange(n_atoms)[struc.constrained==False]: for coord in arange(3): for delta in deltas: filename=name+'.i_atom_'+str(atom)+'.i_coord_'+str(coord)+'.displ_'+\ str(delta)+'.out' folder=name+'.i_atom_'+str(atom)+'.i_coord_'+str(coord)+'.displ_'+\ str(delta) if mode=='0' or mode=='2': # Put new geometry and control.in into folder struc_new=copy.deepcopy(struc) struc_new.atoms[atom].coord[coord]=\ struc_new.atoms[atom].coord[coord]+delta geoname='geometry.i_atom_'+str(atom)+'.i_coord_'+str(coord)+\ '.displ_'+str(delta)+'.in' if not os.path.exists(folder): os.mkdir(folder) new_geo=open(folder+'/geometry.in','w') newline='#\n# temporary structure-file for finite-difference '+\ 'calculation of forces\n' newline=newline+'# displacement {0:8.4f} of \# atom '.format(delta)+\ '{0:5} direction {1:5}\n#\n'.format(atom,coord) new_geo.writelines(newline+struc_new.to_str()) new_geo.close() new_control=open(folder+'/control.in','w') template_control=template_control.replace('relax_geometry', '#relax_geometry') new_control.write(template_control+'compute_forces .true. \n'+\ 'final_forces_cleaned '+\ '.true. \n') new_control.close() os.chdir(folder) # Change directoy print 'Processing atom: '+str(atom+1)+'/'+str(n_atoms)+', coord.: '+\ str(coord+1)+'/'+str(3)+', delta: '+str(delta) if run_aims: os.system(AIMS_CALL+' > '+filename)# Run aims and pipe the output # into a file named 'filename' if submit_script: replace_submission(template_job, name, counter, filename) # os.system('qsub job.sh') # Mind the environment variables os.chdir('..') if mode=='1' or mode=='2': # Read output forces_reached=False atom_count=0 data=open(folder+'/'+filename) for line in data.readlines(): if line.rfind('Dipole correction potential jump')!=-1: dip_jump = float(split_line(line)[-2]) # Periodic if line.rfind('| Total dipole moment [eAng]')!=-1: dip_jump = float64(split_line(line)[-3:]) # Cluster if forces_reached and atom_count<n_atoms: # Read Forces struc.atoms[atom_count].force=float64(split_line(line)[2:]) atom_count=atom_count+1 if atom_count==n_atoms: forces_reached=False if line.rfind('Total atomic forces')!=-1: forces_reached=True data.close() if struc.periodic: pass #dip[index,2]=dip[index,2]+dip_jump*coeff[deltas==delta]*c_zero else: dip[index,:]=dip[index,:]+dip_jump*coeff[deltas==delta]*c_zero forces=array([]) for at_unconstrained in struc.atoms[struc.constrained==False]: forces=append(forces,coeff[deltas==delta]*at_unconstrained.force) hessian[index,:]=hessian[index,:]+forces*c_zero counter=counter+1 index=index+1 if mode=='1' or mode=='2': # Calculate vibrations print 'Entering hessian diagonalization' print 'Number of atoms = '+str(n_atoms) print 'Name of Hessian input file = '+hessianname print 'Name of grad dipole input file = '+graddipolename print 'Name of Masses input file = '+atomicmasses print 'Name of XYZ output file = '+xyzfile print 'Threshold for Matrix elements = '+str(hessian_thresh) if (hessian_thresh < 0.0): print ' All matrix elements are taken'+\ ' into account by default\n' savetxt(hessianname,hessian) savetxt(graddipolename,dip) mass_mat=mass_vector[:,newaxis]*mass_vector[newaxis,:] hessian[abs(hessian)<hessian_thresh]=0.0 hessian=hessian*mass_mat*hessian_factor hessian=(hessian+transpose(hessian))/2. # Diagonalize hessian (scipy) print 'Solving eigenvalue system for Hessian Matrix' freq, eig_vec = eig(hessian) print 'Done ... ' eig_vec=eig_vec[:,argsort(freq)] freq=sort(sign(freq)*sqrt(abs(freq))) ZPE=hbar*(freq)/(2.0*eV) freq = (freq)/(200.*pi*c) grad_dipole = dip * grad_dipole_factor eig_vec = eig_vec*mass_vector[:,newaxis]*ones(len(mass_vector))[newaxis,:] infrared_intensity = sum(dot(transpose(grad_dipole),eig_vec)**2,axis=0)*\ ir_factor reduced_mass=sum(eig_vec**2,axis=0) norm = sqrt(reduced_mass) eig_vec = eig_vec/norm # The rest is output, xyz, IR,... print 'Results\n' print 'List of all frequencies found:' print 'Mode number Frequency [cm^(-1)] Zero point energy [eV] '+\ 'IR-intensity [D^2/Ang^2]' for i in range(len(freq)): print '{0:11}{1:25.8f}{2:25.8f}{3:25.8f}'.format(i+1,freq[i],ZPE[i], infrared_intensity[i]) print '\n' print 'Summary of zero point energy for entire system:' print '| Cumulative ZPE = {0:15.8f} eV'.format(sum(ZPE)) print '| without first six eigenmodes = {0:15.8f} eV\n'.format(sum(ZPE)- sum(ZPE[:6])) print 'Stability checking - eigenvalues should all be positive for a '+\ 'stable structure. ' print 'The six smallest frequencies should be (almost) zero:' string='' for zz in ZPE[:6]: string=string+'{0:25.8f}'.format(zz) print string print 'Compare this with the largest eigenvalue, ' print '{0:25.8f}'.format(freq[-1]) nums=arange(n_atoms)[struc.constrained==False] nums2=arange(n_atoms)[struc.constrained] newline='' newline_ir='[INT]\n' if options.molden: newline_molden='[Molden Format]\n[GEOMETRIES] XYZ\n' newline_molden=newline_molden+'{0:6}\n'.format(n_atoms)+'\n' for i_atoms in range(n_constrained): newline_molden=newline_molden+'{0:6}'.format( struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline_molden=newline_molden+'{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord]) newline_molden=newline_molden+'\n' newline_molden=newline_molden+'[FREQ]\n' for i in range(len(freq)): newline_molden=newline_molden+'{0:10.3f}\n'.format(freq[i]) newline_molden=newline_molden+'[INT]\n' for i in range(len(freq)): newline_molden=newline_molden+'{0:17.6e}\n'.format( infrared_intensity[i]) newline_molden=newline_molden+'[FR-COORD]\n' newline_molden=newline_molden+'{0:6}\n'.format(n_atoms)+'\n' for i_atoms in range(n_constrained): newline_molden=newline_molden+'{0:6}'.format( struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline_molden=newline_molden+'{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord]/bohr) newline_molden=newline_molden+'\n' newline_molden=newline_molden+'[FR-NORM-COORD]\n' for i in range(len(freq)): newline=newline+'{0:6}\n'.format(n_atoms) if freq[i]>0: newline=newline+'stable frequency at ' elif freq[i]<0: newline=newline+'unstable frequency at ' if options.distort and freq[i]<-50: struc_new=copy.deepcopy(struc) for i_atoms in range(n_constrained): for i_coord in range(3): struc_new.atoms[i_atoms].coord[i_coord]=\ struc_new.atoms[i_atoms].coord[i_coord]+\ eig_vec[(i_atoms)*3+i_coord,i] geoname=name+'.distorted.vibration_'+str(i+1)+'.geometry.in' new_geo=open(geoname,'w') newline_geo='#\n# distorted structure-file for based on eigenmodes\n' newline_geo=newline_geo+\ '# vibration {0:5} :{1:10.3f} 1/cm\n#\n'.format(i+1,freq[i]) new_geo.writelines(newline_geo+struc_new.to_str()) new_geo.close() elif freq[i]==0: newline=newline+'translation or rotation ' newline=newline+'{0:10.3f} 1/cm IR int. is '.format(freq[i]) newline=newline+'{0:10.4e} D^2/Ang^2; red. mass is '.format( infrared_intensity[i]) newline=newline+'{0:5.3f} a.m.u.; force const. is '.format( 1.0/reduced_mass[i]) newline=newline+'{0:5.3f} mDyne/Ang.\n'.format(((freq[i]*(200*pi*c))**2)* (1.0/reduced_mass[i])*at_u*1.e-2) if options.molden: newline_molden=newline_molden+\ 'vibration {0:6}\n'.format(i+1) for i_atoms in range(n_constrained): newline=newline+'{0:6}'.format(struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline=newline+'{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord]) for i_coord in range(3): newline=newline+'{0:10.4f}'.format(eig_vec[(i_atoms)*3+i_coord,i]) if options.molden: newline_molden=newline_molden+'{0:10.4f}'.format( eig_vec[(i_atoms)*3+i_coord,i]/bohr) newline=newline+'\n' if options.molden: newline_molden=newline_molden+'\n' for i_atoms in range(n_atoms-n_constrained): newline=newline+'{0:6}'.format(struc.atoms[nums2[i_atoms]].kind) for i_coord in range(3): newline=newline+'{0:10.4f}'.format( struc.atoms[nums2[i_atoms]].coord[i_coord]) for i_coord in range(3): newline=newline+'{0:10.4f}'.format(0.0) newline=newline+'\n' newline_ir=newline_ir+'{0:10.4e}\n'.format(infrared_intensity[i]) xyz=open(xyzfile,'w') xyz.writelines(newline) xyz.close() ir=open(irname,'w') ir.writelines(newline_ir) ir.close() if options.molden: molden=open(moldenname,'w') molden.writelines(newline_molden) molden.close() if (mode=='1' or mode=='2') and options.plot: x=linspace(freq.min()-500,freq.max()+500,1000) z=zeros(len(x)) for i in range(len(freq)): z[argmin(abs(x-freq[i]))]=infrared_intensity[i] window_len=150 lorentzian=lorentz(pi,broadening,arange(250))#signal.gaussian(window_len,broadening) s=r_[z[window_len-1:0:-1],z,z[-1:-window_len:-1]] z_convolve=convolve(lorentzian/lorentzian.sum(),s,mode='same')[ window_len-1:-window_len+1] fig=figure(0) ax=fig.add_subplot(111) ax.plot(x,z_convolve,'r',lw=2) ax.set_xlim([freq.min()-500,freq.max()+500]) ax.set_ylim([-0.01,ax.get_ylim()[1]]) ax.set_yticks([]) ax.set_xlabel('Frequency [1/cm]',size=20) ax.set_ylabel('Intensity [a.u.]',size=20) fig.savefig(name+'_IR_spectrum.pdf') print '\n Done. '
def plot_mea(neuron_dict, ext_sim_dict, neural_sim_dict): pl.close('all') fig_all = pl.figure(figsize=[15, 15]) ax_all = fig_all.add_axes([0.1, 0.1, 0.8, 0.8], frameon=False) for elec in xrange(len(ext_sim_dict['elec_z'])): ax_all.plot(ext_sim_dict['elec_z'][elec], ext_sim_dict['elec_y'][elec], color='b',\ marker='$E%i$'%elec, markersize=20 ) legends = [] for i, neur in enumerate(neuron_dict): folder = os.path.join(neural_sim_dict['output_folder'], neuron_dict[neur]['name']) coor = np.load(os.path.join(folder, 'coor.npy')) x, y, z = coor n_compartments = len(x) fig = pl.figure(figsize=[10, 10]) ax = fig.add_axes([0.1, 0.1, 0.8, 0.8], frameon=False) # Plot the electrodes for elec in xrange(len(ext_sim_dict['elec_z'])): ax.plot(ext_sim_dict['elec_z'][elec], ext_sim_dict['elec_y'][elec], color='b',\ marker='$%i$'%elec, markersize=20 ) # Plot the neuron xmid, ymid, zmid = np.load(folder + '/coor.npy') xstart, ystart, zstart = np.load(folder + '/coor_start.npy') xend, yend, zend = np.load(folder + '/coor_end.npy') diam = np.load(folder + '/diam.npy') length = np.load(folder + '/length.npy') n_compartments = len(diam) for comp in xrange(n_compartments): if comp == 0: xcoords = pl.array([xmid[comp]]) ycoords = pl.array([ymid[comp]]) zcoords = pl.array([zmid[comp]]) diams = pl.array([diam[comp]]) else: if zmid[comp] < 0.400 and zmid[comp] > -.400: xcoords = pl.r_[ xcoords, pl.linspace(xstart[comp], xend[comp], length[comp] * 3 * 1000)] ycoords = pl.r_[ ycoords, pl.linspace(ystart[comp], yend[comp], length[comp] * 3 * 1000)] zcoords = pl.r_[ zcoords, pl.linspace(zstart[comp], zend[comp], length[comp] * 3 * 1000)] diams = pl.r_[ diams, pl.linspace(diam[comp], diam[comp], length[comp] * 3 * 1000)] argsort = pl.argsort(-xcoords) ax.scatter(zcoords[argsort], ycoords[argsort], s=20 * (diams[argsort] * 1000)**2, c=xcoords[argsort], edgecolors='none', cmap='gray') ax_all.plot(zmid[0], ymid[0], marker='$%i$' % i, markersize=20, label='%i: %s' % (i, neur)) #legends.append('%i: %s' %(i, neur)) ax.axis(ext_sim_dict['plot_range']) ax.axis('equal') ax.axis(ext_sim_dict['plot_range']) ax.set_xlabel('z [mm]') ax.set_ylabel('y [mm]') fig.savefig(os.path.join(neural_sim_dict['output_folder'],\ 'neuron_figs', '%s.png' % neur)) ax_all.axis('equal') ax.axis(ext_sim_dict['plot_range']) ax_all.set_xlabel('z [mm]') ax_all.set_ylabel('y [mm]') ax_all.legend() fig_all.savefig(os.path.join(neural_sim_dict['output_folder'], 'fig.png'))
def __init__(self, logger=None, debug=True): hostname = S.socket.gethostname() self.ip = '192.168.168.233' # IP address of az/el arduino self.port = 4321 self.debug = debug self.lock = threading.Lock() self.setLogger(logger) # az conversion self.step2AngleAz = lambda step: 360. * step / 4800. self.angle2StepAz = lambda angle: int(angle * 4800. / 360) # for reference Az #spd = 12. # nominal observing speed in deg/s. hardwired in Arduino #usteps_per_step = 8. #steps_per_motor_rev = 200. #gear_ratio = 3. #usteps_per_peri_rev = usteps_per_step * steps_per_motor_rev * gear_ratio # el conversion stepPole = array([ 0., 100., 200., 300., 400., 500., 600., 700., 800., 900., 1000., 1100., 1200., 1300., 1400., 1500., 1600., 1700., 1800., 1900., 2000., 2100., 2200., 2300., 2400., 2500., 2600., 2700., 2800., 2900. ]) # new Nov 2017 calibration results # measured up and down going. Using only down going # fit a cubic polynomial to angle and interpolated to stepPole positions # to remove measurement error. anglePole = array([ 90.2, 87., 83.9, 80.9, 78., 75.2, 72.4, 69.7, 67., 64.4, 61.8, 59.3, 56.8, 54.3, 51.8, 49.4, 46.9, 44.5, 42., 39.5, 37.1, 34.5, 32., 29.4, 26.8, 24.1, 21.4, 18.6, 15.8, 12.9 ]) # used for 2015 and 2016. New angle calibration in Nov 2017. # anglePole = array([93.75, 90.45, 87.8, 84.95, 82.15, # 79.45 , 76.85, 74.3, 71.85, 69.4, 67., # 64.65, 62.25, 60.05, 57.65, 55.35, 53.15, # 50.85, 48.55, 46.2, 43.95, 41.7, 39.3, 36.95, # 34.5, 32.2, 29.6, 27.05, 24.6, # 21.8, 19.2, 16.55, 13.8, 11.15, 8.94]) stepSummit = array([ 0., 100., 200., 300., 400., 500., 600., 700., 800., 900., 1000., 1100., 1200., 1300., 1400., 1500., 1600., 1700., 1800., 1900., 2000., 2100., 2200., 2300., 2400., 2500., 2600., 2700., 2800., 2900., 3000., 3100., 3200. ]) angleSummit = array([ 92.6, 88.85, 85.35, 82.2, 79.15, 76.35, 73.7, 70.7, 67.95, 65.25, 62.85, 60.35, 57.7, 55.25, 52.65, 50.25, 47.6, 45.2, 42.7, 40.15, 37.5, 35.0, 32.5, 29.55, 27.2, 24.3, 21.25, 18.5, 15.6, 12.5, 9.45, 6.0, 2.65 ]) if 'wvr1' in hostname: angle = anglePole step = stepPole elif 'wvr2' in hostname: angle = angleSummit step = stepSummit else: angle = angleSummit step = stepSummit q = argsort(angle) self.step2AngleEl = scipy.interpolate.interp1d(step, angle, kind='linear') self.angle2StepEl = scipy.interpolate.interp1d(angle[q], step[q], kind='linear') self.initPort()
def scatter_dists(models_df, outdir=FIGS_DIR): df = models_df[['plant', 'model', 'dist']] model_dists = defaultdict(list) unique_plants = 0 for name, group in df.groupby('plant'): if len(group['model'].unique()) != 4: print group for model, group2 in group.groupby('model'): #group2 = group2.head(n=20) model_dists[model].append(pylab.mean(group2['dist'])) unique_plants += 1 print "-------------" print "unique scatter plants", unique_plants print "-------------" order = pylab.argsort(model_dists['plant']) model_colors = {'plant' : 'r', 'centroid' : 'g', 'random' : 'm', 'barabasi' : 'c'} model_markers = {'plant' : 'x', 'centroid' : 'o', 'random' : '^', 'barabasi' : 's'} model_labels = {'plant': 'Plant arbor', 'centroid' : 'Centroid', 'random' : 'Random', 'barabasi' : 'Barabasi-Albert'} max_dist = float('-inf') min_dist = float('inf') pylab.figure() sns.set() for model, dists in model_dists.iteritems(): dists = pylab.array(dists) y = dists[order] if LOG_DIST: y = pylab.log10(y) #y = y[::5] x = pylab.arange(len(y)) max_dist = max(max_dist, max(y)) min_dist = min(min_dist, min(y)) color = model_colors[model] marker = model_markers[model] label = model_labels[model] pylab.scatter(x, y, label=label, c=color, marker=marker) pylab.xlabel('plant index', fontsize=20) ylab = 'distance to Pareto front' if LOG_DIST: ylab = 'log(' + ylab + ')' pylab.ylabel(ylab, fontsize=20) pylab.xticks(fontsize=20) pylab.yticks(fontsize=20) leg = pylab.legend(ncol=2, frameon=True) leg.get_frame().set_linewidth(5) leg.get_frame().set_edgecolor('k') ax = pylab.gca() pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text pylab.ylim(min_dist - 0.1, max_dist + 0.6) pylab.tight_layout() pylab.savefig('%s/pareto_dists.pdf' % outdir, format='pdf')
def Main(): options, _ = MakeOpts().parse_args(sys.argv) assert options.experiment_id and options.plate_ids and options.reading_label plates = map(str.strip, options.plate_ids.split(',')) print 'Reading plates %s from experiment %s' % (', '.join(plates), options.experiment_id) db = MySQLDatabase(host='hldbv02', user='******', passwd='a1a1a1', db='tecan') print 'Calculating growth rates' growth_calc = growth.SlidingWindowGrowthCalculator(window_size=options.window_size, minimum_level=options.lower_bound, maximum_level=options.upper_bound) plate_names = {'1': 'Glucose', '2': 'Gluconate'} colormap = {'1': 'k', '2': 'r'} f1 = pylab.figure(0) for plate_id in plates: p = Plate96.FromDatabase(db, options.experiment_id, plate_id) rates, unused_stationaries = growth_calc.CalculatePlateGrowth( p, options.reading_label) mean_rates = MeanWithConfidenceIntervalDict(rates) means = [] errors = [] concs = [] for label in mean_rates.keys(): conc = TryFloat(label) if conc is False: continue mean, error = mean_rates[label] means.append(mean) errors.append(error) concs.append(conc) means = pylab.array(means) errors = pylab.array(errors) concs = pylab.array(concs) max_mean = max(means) norm_means = means / max_mean norm_errors = errors / max_mean label = plate_names[plate_id] color = colormap[plate_id] """ pylab.subplot(121) pylab.plot(concs, norm_means, color=color, linestyle='None', marker='.', label=label) pylab.errorbar(concs, norm_means, yerr=norm_errors, ecolor=color, fmt=None) """ #pylab.subplot(122) pcts = concs*100 idx = pylab.find(pcts > 1e-4) my_pcts = pcts[idx] my_means = means[idx] my_errs = errors[idx] order = pylab.argsort(my_pcts) my_pcts = my_pcts[order] my_means = my_means[order] my_errs = my_errs[order] pylab.plot(my_pcts, my_means, color=color, linestyle='None', linewidth=4, marker='.', markersize=15, label=label) pylab.errorbar(my_pcts, my_means, yerr=my_errs, ecolor=color, fmt=None, linewidth=1) smoothed = smoothing.WeightedAverageSmoother(pylab.log(my_pcts), my_means, sigma=0.7) log_xs = pylab.arange(pylab.log(1e-4), pylab.log(2.2), 1e-3) xs = pylab.exp(log_xs) ys = smoothed(log_xs) pylab.plot(xs, ys, color=color, linewidth=3, linestyle='--') """ pylab.subplot(121) pylab.xlabel('CAP Concentration (fraction of standard concentration)') pylab.ylabel('Relative Specific Growth Rate (/hour)') pylab.xlim(-0.1,0.2) """ """ pylab.subplot(122) pylab.xlabel('CAP Concentration (fraction of standard concentration)') pylab.ylabel('Absolute Specific Growth Rate (/hour)') """ pylab.xscale('log') pylab.xlabel('Substrate concentration (m/v %)') pylab.ylabel('Specific Growth Rate (/hour)') #pylab.xlim(-0.1,0.2) pylab.legend(loc='upper left') pylab.show()
def Main(): options, _ = MakeOpts().parse_args(sys.argv) assert options.experiment_id and options.plate_ids and options.reading_label plates = map(str.strip, options.plate_ids.split(',')) print 'Reading plates %s from experiment %s' % (', '.join(plates), options.experiment_id) db = MySQLDatabase(host='hldbv02', user='******', passwd='a1a1a1', db='tecan') print 'Calculating growth rates' growth_calc = growth.SlidingWindowGrowthCalculator( window_size=options.window_size, minimum_level=options.lower_bound, maximum_level=options.upper_bound) plate_names = {'1': 'Glucose', '2': 'Gluconate'} colormap = {'1': 'k', '2': 'r'} f1 = pylab.figure(0) for plate_id in plates: p = Plate96.FromDatabase(db, options.experiment_id, plate_id) rates, unused_stationaries = growth_calc.CalculatePlateGrowth( p, options.reading_label) mean_rates = MeanWithConfidenceIntervalDict(rates) means = [] errors = [] concs = [] for label in mean_rates.keys(): conc = TryFloat(label) if conc is False: continue mean, error = mean_rates[label] means.append(mean) errors.append(error) concs.append(conc) means = pylab.array(means) errors = pylab.array(errors) concs = pylab.array(concs) max_mean = max(means) norm_means = means / max_mean norm_errors = errors / max_mean label = plate_names[plate_id] color = colormap[plate_id] """ pylab.subplot(121) pylab.plot(concs, norm_means, color=color, linestyle='None', marker='.', label=label) pylab.errorbar(concs, norm_means, yerr=norm_errors, ecolor=color, fmt=None) """ #pylab.subplot(122) pcts = concs * 100 idx = pylab.find(pcts > 1e-4) my_pcts = pcts[idx] my_means = means[idx] my_errs = errors[idx] order = pylab.argsort(my_pcts) my_pcts = my_pcts[order] my_means = my_means[order] my_errs = my_errs[order] pylab.plot(my_pcts, my_means, color=color, linestyle='None', linewidth=4, marker='.', markersize=15, label=label) pylab.errorbar(my_pcts, my_means, yerr=my_errs, ecolor=color, fmt=None, linewidth=1) smoothed = smoothing.WeightedAverageSmoother(pylab.log(my_pcts), my_means, sigma=0.7) log_xs = pylab.arange(pylab.log(1e-4), pylab.log(2.2), 1e-3) xs = pylab.exp(log_xs) ys = smoothed(log_xs) pylab.plot(xs, ys, color=color, linewidth=3, linestyle='--') """ pylab.subplot(121) pylab.xlabel('CAP Concentration (fraction of standard concentration)') pylab.ylabel('Relative Specific Growth Rate (/hour)') pylab.xlim(-0.1,0.2) """ """ pylab.subplot(122) pylab.xlabel('CAP Concentration (fraction of standard concentration)') pylab.ylabel('Absolute Specific Growth Rate (/hour)') """ pylab.xscale('log') pylab.xlabel('Substrate concentration (m/v %)') pylab.ylabel('Specific Growth Rate (/hour)') #pylab.xlim(-0.1,0.2) pylab.legend(loc='upper left') pylab.show()
def cluster_imgs_pca_kmeans(imgpaths, bb_map=None, k=2, N=3, do_align=True): """ Using PCA and K-means, cluster the imgpaths into 'k' clusters, using the first 'N' principal components. Algorithm details: Input: Set of image patches A, of size NxM 0.) Discretize the image patch into K N'xM' equally-sized slices. 1.) Using the discretized image patches A', run PCA to extract the slices S that maximize the variance 2.) Run k-means (k=2) on the slices S. Input: list imgpaths: (imgpath_i, ...) dict bb_map: If you want to only cluster based on a sub-region of each image, pass in 'bb_map', which is: {str imgpath: (y1,y2,x1,x2)} int k: number of clusters int N: Number of principle components to use. (NOT USED) Output: dict clusters, maps {str clusterID: [imgpath_i, ...]} """ data = imgpaths_to_mat(imgpaths, bb_map=bb_map, do_align=do_align) ''' if bb_map is None: bb_map = {} h_big, w_big = get_largest_img_dims(imgpaths) else: bb_big = get_largest_bb(bb_map.values()) h_big = int(abs(bb_big[0] - bb_big[1])) w_big = int(abs(bb_big[2] - bb_big[3])) # TEMP: Cut off first half, last quarter # w_big = (w_big / 2) - (w_big / 4) # 0.) First, convert images into MxN array, where M is the number # of images, and N is the number of pixels of each image. data = np.zeros((len(imgpaths), h_big*w_big)) for row, imgpath in enumerate(imgpaths): img = scipy.misc.imread(imgpath, flatten=True) # TEMP: Cut off first half # img = img[:, img.shape[1]/2:(3*img.shape[1])/4] # img = util_gui.autothreshold_numpy(img, method="otsu") bb = bb_map.get(imgpath, None) if bb is None: patch = resize_mat(img, (h_big, w_big)) else: # Must make sure that all patches are the same shape. patch = resize_mat(img[bb[0]:bb[1], bb[2]:bb[3]], (h_big, w_big)) # Reshape 'patch' to be a single row of pixels, instead of rows # of pixels. patch = patch.reshape(1, patch.shape[0]*patch.shape[1]) data[row,:] = patch ''' # Inspiration for PCA-related code comes from: # http://glowingpython.blogspot.it/2011/07/pca-and-image-compression-with-numpy.html # 1.) Call PCA on the data matrix, extract first N principle comps M = (data - np.mean(data.T, axis=1)) # subtract mean, along cols (latent, coeff) = np.linalg.eig(np.cov(M)) p = np.size(coeff, axis=1) idx = pylab.argsort(latent) # sort eigenvalues idx = idx[::-1] # ascending order (i.e. by 'relevance') # idx is a sorted list of indices into imgpaths, i.e. if there # are 5 images, and idx is: # idx := [4, 1, 3, 2, 0] # then this means that imgpaths[4] most explains the variance, # followed by imgpaths[1], etc. idx = idx[:k] cluster_centers = data[idx, :] clustering = {} # maps {int clusterID: [imgpath_i, ...]} # 2.) Nearest-Neighbors to cluster_centers for i, imgarray in enumerate(data): best_dist, best_j = None, None for j, clustercenter in enumerate(cluster_centers): dist = np.linalg.norm(imgarray - clustercenter) if best_dist is None or dist < best_dist: best_dist = dist best_j = j clustering.setdefault(best_j, []).append(imgpaths[i]) return clustering
def vals_correlation(df, val1, val2, **kwargs): print "-----------------------------------------------------" print "%s-%s correlation" % (val1, val2) df2 = df.drop_duplicates(subset=['neuron name', 'neuron type']) if 'logtransform' in kwargs and kwargs['logtransform']: xtransform = pylab.log10 ytransform = pylab.log10 df2 = df2[(df2[val1] > 0) & (df2[val2] > 0)] v1 = df2[val1] v2 = df2[val2] xtransform = None ytransform = None if 'xtransform' in kwargs: xtransform = kwargs['xtransform'] if 'ytransform' in kwargs: ytransform = kwargs['ytransform'] if xtransform != None: v1 = xtransform(v1) if ytransform != None: v2 = ytransform(v2) print pearsonr(v1, v2) print spearmanr(v1, v2) regression_df = df2.copy() add_regression_cols(regression_df, val1, val2, xtransform=xtransform,\ ytransform=ytransform) grouping = None if 'grouping' in kwargs: grouping = kwargs['grouping'] else: grouping = 'neuron type' grouping_subset = None if 'grouping_subset' in kwargs: assert 'grouping' in kwargs grouping_subset = kwargs['grouping_subset'] else: #grouping_subset = ['axon', 'truncated axon', 'apical dendrite', 'basal dendrite'] grouping_subset = df2[grouping].unique() sns.set() pylab.figure() nrows = len(grouping_subset) + 1 pylab.subplot(nrows, 1, 1) pylab.scatter(v1, v2) x = v1 y = regression_df['%s_hat' % val2] order = pylab.argsort(x) x = x[order] y = y[order] pylab.plot(x, y, c='g') row = 2 df3 = df2[df2[grouping].isin(grouping_subset)] for name, group in df3.groupby(grouping): print name pylab.subplot(nrows, 1, row) row += 1 v1 = pylab.array(group[val1]) v2 = pylab.array(group[val2]) if xtransform != None: v1 = xtransform(v1) if ytransform != None: v2 = ytransform(v2) print pearsonr(v1, v2) print spearmanr(v1, v2) regression_df = group.copy() add_regression_cols(regression_df, val1, val2, xtransform=xtransform,\ ytransform=ytransform) pylab.scatter(v1, v2) x = pylab.array(v1) y = pylab.array(regression_df['%s_hat' % val2]) order = pylab.argsort(x) x = x[order] y = y[order] pylab.plot(x, y, c='g') pylab.tight_layout() outdir = None if 'outdir' in kwargs: outdir = kwargs['outdir'] else: outdir = FIGS_DIR figname = '%s/%s_%s.pdf' % (outdir, val1, val2) pylab.savefig('%s/%s_%s.pdf' % (outdir, val1, val2), format='pdf') pylab.close()
def scatter_dists(models_df, outdir=FIGS_DIR, subset=False): df = models_df[['neuron name', 'neuron type', 'model', 'dist']] model_dists = defaultdict(list) unique_neurons = 0 for name, group in df.groupby(['neuron name', 'neuron type']): if len(group['model'].unique()) != 4: print group continue unique_neurons += 1 if subset and unique_neurons % 5 != 0: continue for model, group2 in group.groupby('model'): #group2 = group2.head(n=20) model_dists[model].append(pylab.mean(group2['dist'])) print "-------------" print "unique scatter neurons", unique_neurons print "-------------" order = pylab.argsort(model_dists['neural']) #x = pylab.arange(len(model_dists['neural'])) pylab.figure() sns.set() model_colors = { 'neural': 'r', 'centroid': 'g', 'random': 'm', 'barabasi': 'c' } model_markers = { 'neural': 'x', 'centroid': 'o', 'random': '^', 'barabasi': 's' } model_labels = { 'neural': 'Neural arbor', 'centroid': 'Centroid', 'random': 'Random', 'barabasi': u'Barab\u00E1si-Albert' } max_dist = float('-inf') plot_order = ['random', 'barabasi', 'centroid', 'neural'] for model in plot_order: dists = model_dists[model] dists = pylab.array(dists) y = dists[order] if LOG_DIST: y = pylab.log10(y) #y = y[::5] x = pylab.arange(len(y)) max_dist = max(max_dist, max(y)) color = model_colors[model] marker = model_markers[model] label = model_labels[model] pylab.scatter(x, y, label=label, c=color, marker=marker) pylab.xlabel('neural arbor index', fontsize=20) ylab = 'distance to Pareto front' if LOG_DIST: ylab = 'log(' + ylab + ')' pylab.ylabel(ylab, fontsize=20) leg = pylab.legend(ncol=2, frameon=True) leg.get_frame().set_linewidth(5) leg.get_frame().set_edgecolor('k') ax = pylab.gca() pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text pylab.ylim(-0.1, max_dist + 0.9) pylab.xticks(fontsize=15, rotation=75) pylab.yticks(fontsize=15) fname = 'pareto_dists' if subset: fname += '_subset' pylab.tight_layout() pylab.savefig('%s/%s.pdf' % (outdir, fname), format='pdf')
# and how much each branch of the fil overlaps each elongated dendro nbranches = fils.filaments[ifil].branch_properties['number'] pixoverlapbranches = pl.zeros([nbranches, n_elong_dendro], dtype=int) for iz in range(n_elong_dendro): # dendro indices relative to fil. subarray drelx = d[zelong[iz]].indices()[2] - off[0][1] drely = d[zelong[iz]].indices()[1] - off[0][0] zz = pl.where((drelx >= 0) * (drely >= 0) * (drelx < sfarr[1]) * (drely < sfarr[0]))[0] if len(zz) > 0: pixoverlap[iz] = farr[drely[zz], drelx[zz]].sum() farr_overlap[drely[zz], drelx[zz]] = farr[drely[zz], drelx[zz]] zorder = pl.argsort(pixoverlap)[::-1] # now if any dendros overlap this entire fil, if pixoverlap[zorder[0]] > 0: zoverlap = pl.where(pixoverlap[zorder] > 0)[0] # remember which dendros overlap: associated_dendros.append({ "indices": zelong[zorder[zoverlap]], "pixoverlap": pixoverlap[zorder[zoverlap]] }) pl.figure(2) pl.text(off[0][1], off[0][0], ifil, color="m") # go through overlapping dendros and analyze overlap w/subbranches from _labeled_mask thisbranchlist = [] # TODO add longest path from subbranch at least to a hub also?
def main(): import optparse from numpy import sum # Parse command line parser = optparse.OptionParser(usage=USAGE) parser.add_option("-p", "--plot", action="store_true", help="Generate pdf with IR-spectrum") parser.add_option("-i", "--info", action="store_true", help="Set up/ Calculate vibrations & quit") parser.add_option("-s", "--suffix", action="store", help="Call suffix for binary e.g. 'mpirun -n 4 '", default='') parser.add_option("-r", "--run", action="store", help="path to FHI-aims binary", default='') parser.add_option("-x", "--relax", action="store_true", help="Relax initial geometry") parser.add_option("-m", "--molden", action="store_true", help="Output in molden format") parser.add_option("-w", "--distort", action="store_true", help="Output geometry distorted along imaginary modes") parser.add_option("-t", "--submit", action="store", help="""\ Path to submission script, string <jobname> will be replaced by name + counter, string <outfile> will be replaced by filename""") parser.add_option("-d", "--delta", action="store", type="float", help="Displacement", default=0.0025) options, args = parser.parse_args() if options.info: print __doc__ sys.exit(0) if len(args) != 2: parser.error("Need exactly two arguments") AIMS_CALL = options.suffix + ' ' + options.run hessian_thresh = -1 name = args[0] mode = args[1] delta = options.delta run_aims = False if options.run != '': run_aims = True submit_script = options.submit is not None if options.plot: import matplotlib as mpl mpl.use('Agg') from pylab import figure if options.plot or mode == '1': from pylab import savetxt, transpose, eig, argsort, sort,\ sign, pi, dot, sum, linspace, argmin, r_, convolve # Constant from scipy.constants bohr = constants.value('Bohr radius') * 1.e10 hartree = constants.value('Hartree energy in eV') at_u = constants.value('atomic mass unit-kilogram relationship') eV = constants.value('electron volt-joule relationship') c = constants.value('speed of light in vacuum') Ang = 1.0e-10 hbar = constants.value('Planck constant over 2 pi') Avo = constants.value('Avogadro constant') kb = constants.value('Boltzmann constant in eV/K') hessian_factor = eV / (at_u * Ang * Ang) grad_dipole_factor = (eV / (1. / (10 * c))) / Ang #(eV/Ang -> D/Ang) ir_factor = 1 # Asign all filenames inputgeomerty = 'geometry.in.' + name inputcontrol = 'control.in.' + name atomicmasses = 'masses.' + name + '.dat' xyzfile = name + '.xyz' moldenname = name + '.molden' hessianname = 'hessian.' + name + '.dat' graddipolename = 'grad_dipole.' + name + '.dat' irname = 'ir.' + name + '.dat' deltas = array([-delta, delta]) coeff = array([-1, 1]) c_zero = -1. / (2. * delta) f = open('control.in', 'r') # read control.in template template_control = f.read() f.close if submit_script: f = open(options.submit, 'r') # read submission script template template_job = f.read() f.close folder = '' # Dummy ########### Central Point ################################################## if options.relax and mode == '0': # First relax input geometry filename = name + '.out' folder = name + '_relaxation' if not os.path.exists(folder): os.mkdir(folder) # Create folder shutil.copy('geometry.in', folder + '/geometry.in') # Copy geometry new_control = open(folder + '/control.in', 'w') new_control.write(template_control + 'relax_geometry trm 1E-3\n') # Relax! new_control.close() os.chdir(folder) # Change directoy print 'Central Point' if run_aims: os.system(AIMS_CALL + ' > ' + filename) # Run aims and pipe the output # into a file named 'filename' if submit_script: replace_submission(template_job, name, 0, filename) os.chdir('..') ############################################################################ # Check for relaxed geometry if os.path.exists(folder + '/geometry.in.next_step'): geometry = open(folder + '/geometry.in.next_step', 'r') else: geometry = open('geometry.in', 'r') # Read input geometry n_line = 0 struc = structure() lines = geometry.readlines() for line in lines: n_line = n_line + 1 if line.rfind('set_vacuum_level') != -1: # Vacuum Level struc.vacuum_level = float(split_line(line)[-1]) if line.rfind('lattice_vector') != -1: # Lattice vectors and periodic lat = split_line(line)[1:] struc.lattic_vector = append(struc.lattic_vector, float64(array(lat))[newaxis, :], axis=0) struc.periodic = True if line.rfind('atom') != -1: # Set atoms line_vals = split_line(line) at = Atom(line_vals[-1], line_vals[1:-1]) if n_line < len(lines): nextline = lines[n_line] if nextline.rfind( 'constrain_relaxation') != -1: # constrained? at = Atom(line_vals[-1], line_vals[1:-1], True) else: at = Atom(line_vals[-1], line_vals[1:-1]) struc.join(at) geometry.close() n_atoms = struc.n() n_constrained = n_atoms - sum(struc.constrained) # Atomic mass file mass_file = open(atomicmasses, 'w') mass_vector = zeros([0]) for at_unconstrained in struc.atoms[struc.constrained == False]: mass_vector = append(mass_vector, ones(3) * 1. / sqrt(at_unconstrained.mass())) line = '{0:10.5f}'.format(at_unconstrained.mass()) for i in range(3): line = line + '{0:11.4f}'.format(at_unconstrained.coord[i]) line = line + '{0:}\n'.format(at_unconstrained.kind) mass_file.writelines(line) mass_file.close() # Init dip = zeros([n_constrained * 3, 3]) hessian = zeros([n_constrained * 3, n_constrained * 3]) index = 0 counter = 1 # Set up / Read folders for displaced atoms for atom in arange(n_atoms)[struc.constrained == False]: for coord in arange(3): for delta in deltas: filename=name+'.i_atom_'+str(atom)+'.i_coord_'+str(coord)+'.displ_'+\ str(delta)+'.out' folder=name+'.i_atom_'+str(atom)+'.i_coord_'+str(coord)+'.displ_'+\ str(delta) if mode == '0': # Put new geometry and control.in into folder struc_new = copy.deepcopy(struc) struc_new.atoms[atom].coord[coord]=\ struc_new.atoms[atom].coord[coord]+delta geoname='geometry.i_atom_'+str(atom)+'.i_coord_'+str(coord)+\ '.displ_'+str(delta)+'.in' if not os.path.exists(folder): os.mkdir(folder) new_geo = open(folder + '/geometry.in', 'w') newline='#\n# temporary structure-file for finite-difference '+\ 'calculation of forces\n' newline=newline+'# displacement {0:8.4f} of \# atom '.format(delta)+\ '{0:5} direction {1:5}\n#\n'.format(atom,coord) new_geo.writelines(newline + struc_new.to_str()) new_geo.close() new_control = open(folder + '/control.in', 'w') template_control = template_control.replace( 'relax_geometry', '#relax_geometry') new_control.write(template_control+'compute_forces .true. \n'+\ 'final_forces_cleaned '+\ '.true. \noutput dipole \n') new_control.close() os.chdir(folder) # Change directoy print 'Processing atom: '+str(atom+1)+'/'+str(n_atoms)+', coord.: '+\ str(coord+1)+'/'+str(3)+', delta: '+str(delta) if run_aims: os.system(AIMS_CALL + ' > ' + filename) # Run aims and pipe the output # into a file named 'filename' if submit_script: replace_submission(template_job, name, counter, filename) # os.system('qsub job.sh') # Mind the environment variables os.chdir('..') if mode == '1': # Read output forces_reached = False atom_count = 0 data = open(folder + '/' + filename) for line in data.readlines(): if line.rfind( 'Dipole correction potential jump') != -1: dip_jump = float(split_line(line)[-2]) # Periodic if line.rfind('| Total dipole moment [eAng]') != -1: dip_jump = float64( split_line(line)[-3:]) # Cluster if forces_reached and atom_count < n_atoms: # Read Forces struc.atoms[atom_count].force = float64( split_line(line)[2:]) atom_count = atom_count + 1 if atom_count == n_atoms: forces_reached = False if line.rfind('Total atomic forces') != -1: forces_reached = True data.close() if struc.periodic: dip[index, 2] = dip[ index, 2] + dip_jump * coeff[deltas == delta] * c_zero else: dip[index, :] = dip[index, :] + dip_jump * coeff[ deltas == delta] * c_zero forces = array([]) for at_unconstrained in struc.atoms[struc.constrained == False]: forces = append( forces, coeff[deltas == delta] * at_unconstrained.force) hessian[index, :] = hessian[index, :] + forces * c_zero counter = counter + 1 index = index + 1 if mode == '1': # Calculate vibrations print 'Entering hessian diagonalization' print 'Number of atoms = ' + str(n_atoms) print 'Name of Hessian input file = ' + hessianname print 'Name of grad dipole input file = ' + graddipolename print 'Name of Masses input file = ' + atomicmasses print 'Name of XYZ output file = ' + xyzfile print 'Threshold for Matrix elements = ' + str(hessian_thresh) if (hessian_thresh < 0.0): print ' All matrix elements are taken'+\ ' into account by default\n' savetxt(hessianname, hessian) savetxt(graddipolename, dip) mass_mat = mass_vector[:, newaxis] * mass_vector[newaxis, :] hessian[abs(hessian) < hessian_thresh] = 0.0 hessian = hessian * mass_mat * hessian_factor hessian = (hessian + transpose(hessian)) / 2. # Diagonalize hessian (scipy) print 'Solving eigenvalue system for Hessian Matrix' freq, eig_vec = eig(hessian) print 'Done ... ' eig_vec = eig_vec[:, argsort(freq)] freq = sort(sign(freq) * sqrt(abs(freq))) ZPE = hbar * (freq) / (2.0 * eV) freq = (freq) / (200. * pi * c) grad_dipole = dip * grad_dipole_factor eig_vec = eig_vec * mass_vector[:, newaxis] * ones( len(mass_vector))[newaxis, :] infrared_intensity = sum(dot(transpose(grad_dipole),eig_vec)**2,axis=0)*\ ir_factor reduced_mass = sum(eig_vec**2, axis=0) norm = sqrt(reduced_mass) eig_vec = eig_vec / norm # The rest is output, xyz, IR,... print 'Results\n' print 'List of all frequencies found:' print 'Mode number Frequency [cm^(-1)] Zero point energy [eV] '+\ 'IR-intensity [D^2/Ang^2]' for i in range(len(freq)): print '{0:11}{1:25.8f}{2:25.8f}{3:25.8f}'.format( i + 1, freq[i], ZPE[i], infrared_intensity[i]) print '\n' print 'Summary of zero point energy for entire system:' print '| Cumulative ZPE = {0:15.8f} eV'.format(sum(ZPE)) print '| without first six eigenmodes = {0:15.8f} eV\n'.format( sum(ZPE) - sum(ZPE[:6])) print 'Stability checking - eigenvalues should all be positive for a '+\ 'stable structure. ' print 'The six smallest frequencies should be (almost) zero:' string = '' for zz in ZPE[:6]: string = string + '{0:25.8f}'.format(zz) print string print 'Compare this with the largest eigenvalue, ' print '{0:25.8f}'.format(freq[-1]) nums = arange(n_atoms)[struc.constrained == False] nums2 = arange(n_atoms)[struc.constrained] newline = '' newline_ir = '[INT]\n' if options.molden: newline_molden = '[Molden Format]\n[GEOMETRIES] XYZ\n' newline_molden = newline_molden + '{0:6}\n'.format(n_atoms) + '\n' for i_atoms in range(n_constrained): newline_molden = newline_molden + '{0:6}'.format( struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline_molden = newline_molden + '{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord]) newline_molden = newline_molden + '\n' newline_molden = newline_molden + '[FREQ]\n' for i in range(len(freq)): newline_molden = newline_molden + '{0:10.3f}\n'.format(freq[i]) newline_molden = newline_molden + '[INT]\n' for i in range(len(freq)): newline_molden = newline_molden + '{0:17.6e}\n'.format( infrared_intensity[i]) newline_molden = newline_molden + '[FR-COORD]\n' newline_molden = newline_molden + '{0:6}\n'.format(n_atoms) + '\n' for i_atoms in range(n_constrained): newline_molden = newline_molden + '{0:6}'.format( struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline_molden = newline_molden + '{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord] / bohr) newline_molden = newline_molden + '\n' newline_molden = newline_molden + '[FR-NORM-COORD]\n' for i in range(len(freq)): newline = newline + '{0:6}\n'.format(n_atoms) if freq[i] > 0: newline = newline + 'stable frequency at ' elif freq[i] < 0: newline = newline + 'unstable frequency at ' if options.distort and freq[i] < -50: struc_new = copy.deepcopy(struc) for i_atoms in range(n_constrained): for i_coord in range(3): struc_new.atoms[i_atoms].coord[i_coord]=\ struc_new.atoms[i_atoms].coord[i_coord]+\ eig_vec[(i_atoms)*3+i_coord,i] geoname = name + '.distorted.vibration_' + str( i + 1) + '.geometry.in' new_geo = open(geoname, 'w') newline_geo = '#\n# distorted structure-file for based on eigenmodes\n' newline_geo=newline_geo+\ '# vibration {0:5} :{1:10.3f} 1/cm\n#\n'.format(i+1,freq[i]) new_geo.writelines(newline_geo + struc_new.to_str()) new_geo.close() elif freq[i] == 0: newline = newline + 'translation or rotation ' newline = newline + '{0:10.3f} 1/cm IR int. is '.format(freq[i]) newline = newline + '{0:10.4e} D^2/Ang^2; red. mass is '.format( infrared_intensity[i]) newline = newline + '{0:5.3f} a.m.u.; force const. is '.format( 1.0 / reduced_mass[i]) newline = newline + '{0:5.3f} mDyne/Ang.\n'.format( ((freq[i] * (200 * pi * c))**2) * (1.0 / reduced_mass[i]) * at_u * 1.e-2) if options.molden: newline_molden=newline_molden+\ 'vibration {0:6}\n'.format(i+1) for i_atoms in range(n_constrained): newline = newline + '{0:6}'.format( struc.atoms[nums[i_atoms]].kind) for i_coord in range(3): newline = newline + '{0:10.4f}'.format( struc.atoms[nums[i_atoms]].coord[i_coord]) for i_coord in range(3): newline = newline + '{0:10.4f}'.format( eig_vec[(i_atoms) * 3 + i_coord, i]) if options.molden: newline_molden = newline_molden + '{0:10.4f}'.format( eig_vec[(i_atoms) * 3 + i_coord, i] / bohr) newline = newline + '\n' if options.molden: newline_molden = newline_molden + '\n' for i_atoms in range(n_atoms - n_constrained): newline = newline + '{0:6}'.format( struc.atoms[nums2[i_atoms]].kind) for i_coord in range(3): newline = newline + '{0:10.4f}'.format( struc.atoms[nums2[i_atoms]].coord[i_coord]) for i_coord in range(3): newline = newline + '{0:10.4f}'.format(0.0) newline = newline + '\n' newline_ir = newline_ir + '{0:10.4e}\n'.format( infrared_intensity[i]) xyz = open(xyzfile, 'w') xyz.writelines(newline) xyz.close() ir = open(irname, 'w') ir.writelines(newline_ir) ir.close() if options.molden: molden = open(moldenname, 'w') molden.writelines(newline_molden) molden.close() if mode == '1' and options.plot: x = linspace(freq.min() - 500, freq.max() + 500, 1000) z = zeros(len(x)) for i in range(len(freq)): z[argmin(abs(x - freq[i]))] = infrared_intensity[i] window_len = 150 gauss = signal.gaussian(window_len, 10) s = r_[z[window_len - 1:0:-1], z, z[-1:-window_len:-1]] z_convolve = convolve(gauss / gauss.sum(), s, mode='same')[window_len - 1:-window_len + 1] fig = figure(0) ax = fig.add_subplot(111) ax.plot(x, z_convolve, 'r', lw=2) ax.set_xlim([freq.min() - 500, freq.max() + 500]) ax.set_ylim([-0.01, ax.get_ylim()[1]]) ax.set_yticks([]) ax.set_xlabel('Frequency [1/cm]', size=20) ax.set_ylabel('Intensity [a.u.]', size=20) fig.savefig(name + '_IR_spectrum.pdf') print '\n Done. '
def refine(self, edge_errors, gamma=1.4): """ This function iterates through the cells in the mesh, then refines the mesh based on the relative error and the cell's location in the mesh. :param edge_errors : Dolfin edge function containing edge errors of of the current mesh. :param gamma : Scaling factor for determining which edges need be refined. This is determined by the average error of the edge_errors variable """ mesh = self.mesh mesh.init(1,2) mesh.init(0,2) mesh.init(0,1) avg_error = edge_errors.array().mean() error_sorted_edge_indices = p.argsort(edge_errors.array())[::-1] refine_edge = FacetFunction('bool', mesh) for e in edges(mesh): refine_edge[e] = edge_errors[e] > gamma*avg_error coordinates = p.copy(self.mesh.coordinates()) current_new_vertex = len(coordinates) cells_to_delete = [] new_cells = [] for iteration in range(refine_edge.array().sum()): for e in facets(self.mesh): if refine_edge[e] and (e.index()==error_sorted_edge_indices[0]): adjacent_cells = e.entities(2) adjacent_vertices = e.entities(0) if not any([c in cells_to_delete for c in adjacent_cells]): new_x,new_y = e.midpoint().x(),e.midpoint().y() coordinates = p.vstack((coordinates,[new_x,new_y])) for c in adjacent_cells: off_facet_vertex = list(self.mesh.cells()[c]) [off_facet_vertex.remove(ii) for ii in adjacent_vertices] for on_facet_vertex in adjacent_vertices: new_cell = p.sort([current_new_vertex,off_facet_vertex[0],on_facet_vertex]) new_cells.append(new_cell) cells_to_delete.append(c) current_new_vertex+=1 error_sorted_edge_indices = error_sorted_edge_indices[1:] old_cells = self.mesh.cells() keep_cell = p.ones(len(old_cells)) keep_cell[cells_to_delete] = 0 old_cells_parsed = old_cells[keep_cell.astype('bool')] all_cells = p.vstack((old_cells_parsed,new_cells)) n_cells = len(all_cells) e = MeshEditor() refined_mesh = Mesh() e.open(refined_mesh,self.mesh.geometry().dim(),self.mesh.topology().dim()) e.init_vertices(current_new_vertex) for index,x in enumerate(coordinates): e.add_vertex(index,x[0],x[1]) e.init_cells(n_cells) for index,c in enumerate(all_cells): e.add_cell(index,c.astype('uintc')) e.close() refined_mesh.order() self.mesh = refined_mesh
def sort_atoms(self): """Sort the atoms in the cell according to atomic symbol.""" ind = m.argsort(self.atom_symbols) self.atom_symbols = m.array(self.atom_symbols)[ind] self.atoms = self.atoms[ind] self.selective_flags = m.array(self.selective_flags)[ind]
def entropy(points, logp, N_entropy=10000, N_norm=2500): r""" Return entropy estimate and uncertainty from a random sample. *points* is a set of draws from an underlying distribution, as returned by a Markov chain Monte Carlo process for example. *logp* is the log-likelihood for each draw. *N_norm* is the number of points $k$ to use to estimate the posterior density normalization factor $P(D) = \hat N$, converting from $\log( P(D|M) P(M) )$ to $\log( P(D|M)P(M)/P(D) )$. The relative uncertainty $\Delta\hat S/\hat S$ scales with $\sqrt{k}$, with the default *N_norm=2500* corresponding to 2% relative uncertainty. Computation cost is $O(nk)$ where $n$ is number of points in the draw. *N_entropy* is the number of points used to estimate the entropy $\hat S = - \int P(M|D) \log P(M|D)$ from the normalized log likelihood values. """ # Use a random subset to estimate density if N_norm >= len(logp): norm_points = points else: idx = permutation(len(points))[:N_entropy] norm_points = points[idx] # Use a different subset to estimate the scale factor between density # and logp. if N_entropy is None: N_entropy = 10000 if N_entropy >= len(logp): entropy_points, eval_logp = points, logp else: idx = permutation(len(points))[:N_entropy] entropy_points, eval_logp = points[idx], logp[idx] """ # Try again, just using the points from the high probability regions # to determine the scale factor N_norm = min(len(logp), 5000) N_entropy = int(0.8*N_norm) idx = np.argsort(logp) norm_points = points[idx[-N_norm:]] entropy_points = points[idx[-N_entropy:]] eval_logp = logp[idx[-N_entropy:]] """ # Normalize p to a peak probability of 1 so that exp() doesn't underflow. # # This should be okay since for the normalizing constant C: # # u' = e^(ln u + ln C) = e^(ln u)e^(ln C) = C u # # Using eq. 11 of Kramer with u' substituted for u: # # N_est = < u'/p > = < C u/p > = C < u/p > # # S_est = - < ln q > # = - < ln (u'/N_est) > # = - < ln C + ln u - ln (C <u/p>) > # = - < ln u + ln C - ln C - ln <u/p> > # = - < ln u - ln <u/p> > # = - < ln u > + ln <u/p> # # Uncertainty comes from eq. 13: # # N_err^2 = 1/(k-1) sum( (u'/p - <u'/p>)^2 ) # = 1/(k-1) sum( (C u/p - <C u/p>)^2 ) # = C^2 std(u/p)^2 # S_err = std(u'/p) / <u'/p> = (C std(u/p))/(C <u/p>) = std(u/p)/<u/p> # # So even though the constant C shows up in N_est, N_err, it cancels # again when S_est, S_err is formed. log_scale = np.max(eval_logp) # print("max log sample: %g"%log_scale) eval_logp -= log_scale # Compute entropy and uncertainty in nats # Note: if all values are the same in any dimension then we have a dirac # functional with infinite probability at every sample point, and the # differential entropy estimate will yield H = -inf. rho = density(norm_points, entropy_points) #print(rho.min(), rho.max(), eval_logp.min(), eval_logp.max()) frac = exp(eval_logp) / rho n_est, n_err = mean(frac), std(frac) if n_est == 0.: s_est, s_err = -np.inf, 0. else: s_est = log(n_est) - mean(eval_logp) s_err = n_err / n_est #print(n_est, n_err, s_est/LN2, s_err/LN2) ##print(np.median(frac), log(np.median(frac))/LN2, log(n_est)/LN2) if False: import pylab idx = pylab.argsort(entropy_points[:, 0]) pylab.figure() pylab.subplot(221) pylab.hist(points[:, 0], bins=50, normed=True, log=True) pylab.plot(entropy_points[idx, 0], rho[idx], label='density') pylab.plot(entropy_points[idx, 0], exp(eval_logp + log_scale)[idx], label='p') pylab.ylabel("p(x)") pylab.legend() pylab.subplot(222) pylab.hist(points[:, 0], bins=50, normed=True, log=False) pylab.plot(entropy_points[idx, 0], rho[idx], label='density') pylab.plot(entropy_points[idx, 0], exp(eval_logp + log_scale)[idx], label='p') pylab.ylabel("p(x)") pylab.legend() pylab.subplot(212) pylab.plot(entropy_points[idx, 0], frac[idx], '.') pylab.xlabel("P[0] value") pylab.ylabel("p(x)/kernel density") # return entropy and uncertainty in bits return s_est / LN2, s_err / LN2
def cluster_imgs_pca_kmeans(imgpaths, bb_map=None, k=2, N=3, do_align=True): """ Using PCA and K-means, cluster the imgpaths into 'k' clusters, using the first 'N' principal components. Algorithm details: Input: Set of image patches A, of size NxM 0.) Discretize the image patch into K N'xM' equally-sized slices. 1.) Using the discretized image patches A', run PCA to extract the slices S that maximize the variance 2.) Run k-means (k=2) on the slices S. Input: list imgpaths: (imgpath_i, ...) dict bb_map: If you want to only cluster based on a sub-region of each image, pass in 'bb_map', which is: {str imgpath: (y1,y2,x1,x2)} int k: number of clusters int N: Number of principle components to use. (NOT USED) Output: dict clusters, maps {str clusterID: [imgpath_i, ...]} """ data = imgpaths_to_mat(imgpaths, bb_map=bb_map, do_align=do_align) ''' if bb_map == None: bb_map = {} h_big, w_big = get_largest_img_dims(imgpaths) else: bb_big = get_largest_bb(bb_map.values()) h_big = int(abs(bb_big[0] - bb_big[1])) w_big = int(abs(bb_big[2] - bb_big[3])) #TEMP: Cut off first half, last quarter #w_big = (w_big / 2) - (w_big / 4) # 0.) First, convert images into MxN array, where M is the number # of images, and N is the number of pixels of each image. data = np.zeros((len(imgpaths), h_big*w_big)) for row, imgpath in enumerate(imgpaths): img = scipy.misc.imread(imgpath, flatten=True) # TEMP: Cut off first half #img = img[:, img.shape[1]/2:(3*img.shape[1])/4] #img = util_gui.autothreshold_numpy(img, method="otsu") bb = bb_map.get(imgpath, None) if bb == None: patch = resize_mat(img, (h_big, w_big)) else: # Must make sure that all patches are the same shape. patch = resize_mat(img[bb[0]:bb[1], bb[2]:bb[3]], (h_big, w_big)) # Reshape 'patch' to be a single row of pixels, instead of rows # of pixels. patch = patch.reshape(1, patch.shape[0]*patch.shape[1]) data[row,:] = patch ''' # Inspiration for PCA-related code comes from: # http://glowingpython.blogspot.it/2011/07/pca-and-image-compression-with-numpy.html # 1.) Call PCA on the data matrix, extract first N principle comps M = (data - np.mean(data.T, axis=1)) # subtract mean, along cols (latent, coeff) = np.linalg.eig(np.cov(M)) p = np.size(coeff, axis=1) idx = pylab.argsort(latent) # sort eigenvalues idx = idx[::-1] # ascending order (i.e. by 'relevance') # idx is a sorted list of indices into imgpaths, i.e. if there # are 5 images, and idx is: # idx := [4, 1, 3, 2, 0] # then this means that imgpaths[4] most explains the variance, # followed by imgpaths[1], etc. idx = idx[:k] cluster_centers = data[idx, :] clustering = {} # maps {int clusterID: [imgpath_i, ...]} # 2.) Nearest-Neighbors to cluster_centers for i, imgarray in enumerate(data): best_dist, best_j = None, None for j, clustercenter in enumerate(cluster_centers): dist = np.linalg.norm(imgarray - clustercenter) if best_dist == None or dist < best_dist: best_dist = dist best_j = j clustering.setdefault(best_j, []).append(imgpaths[i]) return clustering
tmp1 = str.split(lines[0], ',') tmp2 = [] for val in tmp1: tmp2.append(float(val)) N_ch = len(tmp2) freq = zeros(N_ch) for k in range(0, N_ch): freq[k] = tmp2[k] ch = zeros(N_ch) for k in range(0, N_ch): ch[k] = freq[k] / 0.01 freq_idx = argsort(freq) if args.phases: chan_sel = 2 + 2 * array(freq_idx) else: chan_sel = 1 + 2 * array(freq_idx) for lineIndex in range(1, len(lines), 8): # get 0, 8, 16, ... tmp = str.split(lines[lineIndex + 0], ',') PX_lsq.append([]) PX_lsq[-1] = zeros(N_ch * 2 + 1) for k in range(0, len(tmp)): PX_lsq[-1][k] = float(tmp[k]) tmp = str.split(lines[lineIndex + 1], ',') PX_fit.append([])
def vals_correlation(df, val1, val2, outdir=FIGS_DIR, xtransform=None,\ ytransform=None, logtransform=False): print "-----------------------------------------------------" print "%s-%s correlation" % (val1, val2) df2 = df.drop_duplicates(subset=['neuron name', 'neuron type']) if logtransform: xtransform = pylab.log10 ytransform = pylab.log10 df2 = df2[(df2[val1] > 0) & (df2[val2] > 0)] v1 = df2[val1] v2 = df2[val2] if xtransform != None: v1 = xtransform(v1) if ytransform != None: v2 = ytransform(v2) print pearsonr(v1, v2) print spearmanr(v1, v2) regression_df = df2.copy() add_regression_cols(regression_df, val1, val2, xtransform=xtransform,\ ytransform=ytransform) sns.set() pylab.figure() nrows = len(df2['neuron type'].unique()) + 1 pylab.subplot(nrows, 1, 1) pylab.scatter(v1, v2) x = v1 y = regression_df['%s_hat' % val2] order = pylab.argsort(x) x = x[order] y = y[order] pylab.plot(x, y, c='g') row = 2 for neuron_type, group in df2.groupby('neuron type'): print neuron_type pylab.subplot(nrows, 1, row) row += 1 v1 = pylab.array(group[val1]) v2 = pylab.array(group[val2]) if xtransform != None: v1 = xtransform(v1) if ytransform != None: v2 = ytransform(v2) print pearsonr(v1, v2) print spearmanr(v1, v2) regression_df = group.copy() add_regression_cols(regression_df, val1, val2, xtransform=xtransform,\ ytransform=ytransform) pylab.scatter(v1, v2) x = pylab.array(v1) y = pylab.array(regression_df['%s_hat' % val2]) order = pylab.argsort(x) x = x[order] y = y[order] pylab.plot(x, y, c='g') pylab.tight_layout() figname = '%s/%s_%s.pdf' % (outdir, val1, val2) pylab.savefig('%s/%s_%s.pdf' % (outdir, val1, val2), format='pdf') pylab.close()
figure = pylab.Figure() for simulator in simulators: for num_nodes in nodes: col = 1 subplot = figure.add_axes([x,y0+2.9*dy,w,h]) subplot.set_title("%s (np%d)" % (simulator[:6].upper(),num_nodes), fontsize='x-large') subplot.set_ylabel("Membrane potential (mV)") # Get info about dataset from header of .v file exec(get_header("Results/VAbenchmark_%s_exc_%s_np%d.v" % (benchmark, simulator, num_nodes))) # Plot membrane potential trace allvdata = numpy.loadtxt("Results/VAbenchmark_%s_exc_%s_np%d.v" % (benchmark, simulator, num_nodes), comments='#') cell_ids = allvdata[:,1].astype(int) allvdata = allvdata[:,0] sortmap = pylab.argsort(cell_ids, kind='mergesort') cell_ids = pylab.take(cell_ids,sortmap) allvdata = pylab.take(allvdata,sortmap) for i in 0,1: tdata = pylab.arange(0,(n+1)*dt,dt) vdata = allvdata.compress(cell_ids==i) vdata = pylab.where(vdata>=v_thresh-0.05,0.0,vdata) # add fake APs for plotting if len(tdata) > len(vdata): print "Warning. Shortening tdata from %d to %d elements (%s)" % (len(tdata),len(vdata),simulator) tdata = tdata[0:len(vdata)] assert len(tdata)==len(vdata), "%d != %d (%s)" % (len(tdata),len(vdata),simulator) subplot.plot(tdata,vdata) # Plot spike rasters subplot = figure.add_axes([x,y0+2*dy,w,h]) exc_spikedata = signals.load_spikelist("Results/VAbenchmark_%s_exc_%s_np%d.ras" % (benchmark, simulator, num_nodes))
def refine(self, edge_errors, gamma=1.4): """ This function iterates through the cells in the mesh, then refines the mesh based on the relative error and the cell's location in the mesh. :param edge_errors : Dolfin edge function containing edge errors of of the current mesh. :param gamma : Scaling factor for determining which edges need be refined. This is determined by the average error of the edge_errors variable """ mesh = self.mesh mesh.init(1, 2) mesh.init(0, 2) mesh.init(0, 1) avg_error = edge_errors.array().mean() error_sorted_edge_indices = pl.argsort(edge_errors.array())[::-1] refine_edge = FacetFunction('bool', mesh) for e in edges(mesh): refine_edge[e] = edge_errors[e] > gamma * avg_error coordinates = pl.copy(self.mesh.coordinates()) current_new_vertex = len(coordinates) cells_to_delete = [] new_cells = [] for iteration in range(refine_edge.array().sum()): for e in facets(self.mesh): if refine_edge[e] and (e.index() == error_sorted_edge_indices[0]): adjacent_cells = e.entities(2) adjacent_vertices = e.entities(0) if not any([c in cells_to_delete for c in adjacent_cells]): new_x, new_y = e.midpoint().x(), e.midpoint().y() coordinates = pl.vstack((coordinates, [new_x, new_y])) for c in adjacent_cells: off_facet_vertex = list(self.mesh.cells()[c]) [ off_facet_vertex.remove(ii) for ii in adjacent_vertices ] for on_facet_vertex in adjacent_vertices: new_cell = pl.sort([ current_new_vertex, off_facet_vertex[0], on_facet_vertex ]) new_cells.append(new_cell) cells_to_delete.append(c) current_new_vertex += 1 error_sorted_edge_indices = error_sorted_edge_indices[1:] old_cells = self.mesh.cells() keep_cell = pl.ones(len(old_cells)) keep_cell[cells_to_delete] = 0 old_cells_parsed = old_cells[keep_cell.astype('bool')] all_cells = pl.vstack((old_cells_parsed, new_cells)) n_cells = len(all_cells) e = MeshEditor() refined_mesh = Mesh() e.open(refined_mesh, self.mesh.geometry().dim(), self.mesh.topology().dim()) e.init_vertices(current_new_vertex) for index, x in enumerate(coordinates): e.add_vertex(index, x[0], x[1]) e.init_cells(n_cells) for index, c in enumerate(all_cells): e.add_cell(index, c.astype('uintc')) e.close() refined_mesh.order() self.mesh = refined_mesh
for key in keys[1:]: mus['test'].append(f['test_set/accu/' + key][...][[0, -1]]) f.close() n_layer = len(mus['train']) # test the distance thing print('model before training', MODEL) for layer in range(n_layer - 1): distribution = list() for batch in range(mus['test'][layer].shape[1]): for n in range(64): tester1 = mus['test'][layer][0, batch, n] d1 = (tester1 - dataset['images/test_set'])**2 index = pl.where( pl.argsort(d1.sum((1, 2, 3))) == ((64 * 5 * batch) + n))[0] distribution.append(index) print(pl.mean(distribution), end=', ') print(';;', len(distribution)) print('model after training', MODEL) for layer in range(n_layer - 1): distribution = list() for batch in range(mus['test'][layer].shape[1]): for n in range(64): tester1 = mus['test'][layer][1, batch, n] d1 = (tester1 - dataset['images/test_set'])**2 index = pl.where( pl.argsort(d1.sum((1, 2, 3))) == ((64 * 5 * batch) + n))[0] distribution.append(index) print(layer, pl.mean(distribution), end=', ')
def tf_idf(self): """ Normalize the frequency of stock name in the articles""" print "Initiating the word frequency test..." list_of_prepositions = [u'above', u'about', u'across', u'against',u'The', u'is',u'an', u'along', u'among', u'around', u'at', u'before', u'behind', u'below', u'beneath', u'beside', u'between', u'beyond', u'by', u'during', u'except', u'for', u'from',u'in', u'inside', u'into',u'like',u'near',u'of', u'off', u'on',u'since',u'to', u'toward', u'through',u'under', u'until', u'up', u'upon', u'with', u'within', u'the', u'a', u'and', u'for'] try: #Zipf Distribution to assess threshold paragraphs = " ".join(self.targeted_paragraphs) words = paragraphs.split() frequency = Counter(x for x in words if x not in list_of_prepositions) counts = array(frequency.values()) tokens = frequency.keys() ranks = arange(1, len(counts)+1) indices = argsort(-counts) frequencies = counts[indices] loglog(ranks, frequencies, marker=".") title("Zipf plot for Combined Article Paragraphs") xlabel("Frequency Rank of Token") ylabel("Absolute Frequency of Token") grid(True) for n in list(logspace(-0.5, log10(len(counts)-1), 20).astype(int)): dummy = text(ranks[n], frequencies[n], " " + tokens[indices[n]], verticalalignment="bottom", horizontalalignment="left") #plt.plot(np.unique(ranks), np.poly1d(np.polyfit(ranks, frequencies, 10))(np.unique(ranks))) slope=np.polyfit(ranks, frequencies, 0) for value in slope: print value zipf_threshold = value ## NEED TO ADD- FITTED LINE. Due to an error in the value of threshold its currently not in-use while paragraph extraction.## except Exception as e: print "Error occurred during Zipf Distribution: " + str(e) #Inverse-document Frequency document_size = self.result_scan_web["article_count"] document_success = len(self.article_url) try: idf = math.log(float(document_size)/float(document_success)) except Exception as e: print "No Successful Articles in this webpage: " + str(e) #Term Frequency list_total_count = [] for paragraph in self.targeted_paragraphs: words = paragraph.split() word_frequency = Counter(x for x in words if x not in list_of_prepositions) stock_frequency = word_frequency[self.stock_name] list_total_count.append(stock_frequency) total_count = sum(list_total_count) print "TOTAL WORD COUNT:" , total_count print "COUNT BEFORE:" print "urls = " , len(self.article_url) list_fail_values = [] for paragraph in self.targeted_paragraphs: words = paragraph.split() word_frequency = Counter(x for x in words if x not in list_of_prepositions) stock_frequency = word_frequency[self.stock_name] ticker_symbol_frequency = word_frequency[self.stock_id] print "TERM FREQUENCY: ", stock_frequency print "DIVISION: " , (float(stock_frequency)/float(total_count)) try: tf_idf_weight = (float(stock_frequency)/float(total_count)) * float(idf) except Exception as e: print "Error ocurred during division. total_count is zero, meaning no successful articles: " + str(e) print "TF-IDF-WEIGHT = " , tf_idf_weight if tf_idf_weight <= 0.30: # Gives the articles that lie within the 30th percentile of success. index = self.targeted_paragraphs.index(paragraph) self.targeted_paragraphs.remove(paragraph) url_value = self.article_url[index] title_value = self.article_title[index] fail_dict = {'url': url_value , 'title' : title_value} list_fail_values.append(fail_dict) print "IRRELEVANT ARTICLE DETECTED..." print "DELETING " , url_value del self.article_url[index] del self.article_title[index] else: pass print "COUNT AFTER: " print "urls = " , len(self.article_url) print "FAILED ARTICLES: " print "# " , len(list_fail_values) print "articles:" , list_fail_values #titles_in_fail = {d['title'] for d in list_fail_values if 'title' in d} #self.json_results[:] = [d for d in self.json_results if d.get('title') not in titles_in_fail] self.json_results[:] = [i for i in self.json_results if i not in list_fail_values] self.results_tone_analyzer["successful_articles"] = self.json_results print " ----------------------------" print "Number of Successful Articles = " , len(self.article_url) print "SUCCESSFUL URLs: " , for article in self.article_url: print article print " ----------------------------"
def category_dists(df, categories, outdir=FIGS_DIR, fig_suffix=None,\ category_subset=None, log_plot=True, **kwargs): for category in categories: df2 = df.drop_duplicates( subset=list(set(['neuron name', 'neuron type', category]))) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] else: df2 = remove_small_counts(df2, category,\ min_count=CATEGORY_MIN_COUNTS[category]) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] if log_plot: df2['dist'] = pylab.log10(df2['dist']) cat_vals = [] cat_means = [] print "-----------------------------------------------------" for cat_val, group in df2.groupby(category): dist = pylab.array(group['dist']).copy() cat_vals.append(cat_val) cat_mean = pylab.mean(dist) cat_means.append(cat_mean) if log_plot: dist = 10**dist print cat_val, pylab.mean(dist), "+/-", pylab.std(dist, ddof=1) order = pylab.argsort(cat_means) cat_vals = pylab.array(cat_vals) cat_means = pylab.array(cat_means) sorted_vals = cat_vals[order] sorted_means = cat_means[order] pylab.figure() sns.set() dist_plot = sns.barplot(x=category, y='dist', data=df2, order=sorted_vals) pylab.xticks(rotation=75, size=20) pylab.yticks(size=20) #pylab.xlabel(category, size=20) dist_plot.xaxis.label.set_visible(False) ylab = 'Distance to Pareto front' if log_plot: ylab = 'log(' + ylab + ')' pylab.ylabel(ylab, size=20) if 'ymin' in kwargs: pylab.ylim(ymin=kwargs['ymin']) if 'ymax' in kwargs: pylab.ylim(ymax=kwargs['ymax']) pylab.tight_layout() fname = 'pareto_dists_%s' % category.replace(' ', '_') if fig_suffix != None: fname += '_%s' % fig_suffix pylab.savefig('%s/%s.pdf' % (outdir, fname), bbox_inches='tight')
def raster_tuning(ax): fullbehaviorDir = behaviorDir + subject + '/' behavName = subject + '_tuning_curve_' + tuningBehavior + '.h5' tuningBehavFileName = os.path.join(fullbehaviorDir, behavName) tuning_bdata = loadbehavior.BehaviorData(tuningBehavFileName, readmode='full') freqEachTrial = tuning_bdata['currentFreq'] possibleFreq = np.unique(freqEachTrial) numberOfTrials = len(freqEachTrial) # -- The old way of sorting (useful for plotting sorted raster) -- sortedTrials = [] numTrialsEachFreq = [ ] #Used to plot lines after each group of sorted trials for indf, oneFreq in enumerate( possibleFreq ): #indf is index of this freq and oneFreq is the frequency indsThisFreq = np.flatnonzero( freqEachTrial == oneFreq) #this gives indices of this frequency sortedTrials = np.concatenate( (sortedTrials, indsThisFreq)) #adds all indices to a list called sortedTrials numTrialsEachFreq.append( len(indsThisFreq)) #finds number of trials each frequency has sortingInds = argsort( sortedTrials) #gives array of indices that would sort the sortedTrials # -- Load event data and convert event timestamps to ms -- tuning_ephysDir = os.path.join(settings.EPHYS_PATH, subject, tuningEphys) tuning_eventFilename = os.path.join(tuning_ephysDir, 'all_channels.events') tuning_ev = loadopenephys.Events( tuning_eventFilename) #load ephys data (like bdata structure) tuning_eventTimes = np.array( tuning_ev.timestamps ) / SAMPLING_RATE #get array of timestamps for each event and convert to seconds by dividing by sampling rate (Hz). matches with eventID and tuning_evID = np.array( tuning_ev.eventID ) #loads the onset times of events (matches up with eventID to say if event 1 went on (1) or off (0) tuning_eventOnsetTimes = tuning_eventTimes[ tuning_evID == 1] #array that is a time stamp for when the chosen event happens. #ev.eventChannel woul load array of events like trial start and sound start and finish times (sound event is 0 and trial start is 1 for example). There is only one event though and its sound start while (numberOfTrials < len(tuning_eventOnsetTimes)): tuning_eventOnsetTimes = tuning_eventOnsetTimes[:-1] ####################################################################################################### ###################THIS IS SUCH A HACK TO GET SPKDATA FROM EPHYSCORE################################### ####################################################################################################### thisCell = celldatabase.CellInfo( animalName=subject, ############################################ ephysSession=tuningEphys, tuningSession='DO NOT NEED THIS', tetrode=tetrode, cluster=cluster, quality=1, depth=0, tuningBehavior='DO NOT NEED THIS', behavSession=tuningBehavior) tuning_spkData = ephyscore.CellData(thisCell) tuning_spkTimeStamps = tuning_spkData.spikes.timestamps (tuning_spikeTimesFromEventOnset, tuning_trialIndexForEachSpike, tuning_indexLimitsEachTrial) = spikesanalysis.eventlocked_spiketimes( tuning_spkTimeStamps, tuning_eventOnsetTimes, tuning_timeRange) #print 'numTrials ',max(tuning_trialIndexForEachSpike)##################################### ''' Create a vector with the spike timestamps w.r.t. events onset. (spikeTimesFromEventOnset,trialIndexForEachSpike,indexLimitsEachTrial) = eventlocked_spiketimes(timeStamps,eventOnsetTimes,timeRange) timeStamps: (np.array) the time of each spike. eventOnsetTimes: (np.array) the time of each instance of the event to lock to. timeRange: (list or np.array) two-element array specifying time-range to extract around event. spikeTimesFromEventOnset: 1D array with time of spikes locked to event. o trialIndexForEachSpike: 1D array with the trial corresponding to each spike. The first spike index is 0. indexLimitsEachTrial: [2,nTrials] range of spikes for each trial. Note that the range is from firstSpike to lastSpike+1 (like in python slices) spikeIndices ''' tuning_sortedIndexForEachSpike = sortingInds[ tuning_trialIndexForEachSpike] #Takes values of trialIndexForEachSpike and finds value of sortingInds at that index and makes array. This array gives an array with the sorted index of each trial for each spike # -- Calculate tuning -- #nSpikes = spikesanalysis.spiketimes_to_spikecounts(spikeTimesFromEventOnset,indexLimitsEachTrial,responseRange) #array of the number of spikes in range for each trial '''Count number of spikes on each trial in a given time range. spikeTimesFromEventOnset: vector of spikes timestamps with respect to the onset of the event. indexLimitsEachTrial: each column contains [firstInd,lastInd+1] of the spikes on a trial. timeRange: time range to evaluate. Spike times exactly at the limits are not counted. returns nSpikes ''' ''' meanSpikesEachFrequency = np.empty(len(possibleFreq)) #make empty array of same size as possibleFreq # -- This part will be replace by something like behavioranalysis.find_trials_each_type -- trialsEachFreq = [] for indf,oneFreq in enumerate(possibleFreq): trialsEachFreq.append(np.flatnonzero(freqEachTrial==oneFreq)) #finds indices of each frequency. Appends them to get an array of indices of trials sorted by freq # -- Calculate average firing for each freq -- for indf,oneFreq in enumerate(possibleFreq): meanSpikesEachFrequency[indf] = np.mean(nSpikes[trialsEachFreq[indf]]) ''' #clf() #if (len(tuning_spkTimeStamps)>0): #ax1 = plt.subplot2grid((4,4), (3, 0), colspan=1) #spikesorting.plot_isi_loghist(spkData.spikes.timestamps) #ax3 = plt.subplot2grid((4,4), (3, 3), colspan=1) #spikesorting.plot_events_in_time(tuning_spkTimeStamps) #samples = tuning_spkData.spikes.samples.astype(float)-2**15 #samples = (1000.0/tuning_spkData.spikes.gain[0,0]) *samples #ax2 = plt.subplot2grid((4,4), (3, 1), colspan=2) #spikesorting.plot_waveforms(samples) #ax4 = plt.subplot2grid((4,4), (0, 0), colspan=3,rowspan = 3) plot(tuning_spikeTimesFromEventOnset, tuning_sortedIndexForEachSpike, '.', ms=3) #axvline(x=0, ymin=0, ymax=1, color='r') #The cumulative sum of the list of specific frequency presentations, #used below for plotting the lines across the figure. numTrials = cumsum(numTrialsEachFreq) #Plot the lines across the figure in between each group of sorted trials for indf, num in enumerate(numTrials): ax.axhline(y=num, xmin=0, xmax=1, color='0.90', zorder=0) tickPositions = numTrials - mean(numTrialsEachFreq) / 2 tickLabels = [ "%0.2f" % (possibleFreq[indf] / 1000) for indf in range(len(possibleFreq)) ] ax.set_yticks(tickPositions) ax.set_yticklabels(tickLabels) ax.set_ylim([-1, numberOfTrials]) ylabel('Frequency Presented (kHz), {} total trials'.format(numTrials[-1])) #title(ephysSession+' T{}c{}'.format(tetrodeID,clusterID)) xlabel('Time (sec)') ''' ax5 = plt.subplot2grid((4,4), (0, 3), colspan=1,rowspan=3) ax5.set_xscale('log') plot(possibleFreq,meanSpikesEachFrequency,'o-') ylabel('Avg spikes in window {0}-{1} sec'.format(*responseRange)) xlabel('Frequency') ''' #show() '''
def plot_alot(signal_at_elec, cell, t_range, Mea, out_name): pl.close('all') n_elecs = Mea.n_elecs electrode_separation = Mea.electrode_separation t_array = cell.tvec fig = pl.figure(figsize=[10, 10]) ax = fig.add_axes([0.25, 0.05, 0.7, .9], frameon=False) ax.axis([-0.3,0.6,-0.2,1.1]) #ax.axis('equal') n_compartments = len(cell.diam)#neuron.total_n_compartments stim_points = [615, 671,623, 628] comp_list = np.r_[0, 615, 646, 671, 623, 628, 657] for i in xrange(n_compartments): if i == 0: xcoords = pl.array([cell.xmid[i]/1000]) ycoords = pl.array([cell.ymid[i]/1000]) zcoords = pl.array([cell.zmid[i]/1000]) diams = pl.array([cell.diam[i]/1000]) else: if cell.zmid[i] < 100 and cell.zmid[i] > -100 and \ cell.xmid[i] < 100 and cell.xmid[i] > -100: xcoords = pl.r_[xcoords, pl.linspace(cell.xstart[i], \ cell.xend[i], cell.length[i]*3)/1000] ycoords = pl.r_[ycoords, pl.linspace(cell.ystart[i], \ cell.yend[i], cell.length[i]*3)/1000] zcoords = pl.r_[zcoords, pl.linspace(cell.zstart[i], \ cell.zend[i], cell.length[i]*3)/1000] diams = pl.r_[diams, pl.linspace(cell.diam[i], \ cell.diam[i],cell.length[i]*3)] argsort = pl.argsort(-xcoords) ax.scatter(zcoords[argsort], ycoords[argsort], s=3*diams[argsort]**2,\ edgecolors='none', c='gray') ax.scatter(cell.zmid[0]/1000, cell.ymid[0]/1000, \ s=3*cell.diam[0]**2, edgecolors='none', c='gray') ax.scatter(Mea.elec_z, Mea.elec_y, color='m', s= 50) for comp in stim_points: ax.plot(cell.zmid[comp]/1000, cell.ymid[comp]/1000, '*', color='y') t_start = t_range[0] t_stop = t_range[1] t_start_index = np.abs(t_array[:] - t_start).argmin() t_stop_index = np.abs(t_array[:] - t_stop).argmin() t_array = t_array[t_start_index:t_stop_index] - t_array[t_start_index] signal_at_elec = signal_at_elec[:,t_start_index:t_stop_index] #chosen_imem = chosen_imem[t_start_index:t_stop_index] time_factor = 0.7*1./(t_stop - t_start)*electrode_separation/2 signal_range = 15 pot_factor = 0.5*electrode_separation/signal_range for elec in xrange(n_elecs): t = t_array*time_factor + Mea.elec_z[elec] trace = (signal_at_elec[elec])*pot_factor + Mea.elec_y[elec] ax.plot(t, trace, color='r', lw = 3) #pl.figure() #pl.plot(t_array, chosen_imem, color='r', lw = 3) #pl.plot(t_array[130:-95], chosen_imem[130:-95], color='k', lw = 3) #ax.axis([-0.1, 0.1, -0.1, 0.1]) #ax.axis([1.1*np.min(Mea.elec_z),1.1*np.max(Mea.elec_z) , 1.1*np.min(Mea.elec_y),1.1*np.max(Mea.elec_y) ]) ax.plot([0.2, 0.2 + time_factor *(t_stop - t_start)], \ [-0.13, -0.13], color='k', lw = 4) ax.text(0.205, -0.155, '%g ms' % int(t_stop-t_start)) #ax.plot([0.305, 0.355], [0.01, 0.01], color='k', lw = 4) #ax.text(0.32, 0.06, '50 $\mu$m') ax.plot([0.2, 0.2], [-0.13, -0.13 +pot_factor * signal_range],\ color='r', lw=4) ax.text(0.205, -.21 +electrode_separation/2 , '%g $\mu$V' % signal_range) #pl.xlabel('z [$\mu$m]') #pl.ylabel('y [$\mu$m]') ax.set_xticks([]) ax.set_yticks([]) ax2 = fig.add_axes([0.1, 0.1, 0.25, 0.2]) ax2.plot([20],[8], '*', color='k') ax2.plot([40],[8], '*', color='k') ax2.plot([60],[8], '*', color='k') #ax2.text(pos[20,5], pos[1,5]-0.15, "Arrow", ha="center", # family=font, size=14) #p = ax2.axvspan(20, 21, facecolor='0.5', alpha=0.2) #p = ax2.axvspan(40, 41, facecolor='0.5', alpha=0.2) #p = ax2.axvspan(60, 61, facecolor='0.5', alpha=0.2) ax2.axis([10,100, -100,20]) ax2.set_title('Membrane voltage') ax2.set_xlabel('Time [ms]') ax2.set_ylabel('mV') #ax2.plot(cell.tvec[t_start_index:t_stop_index], \ # cell.vmem[0,t_start_index:t_stop_index]) #import random #comp = random.choice(cell.get_idx_section('apic[63]')) comp_list = np.r_[0, 615, 646, 671, 623, 628, 657] for comp in comp_list: ax2.plot(cell.tvec[t_start_index:t_stop_index], \ cell.vmem[comp,t_start_index:t_stop_index]) ax3 = fig.add_axes([0.1, 0.4, 0.25, 0.2]) ax3.set_title('Membrane currents') ax3.set_xlabel('Time [ms]') ax3.axis([10,100, -1,1]) ax3.plot([20],[.8], '*', color='k') ax3.plot([40],[.8], '*', color='k') ax3.plot([60],[.8], '*', color='k') ax3.set_ylabel('nA') #ax2.plot(cell.tvec[t_start_index:t_stop_index], \ # cell.vmem[0,t_start_index:t_stop_index]) #import random #comp = random.choice(cell.get_idx_section('apic[63]')) for comp in xrange(len(cell.imem[:,0])):#comp_list: ax3.plot(cell.tvec[t_start_index:t_stop_index], \ cell.imem[comp,t_start_index:t_stop_index]) ax4 = fig.add_axes([0.1, 0.7, 0.25, 0.2]) ax4.plot([20],[8], '*', color='k') ax4.plot([40],[8], '*', color='k') ax4.plot([60],[8], '*', color='k') ax4.set_title('Signal at electrodes') ax4.set_xlabel('Time [ms]') ax4.set_ylabel('$\mu$V') ax4.axis([10,100, -15,10]) for elec in xrange(Mea.n_elecs): ax4.plot(cell.tvec[t_start_index:t_stop_index], \ signal_at_elec[elec,:]) pl.savefig(out_name)
def entropy(points, logp, N_entropy=10000, N_norm=2500): r""" Return entropy estimate and uncertainty from a random sample. *points* is a set of draws from an underlying distribution, as returned by a Markov chain Monte Carlo process for example. *logp* is the log-likelihood for each draw. *N_norm* is the number of points $k$ to use to estimate the posterior density normalization factor $P(D) = \hat N$, converting from $\log( P(D|M) P(M) )$ to $\log( P(D|M)P(M)/P(D) )$. The relative uncertainty $\Delta\hat S/\hat S$ scales with $\sqrt{k}$, with the default *N_norm=2500* corresponding to 2% relative uncertainty. Computation cost is $O(nk)$ where $n$ is number of points in the draw. *N_entropy* is the number of points used to estimate the entropy $\hat S = - \int P(M|D) \log P(M|D)$ from the normalized log likelihood values. """ # Use a random subset to estimate density if N_norm >= len(logp): norm_points = points else: idx = permutation(len(points))[:N_entropy] norm_points = points[idx] # Use a different subset to estimate the scale factor between density # and logp. if N_entropy is None: N_entropy = 10000 if N_entropy >= len(logp): entropy_points, eval_logp = points, logp else: idx = permutation(len(points))[:N_entropy] entropy_points, eval_logp = points[idx], logp[idx] """ # Try again, just using the points from the high probability regions # to determine the scale factor N_norm = min(len(logp), 5000) N_entropy = int(0.8*N_norm) idx = np.argsort(logp) norm_points = points[idx[-N_norm:]] entropy_points = points[idx[-N_entropy:]] eval_logp = logp[idx[-N_entropy:]] """ # Normalize p to a peak probability of 1 so that exp() doesn't underflow. # # This should be okay since for the normalizing constant C: # # u' = e^(ln u + ln C) = e^(ln u)e^(ln C) = C u # # Using eq. 11 of Kramer with u' substituted for u: # # N_est = < u'/p > = < C u/p > = C < u/p > # # S_est = - < ln q > # = - < ln (u'/N_est) > # = - < ln C + ln u - ln (C <u/p>) > # = - < ln u + ln C - ln C - ln <u/p> > # = - < ln u - ln <u/p> > # = - < ln u > + ln <u/p> # # Uncertainty comes from eq. 13: # # N_err^2 = 1/(k-1) sum( (u'/p - <u'/p>)^2 ) # = 1/(k-1) sum( (C u/p - <C u/p>)^2 ) # = C^2 std(u/p)^2 # S_err = std(u'/p) / <u'/p> = (C std(u/p))/(C <u/p>) = std(u/p)/<u/p> # # So even though the constant C shows up in N_est, N_err, it cancels # again when S_est, S_err is formed. log_scale = np.max(eval_logp) # print("max log sample: %g"%log_scale) eval_logp -= log_scale # Compute entropy and uncertainty in nats # Note: if all values are the same in any dimension then we have a dirac # functional with infinite probability at every sample point, and the # differential entropy estimate will yield H = -inf. rho = density(norm_points, entropy_points) #print(rho.min(), rho.max(), eval_logp.min(), eval_logp.max()) frac = exp(eval_logp)/rho n_est, n_err = mean(frac), std(frac) if n_est == 0.: s_est, s_err = -np.inf, 0. else: s_est = log(n_est) - mean(eval_logp) s_err = n_err/n_est #print(n_est, n_err, s_est/LN2, s_err/LN2) ##print(np.median(frac), log(np.median(frac))/LN2, log(n_est)/LN2) if False: import pylab idx = pylab.argsort(entropy_points[:, 0]) pylab.figure() pylab.subplot(221) pylab.hist(points[:, 0], bins=50, normed=True, log=True) pylab.plot(entropy_points[idx, 0], rho[idx], label='density') pylab.plot(entropy_points[idx, 0], exp(eval_logp+log_scale)[idx], label='p') pylab.ylabel("p(x)") pylab.legend() pylab.subplot(222) pylab.hist(points[:, 0], bins=50, normed=True, log=False) pylab.plot(entropy_points[idx, 0], rho[idx], label='density') pylab.plot(entropy_points[idx, 0], exp(eval_logp+log_scale)[idx], label='p') pylab.ylabel("p(x)") pylab.legend() pylab.subplot(212) pylab.plot(entropy_points[idx, 0], frac[idx], '.') pylab.xlabel("P[0] value") pylab.ylabel("p(x)/kernel density") # return entropy and uncertainty in bits return s_est/LN2, s_err/LN2
def val_distribution(df, val, categories, plot_func, plot_descriptor,\ outdir=FIGS_DIR, fig_suffix=None, category_subset=None,\ **kwargs): for category in categories: subset_cols = ['neuron name', 'neuron type', 'alpha'] if category != 'neuron type': subset_cols.append(category) df2 = df.drop_duplicates(subset=subset_cols) if category_subset != None: df2 = df2[df2[category].isin(category_subset)] else: df2 = remove_small_counts(df2, category,\ min_count=CATEGORY_MIN_COUNTS[category]) log_transform = False if ('log_transform' in kwargs) and kwargs['log_transform']: log_transform = True if log_transform: df2[val] = pylab.log10(df2[val]) cat_vals = [] order_vals = [] order_val = kwargs['order_val'] print "-----------------------------------------------------" for name, group in df2.groupby(category): cat_vals.append(name) if order_val == None: order_val.append(pylab.median(group[val])) else: order_vals.append(pylab.mean(group[order_val])) print name, val, pylab.mean(group[val]), "+/-", pylab.std( group[val], ddof=1) cat_vals = pylab.array(cat_vals) #mean = pylab.array(medians) order = pylab.argsort(order_vals) order = cat_vals[order] pylab.figure() sns.set() dist_plot = plot_func(x=category, y=val, data=df2, order=order) dist_plot.tick_params(axis='x', labelsize=20, rotation=75) dist_plot.tick_params(axis='y', labelsize=20) #pylab.xlabel(category, fontsize=20) dist_plot.xaxis.label.set_visible(False) ylab = None if 'ylab' in kwargs: ylab = kwargs['ylab'] else: ylab = val if log_transform: ylab = 'log(' + ylab + ')' pylab.ylabel(ylab, fontsize=20) pylab.tight_layout() fname = '%s_%ss_%s' % (category.replace(' ', '_'),\ val.replace(' ', '_'), plot_descriptor) if fig_suffix != None: fname += '_%s' % fig_suffix pylab.savefig('%s/%s.pdf' % (outdir, fname), format='pdf') pylab.close()