def follow_trajectory_with_grabs(pr2, bodypart2traj, ignore_failure=False): """ bodypart2traj is a dictionary with zero or more of the following fields: {l/r}_grab, {l/r}_gripper, {l/r_arm} We'll follow all of these bodypart trajectories simultaneously Also, if the trajectory involves grabbing with the gripper, and the grab fails, the trajectory will abort """ T = len(bodypart2traj.values()[0]) l_grab = bodypart2traj["l_grab"] if "l_grab" in bodypart2traj else np.zeros(T,bool) r_grab = bodypart2traj["r_grab"] if "r_grab" in bodypart2traj else np.zeros(T,bool) l_after_grab_inds = np.flatnonzero(l_grab[1:] > l_grab[:-1])+1 r_after_grab_inds = np.flatnonzero(r_grab[1:] > r_grab[:-1])+1 inds_sides = sorted([(i,'l') for i in l_after_grab_inds] + [(i,'r') for i in r_after_grab_inds]) inds_sides = merge_nearby_grabs(inds_sides) num_grabs = len(inds_sides) print "breaking trajectory into %i segments"%(num_grabs+1) i_begin = 0 for (i_grab, side) in inds_sides: t_start = rospy.Time.now().to_sec() success = follow_trajectory(pr2, slice_traj(bodypart2traj, i_begin, i_grab)) rospy.loginfo("follow traj result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start) if not success and not ignore_failure: return False t_start = rospy.Time.now().to_sec() success = close_gripper(pr2, side) rospy.loginfo("close gripper result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start) if not success and not ignore_failure: return False i_begin = i_grab t_start = rospy.Time.now().to_sec() success = follow_trajectory(pr2, slice_traj(bodypart2traj, i_begin, -1)) rospy.loginfo("follow traj result: %s, duration: %.2f", success, rospy.Time.now().to_sec() - t_start) if not success and not ignore_failure: return False return True
def coloring(m, inv_row_p, inv_col_p): # The permutation must give a matrix in Hessenberg form row_major, col_major = get_permuted_sp_matrices(m, inv_row_p, inv_col_p) ncols = col_major.shape[1] # Initially all column colors are set to the invalid "color" -1 column_colors = np.full(ncols, -1, np.int32) # If color i is available: color_available[i] is 1, otherwise 0 # Initially, "color" 0 is available color_available = np.ones(1, np.int8) # Iterate through the columns backwards, from right to left for c in reversed(irange(ncols)): # All rows containing c for r in rows_in_col(col_major, c): # All other columns in those rows, c's "neighbors" cols = cols_in_row(row_major, r) idx = np.flatnonzero(cols==c) # cols in r: [ left | c | right] # only the right index set has been processed so far right = cols[idx+1:] used_colors = column_colors[right] color_available[used_colors] = 0 #print('Colors available:\n%s' % color_available) index = np.flatnonzero(color_available)[0] column_colors[c] = index # introduce new "color" if index == color_available.size-1 color_available = np.ones(max(index+2, color_available.size), np.int8) print('Colors:\n%s' % column_colors) # chromatic number >= max nonzeros in a row lb_chromatic_number = np.max(np.diff(row_major.indptr)) color_count = color_available.size-1 print('Color count: %d (>=%d)' % (color_count, lb_chromatic_number)) return column_colors, color_count
def dump_pre_post_stim_spike_count(ffname, outprefix, celltype, window=10e-3): """Dump the standard deviation in population spike before and after stimulus. """ with open('{}_prepost_spikes_{}_{}ms_window.csv'.format(outprefix, celltype, window*1e3), 'wb') as fd: writer = csv.writer(fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) norm_files = get_dbcnt_dict(ffname) writer.writerow(['dbcount', 'filename', 'premean', 'premedian', 'prestd', 'postmean', 'postmedian', 'poststd']) for dbcnt, flist in norm_files.items(): for fname in flist: data = TraubData(makepath(fname)) pop_train_list = [] bgtimes, probetimes = get_stim_times(data, correct_tcr=False) times = np.concatenate((bgtimes, probetimes)) times.sort() for cell, train in data.spikes.items(): if cell.startswith(celltype): pop_train_list.append(train) pop_train = np.concatenate(pop_train_list) pop_train.sort() pre = [] post = [] for t in times: pre.append(np.flatnonzero((pop_train < t) & (pop_train > t - window/2)).shape[0]*1.0 / data.cellcounts._asdict()[celltype]) post.append(np.flatnonzero((pop_train > t) & (pop_train < t + window/2)).shape[0]*1.0 / data.cellcounts._asdict()[celltype]) writer.writerow([dbcnt, fname] + [np.mean(pre), np.median(pre), np.std(pre, ddof=1), np.mean(post), np.median(post), np.std(post, ddof=1)])
def sort_on_centroids(fn_mwk, fn_nev,k_cent_all,pc_sd_all,T_all,sel=SEL_ELEC_A, nmax=N_SNIPPET_MAX, npts=N_SNIPPET_PTS,c_v=VOLT_CONV): #this spikes sorts based on how close a spike is to the farthest centroid from center of mass of all the centroids sorted_data = {} for arg in getspk(fn_mwk, fn_nev, override_elecs=sel): # -- preps w = np.array(arg['wav']['unsorted']) w *= c_v ch = arg['ch'] if ch not in sorted_data.keys(): sorted_data[ch] = {'good_sp':[],'bad_sp':[],'g_sp_t_abs':[],'g_sp_t_rel':[],'g_sp_t_imgonset':[],'g_sp_im_id':[],'g_sp_im_ind':[],'b_sp_t_abs':[],'b_sp_t_rel':[],'b_sp_t_imgonset':[],'b_sp_im_id':[],'b_sp_im_ind':[]} ch_ind = sel.index(ch) #pick the centroid and the spikes sd for that channel pc_sd = pc_sd_all[ch_ind] k_cent = k_cent_all[ch_ind] k_dist = np.array([fastnorm(k-k_cent.mean(axis=0)) for k in k_cent]) T = T_all[ch_ind] b_c = np.flatnonzero(k_dist == k_dist.max()) #compute the distance of the waveform from the the centroids and determine whether it is a good spike or a bad spike. w_dist = np.array([fastnorm((np.dot(w,T)/pc_sd)-k_cent[k]) for k in range(len(k_cent))]) if np.flatnonzero(w_dist == w_dist.min())==b_c and k_dist.max() > 1.75: sorted_data[ch]['good_sp'].append(w) sorted_data[ch]['g_sp_t_abs'].append(arg['t_abs']) sorted_data[ch]['g_sp_t_rel'].append(arg['t_rel']) sorted_data[ch]['g_sp_t_imgonset'].append(arg['t_imgonset']) sorted_data[ch]['g_sp_im_id'].append(arg['imgid']) sorted_data[ch]['g_sp_im_ind'].append(arg['iimg']) else: sorted_data[ch]['b_sp_t_abs'].append(arg['t_abs']) sorted_data[ch]['b_sp_t_rel'].append(arg['t_rel']) sorted_data[ch]['b_sp_t_imgonset'].append(arg['t_imgonset']) sorted_data[ch]['b_sp_im_id'].append(arg['imgid']) sorted_data[ch]['b_sp_im_ind'].append(arg['iimg']) return sorted_data
def align(s_x, s_y, mode="front"): """Function to align the input series :param s_x: First Series :type s_x: :mod:`pandas.Series` :param s_y: Second Series :type s_y: :mod:`pandas.Series` :param mode: Align Front/Back :type mode: str """ p_x = np.flatnonzero(s_x) p_y = np.flatnonzero(s_y) if not len(p_x) or not len(p_y): return s_x, s_y, 0 if mode == "front": p_x = p_x[0] p_y = p_y[0] if mode == "back": p_x = p_x[-1] p_y = p_y[-1] shift = p_x - p_y s_x, s_y = shift_series(s_x, s_y, shift) return s_x, s_y, shift
def _mmd_ab(a, b, one_ring_neighbour): """ Compute median minimal distance between a,b part computational completion of median_minimal_distance Parameters: ----------- a, b: array with 1 label one_ring_neighbour: one ring neighbour matrix Return: ------- h: minimal distance """ a = np.array(a) b = np.array(b) h = [] for i in np.flatnonzero(a): hd = np.inf for j in np.flatnonzero(b): d = surf_dist(i, j, one_ring_neighbour) if d<hd: hd = d h.append(hd) return h
def get_cap_check_indices(i): # Assumes that there is a test pulse followed by the stimulus pulses (downward first) di = np.diff(i) up_idx = np.flatnonzero(di > 0) down_idx = np.flatnonzero(di < 0) return up_idx[2::2], down_idx[1::2]
def update_nogrid(self, params): endog = self.model.endog_li cached_means = self.model.cached_means varfunc = self.model.family.variance dep_params = np.zeros(self.max_lag + 1) dn = np.zeros(self.max_lag + 1) for i in range(self.model.num_group): expval, _ = cached_means[i] stdev = np.sqrt(varfunc(expval)) resid = (endog[i] - expval) / stdev j1, j2 = np.tril_indices(len(expval)) dx = np.abs(self.time[i][j1] - self.time[i][j2]) ii = np.flatnonzero(dx <= self.max_lag) j1 = j1[ii] j2 = j2[ii] dx = dx[ii] vs = np.bincount(dx, weights=resid[ j1] * resid[j2], minlength=self.max_lag + 1) vd = np.bincount(dx, minlength=self.max_lag + 1) ii = np.flatnonzero(vd > 0) dn[ii] += 1 if len(ii) > 0: dep_params[ii] += vs[ii] / vd[ii] dep_params /= dn self.dep_params = dep_params[1:] / dep_params[0]
def _hausdoff_ab(a, b, one_ring_neighbour): """ Compute hausdoff distance of h(a,b) part unit of function hausdoff_distance Parameters: ----------- a: array with 1 label b: array with 1 label one_ring_neighbour: one ring neighbour matrix Return: ------- h: hausdoff(a,b) """ a = np.array(a) b = np.array(b) h = 0 for i in np.flatnonzero(a): hd = np.inf for j in np.flatnonzero(b): d = surf_dist(i,j, one_ring_neighbour) if d<hd: hd = copy.deepcopy(d) if hd>h: h = hd return h
def _calculateSegmentActivity(connections, activeInput, connectedPermanence, activationThreshold, minThreshold): """ Calculate the active and matching segments for this timestep. @param connections (SparseMatrixConnections) @param activeInput (numpy array) @return (tuple) - activeSegments (numpy array) Dendrite segments with enough active connected synapses to cause a dendritic spike - matchingSegments (numpy array) Dendrite segments with enough active potential synapses to be selected for learning in a bursting column - potentialOverlaps (numpy array) The number of active potential synapses for each segment. Includes counts for active, matching, and nonmatching segments. """ # Active overlaps = connections.computeActivity(activeInput, connectedPermanence) activeSegments = np.flatnonzero(overlaps >= activationThreshold) # Matching potentialOverlaps = connections.computeActivity(activeInput) matchingSegments = np.flatnonzero(potentialOverlaps >= minThreshold) return (activeSegments, matchingSegments, potentialOverlaps)
def connection_field_plot_continuous(self, index, afferent=True, density=30): weights = numpy.array(self.proj.get('weight', format='list', gather=True)) if afferent: idx = numpy.array(numpy.flatnonzero(weights[:,1].flatten()==index)) x = self.proj.pre.positions[0][weights[idx,0].astype(int)] y = self.proj.pre.positions[1][weights[idx,0].astype(int)] w = weights[idx,2] else: idx = numpy.flatnonzero(weights[:,0]==index) x = self.proj.post.positions[0][weights[idx,1].astype(int)] y = self.proj.post.positions[1][weights[idx,1].astype(int)] w = weights[idx,2] xi = numpy.linspace(min(x), max(x), 100) yi = numpy.linspace(min(y), max(y), 100) zi = griddata(x, y, w, xi, yi) pylab.figure() #pylab.imshow(zi) pylab.scatter(x,y,marker='o',c=w,s=50) pylab.xlim(-self.source.parameters.sx/2,self.source.parameters.sx/2) pylab.ylim(-self.source.parameters.sy/2,self.source.parameters.sy/2) pylab.colorbar() pylab.title('Connection field from %s to %s of neuron %d' % (self.source.name, self.target.name, index))
def get_feature_mask(self, mask): """ Parameters ---------- mask: (N, F) or (F,) ndarray of bool True for unobserved features. Returns ------- feature_mask: (N, D) or (D,) ndarray of bool True for unobserved features. """ assert(mask.dtype == bool) if mask.ndim == 1: assert(mask.shape[0] == self.F) feature_mask = np.ones(self.D).astype(bool) for ind in np.flatnonzero(~mask): feature_mask[slice(*self.feature_bounds[ind])] = False else: assert(mask.shape[1] == self.F) N = mask.shape[0] feature_mask = np.ones((N, self.D)).astype(bool) for i in xrange(N): for ind in np.flatnonzero(~mask[i]): feature_mask[i, slice(*self.feature_bounds[ind])] = False return feature_mask
def _split_indices(self, vec): null = pd.isnull(vec) ix_obs = np.flatnonzero(~null) ix_miss = np.flatnonzero(null) if len(ix_obs) == 0: raise ValueError("variable to be imputed has no observed values") return ix_obs, ix_miss
def elam_spline(xin, yin, yspl_in, xout): """ interpolate values from Elam photoabsorption and scattering tables, according to Elam, and following standard interpolation methods. Calc borrowed from D. Dale. Parameters: xin (ndarray): x values for interpolation data yin (ndarray): y values for interpolation data yspl_in (ndarray): spline coefficients (second derivatives of y) for interpolation data xout(float or ndarray): x values to be evaluated at Returns: ndarray: interpolated values """ x = as_ndarray(xout) x[np.where(x < min(xin))] = min(xin) x[np.where(x > max(xin))] = max(xin) lo, hi = np.array([(np.flatnonzero(xin < e)[-1], np.flatnonzero(xin > e)[0]) for e in x]).transpose() diff = xin[hi] - xin[lo] if any(diff <= 0): raise ValueError('x must be strictly increasing') a = (xin[hi] - x) / diff b = (x - xin[lo]) / diff return (a * yin[lo] + b * yin[hi] + (diff*diff/6) * ((a*a - 1) * a * yspl_in[lo] + (b*b - 1) * b * yspl_in[hi]))
def retrieve_coverage_information(self, cov_src=""): u""" Retrieve coverage information, i.e. whether locations to have spectra extracted have valid data and are not covered by clouds, margins etc. Information is retrieved from an external file. """ self.coverage = dict() if cov_src and os.path.isfile(cov_src): for img_id in self.image_data: self.coverage[img_id] = list() try: data = np.loadtxt(cov_src) except: data = np.genfromtxt(cov_src, converters={0: lambda s: s.strip()}) for d in data: # extracting row_id, usually a location id, i.e. for a plot try: row_id = int(d[0]) except: row_id = d[0] # retrieving non-zero elements for each row, i.e. indicators # for visibility and coverage for each image id try: nz = np.flatnonzero(d[1:]) except: # this line is necessary to deal with zero-rank arrays # that are generated by np.genfromtxt above if the # original location ids cannot be converted to float, # i.e. if they contain letters nz = np.flatnonzero(np.array(d.tolist()[1:])) for n in nz: self.coverage[n].append(row_id)
def __init__(self, alphabet, **kw): _ContinuousSubstitutionModel.__init__(self, alphabet, **kw) alphabet = self.getAlphabet() # as may be altered by recode_gaps etc. mask = self._instantaneous_mask N = len(alphabet) self.param_pick = numpy.zeros([N,N], int) self.parameter_order = [] self.last_in_column = [] for (d, (row, col)) in enumerate(zip(mask, mask.T)): row = list(numpy.flatnonzero(row[d:])+d) col = list(numpy.flatnonzero(col[d:])+d) if col: self.last_in_column.append((col.pop(), d)) else: assert not row inst = [(d,j) for j in row] + [(i,d) for i in col] for (i, j) in inst: (x,y) = [alphabet[k] for k in [i,j]] self.parameter_order.append('%s/%s'%(x,y)) self.param_pick[i,j] = len(self.parameter_order) if self._do_scaling: const_param = self.parameter_order.pop() self.symmetric = False self.checkParamsExist()
def readPulsar(self, psr, psrname): print("WARNING: readPulsar has been deprecated!") psr.name = psrname # Read the content of the par/tim files in a string psr.parfile_content = str(self.getData(psrname, 'parfile', required=False)) psr.timfile_content = str(self.getData(psrname, 'timfile', required=False)) # Read the timing model parameter descriptions psr.ptmdescription = map(str, self.getData(psrname, 'tmp_name')) psr.ptmpars = np.array(self.getData(psrname, 'tmp_valpre')) psr.ptmparerrs = np.array(self.getData(psrname, 'tmp_errpre')) psr.flags = map(str, self.getData(psrname, 'efacequad', 'Flags')) # Read the position of the pulsar if self.hasField(psrname, 'raj'): psr.raj = np.float(self.getData(psrname, 'raj')) else: rajind = np.flatnonzero(np.array(psr.ptmdescription) == 'RAJ') psr.raj = np.array(self.getData(psrname, 'tmp_valpre'))[rajind] if self.hasField(psrname, 'decj'): psr.decj = np.float(self.getData(psrname, 'decj')) else: decjind = np.flatnonzero(np.array(psr.ptmdescription) == 'DECJ') psr.decj = np.array(self.getData(psrname, 'tmp_valpre'))[decjind] # Obtain residuals, TOAs, etc. psr.toas = np.array(self.getData(psrname, 'TOAs')) psr.toaerrs = np.array(self.getData(psrname, 'toaErr')) psr.prefitresiduals = np.array(self.getData(psrname, 'prefitRes')) psr.residuals = np.array(self.getData(psrname, 'postfitRes')) psr.detresiduals = np.array(self.getData(psrname, 'prefitRes')) psr.freqs = np.array(self.getData(psrname, 'freq')) psr.Mmat = np.array(self.getData(psrname, 'designmatrix'))
def split_trajectory_by_gripper(seg_info, thresh = .04, ms_thresh = 2): # thresh: open/close threshold rgrip = asarray(seg_info["r_gripper_joint"]) lgrip = asarray(seg_info["l_gripper_joint"]) n_steps = len(lgrip) # indices BEFORE transition occurs l_openings = np.flatnonzero((lgrip[1:] >= thresh) & (lgrip[:-1] < thresh)) r_openings = np.flatnonzero((rgrip[1:] >= thresh) & (rgrip[:-1] < thresh)) l_closings = np.flatnonzero((lgrip[1:] < thresh) & (lgrip[:-1] >= thresh)) r_closings = np.flatnonzero((rgrip[1:] < thresh) & (rgrip[:-1] >= thresh)) before_transitions = np.r_[l_openings, r_openings, l_closings, r_closings] after_transitions = before_transitions+1 seg_starts = np.unique(np.r_[0, after_transitions]) seg_ends = np.unique(np.r_[before_transitions, n_steps-1]) # Only count gripper open/closes if the number of time steps is greater # than ms_thresh lr_open = {lr:[] for lr in 'lr'} new_seg_starts = [] new_seg_ends = [] for i in range(len(seg_starts)): if seg_ends[i]- seg_starts[i] >= ms_thresh: new_seg_starts.append(seg_starts[i]) new_seg_ends.append(seg_ends[i]) lval = True if lgrip[seg_starts[i]] >= thresh else False lr_open['l'].append(lval) rval = True if rgrip[seg_starts[i]] >= thresh else False lr_open['r'].append(rval) return new_seg_starts, new_seg_ends, lr_open
def write_file(file_cc,file_bim,outfile): """ This function write a output file (outfile) from information in .epi.cc and .bim. The output file has 11 columns and the last two columns are ID (prb1 and prb2) for each snp. columns: CHR1 | SNP1 | CHR2 | SNP2 | OR_INT | STAT | P | POSITION1 | POSITION2 | prb1 | prb2 """ print "Reading ..." data_cc = get_data_cc(file_cc) data_bim = get_data_bim(file_bim) rs = np.unique(np.append( data_cc['SNP1'],data_cc['SNP2'] )) shape_data=np.shape(data_cc) ofile = open(outfile,'w') # open file for writing print "Writing ..." for i in range(shape_data[0]): ofile.write(str(data_cc[i]['CHR1'])+" " +str(data_cc[i]['SNP1'])+" " +str(data_cc[i]['CHR2'])+" " +str(data_cc[i]['SNP2'])+" " +str(data_cc[i]['OR_INT'])+" " +str(data_cc[i]['STAT'])+" " +str(data_cc[i]['P'])+" " +str(data_bim['position'][data_bim['rs']==data_cc[i]['SNP1']][0])+" " #position1 +str(data_bim['position'][data_bim['rs']==data_cc[i]['SNP2']][0])+" " #position2 +str(np.flatnonzero(rs==data_cc[i]['SNP1'])[0])+" " #id1 +str(np.flatnonzero(rs==data_cc[i]['SNP2'])[0])+" " #id2 np.flatnonzero( +"\n" ) ofile.close
def gen_crossed_logit_pandas(nc, cs, s1, s2): np.random.seed(3799) a = np.kron(np.arange(nc), np.ones(cs)) b = np.kron(np.ones(cs), np.arange(nc)) fe = np.ones(nc * cs) vc = np.zeros(nc * cs) for i in np.unique(a): ii = np.flatnonzero(a == i) vc[ii] += s1*np.random.normal() for i in np.unique(b): ii = np.flatnonzero(b == i) vc[ii] += s2*np.random.normal() lp = -0.5 * fe + vc pr = 1 / (1 + np.exp(-lp)) y = 1*(np.random.uniform(size=nc*cs) < pr) ident = np.zeros(2*nc, dtype=np.int) ident[nc:] = 1 df = pd.DataFrame({"fe": fe, "a": a, "b": b, "y": y}) return df
def getDBFormat(factX, modes, axisDict): """ Get the database format for this set of factorizations Parameters ------------ factX : the list of tensor factorizations modes: a list of all the unique modes in the tensor list axisDict : the mapping between the tensor modes and the axis information Output ----------- outStack: a 2-D array with 4 columns with mode, feature_name (derived from axisDict), factor, and value """ outStack = np.zeros((1, 4)) ## Get the lambdas for i in range(len(factX)): idx = np.flatnonzero(factX[i].lmbda) tmp = np.column_stack((np.repeat(-1 * (i+1), len(idx)), np.repeat("lambda", len(idx)).astype('S100'), idx, factX[i].lmbda[idx])) outStack = np.vstack((outStack, tmp)) midx = 0 for m in modes: tensorIdx = m.flatten()[0] tensorDim = m.flatten()[1] tTuple = (tensorIdx, tensorDim) print tTuple for r in range(factX[tensorIdx].R): idx = np.flatnonzero(factX[tensorIdx].U[tensorDim][:,r]) tmp = np.column_stack((np.repeat(midx, len(idx)), np.array(axisDict[tTuple], dtype="S100")[idx], np.repeat(r, len(idx)), factX[tensorIdx].U[tensorDim][idx, r])) outStack = np.vstack((outStack, tmp)) midx = midx + 1 return outStack
def __entrofy(X, k, w=None, q=None, pre_selects=None): '''See entrofy() for documentation''' n_participants, n_attributes = X.shape if w is None: w = np.ones(n_attributes) if q is None: q = 0.5 * np.ones(n_attributes) assert 0 < k <= n_participants assert not np.any(w < 0) assert np.all(q >= 0.0) and np.all(q <= 1.0) assert len(w) == n_attributes assert len(q) == n_attributes if k == n_participants: return np.arange(n_participants) # Initialization y = np.zeros(n_participants, dtype=bool) if pre_selects is None: # Select one at random pre_selects = np.random.choice(n_participants, size=1) y[pre_selects] = True # Where do we have missing data? Xn = np.isnan(X) while True: i = y.sum() if i >= k: break # Initialize the distribution vector p = np.nanmean(X[y], axis=0) p[np.isnan(p)] = 0.0 # Compute the candidate distributions p_new = (p * i + X) / (i + 1.0) # Wherever X is nan, propagate the old p since we have no new information p_new[Xn] = (Xn * p)[Xn] # Compute marginal gain for each candidate delta = obj(p_new, w, q) - obj(p, w, q) # Knock out the points we've already taken delta[y] = -np.inf # Select the top score. Break near-ties randomly. target_score = delta.max() target_score = target_score - 1e-3 * np.abs(target_score) new_idx = np.random.choice(np.flatnonzero(delta >= target_score)) y[new_idx] = True return obj(np.nanmean(X[y], axis=0), w, q), np.flatnonzero(y)
def __ComputeHistogram__(self, x, x_range): """Function to compute the histogram of the data Parameters ---------- x: ndarray Array containing the data x_range: tuple Tuple containing the minimum and maxumum to consider to build the histogram of the data x Returns ------- pdf: 1d-array Return the PDF of x using the range provided by the user bin_edge: 1d-array Return the bins corresponding to the PDF """ # Compute the histogram for the data x with unit bins pdf_rel, bin_edges_rel = np.histogram(x, bins=(np.max(x) - np.min(x)), density=True) # We need to translate the pdf depending of the range given by x_range ### Create an array with unit bins depending of x_range ### We need max - min + 1 bins pdf_abs = np.zeros((x_range[1] - x_range[0],)) bin_edges_abs = np.array(range(x_range[0], x_range[1] + 1)) ### Copy the relative pdf at the right position pdf_abs[np.flatnonzero(bin_edges_abs==bin_edges_rel[0])[0] : np.flatnonzero(bin_edges_abs==bin_edges_rel[-1])[0]] = pdf_rel[:] return (pdf_abs, bin_edges_abs)
def fixgaps(x): """FIXGAPS: Linearly interpolates gaps in a time series YOUT=FIXGAPS(YIN) linearly interpolates over NaN in the input time series (may be complex), but ignores trailing and leading NaNs. R. Pawlowicz 11/6/99 Version 1.0 """ #find nans bd = np.isnan(x) #early exit if there are no nans if not bd.any(): return x #find nonnans index numbers gd = np.flatnonzero(~bd) #ignore leading and trailing nans bd[:gd.min()]=False bd[(gd.max()+1):]=False #interpolate nans x[bd] = np.interp(np.flatnonzero(bd),gd,x[gd]) return x
def preprocessRawData(raw_data,y_raw): # if age >= 38, labeled with 1 # else, labeled with -1. labelvec = np.array(y_raw) y = np.ones(len(labelvec)) neg = labelvec < 38 y[neg] = -1 # positive examples: larger than or equal to 38 # negative examples: smaller than 38 num_pos = len(np.flatnonzero(y > 0)) num_neg = len(np.flatnonzero(y < 0)) #print('Number of positive/negative examples = %d/%d' % (num_pos, num_neg)) headers = list(raw_data.columns.values) # get the features' name raw_feat = np.array(raw_data[headers]) # feature matrix without age feature avg = np.mean(raw_feat,axis = 0) std_dev = np.std(raw_feat, axis = 0) X = (raw_feat-avg)/std_dev # scaled features matrix [-1,1] #print X.shape # X is N x n, where N is 342 subjects, n is 7 features #print y.shape # y is N x 1, where N is 342 subjects return X,y
def plotTrialPath(jsonData, trialNum, preTime=0, postTime=0, bPreRelStart=True, bPostRelEnd=True, bFolded=False, bLabels=True): if type(trialNum) == type(int()): trialNum = [trialNum] for tn in trialNum: trial = jsonData['trials'][tn] trialPath = getTrialPath(jsonData, tn, preTime, postTime, bPreRelStart, bPostRelEnd, bFolded) if trial['bAvoidedShock']: pyplot.plot(trialPath[:,1],trialPath[:,2], 'g') else: pyplot.plot(trialPath[:,1],trialPath[:,2], 'r') #plot the LED if bFolded or trial['side']==0: pyplot.plot(0,35/2.0,marker='o',mec='r',mfc='r',ms=10) pyplot.plot(80,35/2.0,marker='o',mec='r',mfc='None',ms=10) else: pyplot.plot(0,35/2.0,marker='o',mec='r',mfc='None',ms=10) pyplot.plot(80,35/2.0,marker='o',mec='r',mfc='r',ms=10) #plot dot where LED turned on. ndxLED = np.flatnonzero(trialPath[:,0]>0) if len(ndxLED)>0: pyplot.plot(trialPath[ndxLED[0],1],trialPath[ndxLED[0],2],marker='o',mec='r',mfc='r',ms=5) #plot dot where Shock starts. ndxShock = np.flatnonzero(trialPath[:,0]>jsonData['parameters']['LEDTimeMS']/1000.0 - .1) if not trial['bAvoidedShock'] and len(ndxShock)>0: pyplot.plot(trialPath[ndxShock[0],1],trialPath[ndxShock[0],2],marker='o',mec='y',mfc='y',ms=5) pyplot.text(trialPath[-1,1],trialPath[-1,2], str(tn)) pyplot.xlim([0,80]) pyplot.ylim([0,35]) pyplot.axvline(x=getEscapePosition(jsonData),color='k',ls='--') if bLabels: pyplot.xlabel('mm') pyplot.ylabel('mm')
def get_square_stim_characteristics(i, t, no_test_pulse=False): ''' Identify the start time, duration, amplitude, start index, and end index of a square stimulus. This assumes that there is a test pulse followed by the stimulus square. ''' di = np.diff(i) up_idx = np.flatnonzero(di > 0) down_idx = np.flatnonzero(di < 0) idx = 0 if no_test_pulse else 1 # second square is the stimulus if up_idx[idx] < down_idx[idx]: # positive square start_idx = up_idx[idx] + 1 # shift by one to compensate for diff() end_idx = down_idx[idx] + 1 else: # negative square start_idx = down_idx[idx] + 1 end_idx = up_idx[idx] + 1 stim_start = float(t[start_idx]) stim_dur = float(t[end_idx] - t[start_idx]) stim_amp = float(i[start_idx]) return (stim_start, stim_dur, stim_amp, start_idx, end_idx)
def do_pca_analysis(profiles, lens, name='', plot=False): L = np.array(0.446*(lens-np.mean(lens)), dtype='float64') pr = [] for i,p in enumerate(profiles): mask = np.isnan(p) p[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), p[~mask]) av, va = moving_average(np.log(p+0.001), 46, 100) pr.append(av) y = np.array(pr) pca = PCA(n_components=2) pca.fit(y) print pca.explained_variance_ratio_ yp = pca.transform(y) m,b,r,p,_ = stats.linregress(L, yp[:,0]) p1 = [p] r1 = [r] for _ in xrange(300): sample = np.random.choice(L.shape[0], L.shape[0], replace=True) m,b,r,p,_ = stats.linregress(L[~sample], yp[~sample,0]) p1.append(p) r1.append(r) m,b,r,p,_ = stats.linregress(L, yp[:,1]) p2 = [p] r2 = [r] for _ in xrange(300): sample = np.random.choice(L.shape[0], L.shape[0], replace=True) m,b,r,p,_ = stats.linregress(L[~sample], yp[~sample,1]) p2.append(p) r2.append(r) if plot: plot_pca(y, pca, yp, L, name) return r1, p1, r2, p2, L.shape[0], name, np.std(L)
def splint(xa, ya, y2a, x): '''spline interpolation''' try: len(x) except TypeError: x = np.array([x]) try: klo, khi = np.array([ (np.flatnonzero(xa < i)[-1], np.flatnonzero(xa > i)[0]) for i in x ]).transpose() except IndexError: raise ValueError( 'Input values must be between %s and %s' % (np.exp(xa[0]), np.exp(xa[-1])) ) h = xa[khi] - xa[klo] if any(h <= 0): raise ValueError, 'xa input must be strictly increasing' a = (xa[khi] - x) / h b = (x - xa[klo]) / h res = ( a * ya[klo] + b * ya[khi] + ((a**3 - a) * y2a[klo] + (b**3 - b) * y2a[khi]) * (h**2) / 6 ) return res
def time_of_state_transition(self,prevStateID,nextStateID): '''Returns the time of the transition from prevStateID to nextStateID on each trial. If prevStateID is an empty string it calculates transition from any state. It assumes that the states have the same ID on all trials. If the transition did not occur, it returns NaN for that trial. If it occurs more than once, it returns the first time it happens. ''' # -- If input is a state name, find its ID -- if isinstance(nextStateID,str): nextStateIDval = self.find_stateID(nextStateID) else: nextStateIDval = nextStateID if len(prevStateID): USE_PREVSTATE = True if isinstance(prevStateID,str): prevStateIDval = self.find_stateID(prevStateID) else: prevStateIDval = prevStateID else: USE_PREVSTATE = False transitionEventTimes = np.empty(self['nTrials']) transitionEventTimes.fill(np.nan) eventsIndList = events_each_trial(self['RawEventsTrialID']) for trialInd,eventsInd in enumerate(eventsIndList): if USE_PREVSTATE: transitions = (self['RawEvents'][eventsInd,0]==prevStateIDval) &\ (self['RawEvents'][eventsInd,3]==nextStateIDval) transitionEventInds = np.flatnonzero(transitions) else: transitionEventInds = np.flatnonzero(self['RawEvents'][eventsInd,3]==nextStateIDval) if len(transitionEventInds): # Note that we store only the first transition eventOfInterest = eventsInd[transitionEventInds[0]] transitionEventTimes[trialInd] = self['RawEvents'][eventOfInterest,2] return transitionEventTimes
def plot_predictions(predictions, view_date_begin=None, view_date_end=None, ax=None): """Creates a plot of the observed series and model predictions. Creates a `matplotlib` plot of the observed time series with intervals from the model, predictions for any unobserved points, and the locations of any anomalies. Args: predictions: instance of `PredictionOutput` as returned by `detect_anomalies`. This should contain predictions for a single series with no batch dimensions. view_date_begin: Optional `datetime.datetime` instance. view_date_end: Optional `datetime.datetime` instance. ax: Optional `matplotlib` figure axis. """ # pylint: disable=g-import-not-at-top from matplotlib import pylab as plt from matplotlib import dates as mdates # pylint: enable=g-import-not-at-top if len(predictions.observed_time_series.shape) > 1: raise ValueError( 'Time series must be one-dimensional; batches are not ' 'supported. Saw shape: {}.'.format( predictions.observed_time_series.shape)) num_steps = len(predictions.times) time_delta = predictions.times[1] - predictions.times[0] time_period_length = predictions.times[-1] - predictions.times[0] if view_date_begin is None: view_date_begin = predictions.times[0] - 0.04 * time_period_length if view_date_end is None: view_date_end = predictions.times[-1] + 0.04 * time_period_length if not (isinstance(view_date_begin, datetime.datetime) and isinstance(view_date_end, datetime.datetime)): raise ValueError('View date start and end must be `datetime.datetime` ' 'instances.') if ax is None: # Create default axis. fig = plt.figure(figsize=(15, 5), constrained_layout=True) ax = fig.add_subplot(1, 1, 1) # Plot series with upper and lower limits. ax.plot(predictions.times, predictions.observed_time_series, color='black', alpha=0.8) ax.fill_between(predictions.times, predictions.lower_limit, predictions.upper_limit, color='tab:blue', alpha=0.3) # At steps where no time series was observed, plot the predictive mean. ax.plot(predictions.times, np.where(np.isnan(predictions.observed_time_series), predictions.mean, np.nan), color='black', alpha=0.8, ls='--') # Highlight anomalies. for anomaly_idx in np.flatnonzero(predictions.is_anomaly): x = predictions.times[anomaly_idx] y = predictions.observed_time_series[anomaly_idx] ax.scatter(x, y, s=100, alpha=0.4, c='red') ax.annotate(str(x), (x, y)) ax.set_ylabel('Series') ax.label_outer() # Use smart date formatting for the x axis. locator = mdates.AutoDateLocator(minticks=3, maxticks=7) ax.xaxis.set_major_locator(locator) ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(locator)) ax.set_xlim([view_date_begin, view_date_end]) ax.grid(True, color='whitesmoke') # Display the grid *underneath* the rest of the plot # (see https://github.com/matplotlib/matplotlib/issues/5045/). ax.set_axisbelow(True)
def fit(self, X, y): X, y = check_X_y(X, y, accept_sparse=True) self.is_fitted_ = True # transforming data raw_data = np.insert(X, 0, y, axis=1) N = raw_data.shape[0] # setup Y vector and Y_name Y_col_idx = [0] Y = raw_data[:, Y_col_idx] Y_name = self.data_headers[Y_col_idx[0]] Y[Y == 0] = -1 # setup X and X_names X_col_idx = [j for j in range(raw_data.shape[1]) if j not in Y_col_idx] X = raw_data[:, X_col_idx] variable_names = [self.data_headers[j] for j in X_col_idx] # insert a column of ones to X for the intercept X = np.insert(arr=X, obj=0, values=np.ones(N), axis=1) variable_names.insert(0, '(Intercept)') if self.sample_weights is None or len(self.sample_weights) != N: self.sample_weights = np.ones(N) self.data = { 'X': X, 'Y': Y, 'variable_names': variable_names, 'outcome_name': Y_name, 'sample_weights': self.sample_weights, } #load folds if self.fold_csv_file is not None: if not os.path.isfile(self.fold_csv_file): raise IOError('could not find fold_csv_file: %s' % self.fold_csv_file) else: fold_idx = pd.read_csv(self.fold_csv_file, sep=',', header=None) fold_idx = fold_idx.values.flatten() K = max(fold_idx) all_fold_nums = np.sort(np.unique(fold_idx)) assert len( fold_idx ) == N, "dimension mismatch: read %r fold indices (expected N = %r)" % ( len(fold_idx), N) assert np.all(all_fold_nums == np.arange( 1, K + 1)), "folds should contain indices between 1 to %r" % K assert fold_num in np.arange( 0, K + 1 ), "fold_num should either be 0 or an integer between 1 to %r" % K if fold_num >= 1: test_idx = fold_num == fold_idx train_idx = fold_num != fold_idx data['X'] = data['X'][train_idx, ] data['Y'] = data['Y'][train_idx] data['sample_weights'] = data['sample_weights'][train_idx] assert check_data(self.data) # create coefficient set and set the value of the offset parameter coef_set = CoefficientSet(variable_names=self.data['variable_names'], lb=-self.max_coefficient, ub=self.max_coefficient, sign=0) conservative_offset = get_conservative_offset(self.data, coef_set, self.max_L0_value) self.max_offset = min(self.max_offset, conservative_offset) coef_set['(Intercept)'].ub = self.max_offset coef_set['(Intercept)'].lb = -self.max_offset # edit contraints here constraints = { 'L0_min': 0, 'L0_max': self.max_L0_value, 'coef_set': coef_set, } # initialize MIP for lattice CPA mip_objects = setup_lattice_cpa(self.data, constraints, self.settings) # add operational constraints mip = mip_objects['mip'] indices = mip_objects['indices'] get_alpha_name = lambda var_name: 'alpha_' + str(self.data[ 'variable_names'].index(var_name)) get_alpha_ind = lambda var_names: [ get_alpha_name(v) for v in var_names ] # applies mutual exclusivity feature contraints if self.op_constraints is not None: names = [] expressions = [] for key in self.op_constraints.keys(): names.append("mutually_exclusive_%s" % key) expressions.append( cplex.SparsePair( ind=get_alpha_ind(self.op_constraints[key]), val=[1.0] * len(self.op_constraints[key]))) mip.linear_constraints.add( names=names, lin_expr=expressions, senses=["L"] * len(self.op_constraints.keys()), rhs=[1.0] * len(self.op_constraints.keys())) mip_objects['mip'] = mip # fit using ltca model_info, mip_info, lcpa_info = finish_lattice_cpa( self.data, constraints, mip_objects, self.settings) rho = model_info['solution'] self.model_info = model_info if np.sum(rho[1:]) != 0: print_model(model_info['solution'], self.data) print("solver_time = %d" % model_info['solver_time']) print("optimality_gap = %.3f" % model_info['optimality_gap']) print(rho) variable_names = self.data['variable_names'] rho_values = np.copy(rho) rho_names = list(variable_names) # removes intercept value or sets it to 0 if '(Intercept)' in rho_names: intercept_ind = variable_names.index('(Intercept)') self.intercept_val = int(rho[intercept_ind]) rho_values = np.delete(rho_values, intercept_ind) rho_names.remove('(Intercept)') else: self.intercept_val = 0 self.filter_mask = np.array(rho_values) != 0 # removes zero values if not self.show_omitted_variables: selected_ind = np.flatnonzero(rho_values) self.rho_values = rho_values[selected_ind] self.rho_names = [rho_names[i] for i in selected_ind] return self
def read(self, t=None, z=None, y=None, x=None, N=None, K=None, J=None, I=None, var=None, nonan=True, result='full', profile=False, dummy=False): """Reads dataset. PARAMETERS t, z, y, x (array like, optional) : Sets the time, height, latitude and longitude for which the data will be read. N, K, J, I (array like, optional) : Sets the temporal, vertical, meridional and zonal indices for which the data will be read. var (string, optional) : Indicates which variable of the grid will be read. If the parameter is a list of variables, then the data will be returned as a list of arrays. nonan (boolean, optional) : If set to true (default) changes data values containing NaN to zero, preserving the mask. result (string, optional) : Determines wheter all time, height, latitude, longitude and data will be returned ('full', default), if temporal, vertical, meridional and zonal indices are returned instead ('indices'), or if only variable data is returned ('var only'). components (list, optional) : A list containing which components will be included in the calculation. Options are the seasonal cycle ('seasonal'), westward propagating planetary waves ('planetary'), eddy fields ('eddy') and noise ('noise'). profile (boolean, optional) : Sets whether the status is send to screen. dummy (boolean, optional) : If set to true, does not load data and returns the shape of the array that would have been returned. RETURNS t, z, y, x, dat (array like) : If 'result' is set to 'full', then all coordinates and data variables are returned. N, K, J, I, var (array like) : If 'result' is set to 'indices', then all indices and data variables are returned. dat (array like) : If 'result' is set to 'var only', then the data is returned. """ global DEBUG t1 = time() # Checks input variables for consistency. if (t != None) & (N != None): raise ValueError('Both time and temporal index were provided.') if (z != None) & (K != None): raise ValueError('Both height and vertical index were provided.') if (y != None) & (J != None): raise ValueError( 'Both latitude and meridional index were provided.') if (x != None) & (I != None): raise ValueError('Both latitude and zonal index were provided.') # Checks for variables indices. Intersects desired input values with # dataset dimesion data. In this dataset, since only surface data is # available, the height values are always zero. if t != None: N = flatnonzero(in1d(self.variables['time'].data, t)) elif N == None: N = arange(self.dimensions['n']) if z != None: K = [0] elif K == None: K = [0] elif K != None: K = [0] if y != None: J = flatnonzero(in1d(self.variables['latitude'].data, y)) elif J == None: J = arange(self.dimensions['j']) if x != None: I = flatnonzero(in1d(self.variables['longitude'].data, x)) elif I == None: I = arange(self.dimensions['i']) # Sets the shape of the data array. shape = (len(N), 1, len(J), len(I)) if dummy: return shape # Selects data according to indices. t = self.variables['time'].data[N] z = self.variables['height'].data y = self.variables['latitude'].data[J] x = self.variables['longitude'].data[I] xx, yy = meshgrid(x, y) II, JJ = meshgrid(I, J) var = ma.zeros(shape) # Walks through every time index and loads data range from maps. for n, T in enumerate(t): t2 = time() if profile: s = '\rLoading data... %s ' % (profiler( shape[0], n + 1, 0, t1, t2), ) stdout.write(s) stdout.flush() # Uncompresses and reads HDF file HDF = self._open_HDF( '%s/%s' % (self.params['path'], self.params['file_list'][N[n]])) HDF_att = HDF.attributes() # Loads scientific dataset (SDS) and calculates the parameter # value using scalling equation, slope and intercept. SDS_name = HDF.datasets().keys()[0] SDS = HDF.select(SDS_name) SDS_att = SDS.attributes() if (('lon_i' in self.params.keys()) & ('lat_j' in self.params.keys())): P = SDS[:, :][self.params['lat_j'], self.params['lon_i']][JJ, II] else: P = SDS[:, :][JJ, II] P[P <= SDS_att['Fill']] = nan P = (SDS_att['Slope'] * P + SDS_att['Intercept']) P = ma.masked_where(isnan(P), P) if nonan: P.data[P.mask] = 0 # var[n, 0, :, :] = P[None, None, :, :] if profile: stdout.write('\r\n') stdout.flush() if DEBUG: print 't: ', t print 'z: ', z print 'y:', y print 'x:', x print 'var: ', var print 'N: ', N print 'K: ', K print 'J: ', J print 'I:', I print 'shape: ', shape if result == 'full': return t, z, y, x, var elif result == 'indices': return N, K, J, I, var elif result == 'var only': return var else: raise Warning("Result parameter set imporperly to '%s', " "assuming 'var only'." % (result)) return var
def reconstructed_codes(self) -> List[np.ndarray]: # get unique result indices, and prepend 0 as groupby starts from the first return [np.r_[0, np.flatnonzero(self.bins[1:] != self.bins[:-1]) + 1]]
def onset_backtrack(events, energy): '''Backtrack detected onset events to the nearest preceding local minimum of an energy function. This function can be used to roll back the timing of detected onsets from a detected peak amplitude to the preceding minimum. This is most useful when using onsets to determine slice points for segmentation, as described by [1]_. .. [1] Jehan, Tristan. "Creating music by listening" Doctoral dissertation Massachusetts Institute of Technology, 2005. Parameters ---------- events : np.ndarray, dtype=int List of onset event frame indices, as computed by `onset_detect` energy : np.ndarray, shape=(m,) An energy function Returns ------- events_backtracked : np.ndarray, shape=events.shape The input events matched to nearest preceding minima of `energy`. Examples -------- >>> y, sr = librosa.load(librosa.util.example_audio_file(), ... offset=30, duration=2.0) >>> oenv = librosa.onset.onset_strength(y=y, sr=sr) >>> # Detect events without backtracking >>> onset_raw = librosa.onset.onset_detect(onset_envelope=oenv, ... backtrack=False) >>> # Backtrack the events using the onset envelope >>> onset_bt = librosa.onset.onset_backtrack(onset_raw, oenv) >>> # Backtrack the events using the RMS energy >>> rmse = librosa.feature.rmse(S=np.abs(librosa.stft(y=y))) >>> onset_bt_rmse = librosa.onset.onset_backtrack(onset_raw, rmse[0]) >>> # Plot the results >>> import matplotlib.pyplot as plt >>> plt.figure() >>> plt.subplot(2,1,1) >>> plt.plot(oenv, label='Onset strength') >>> plt.vlines(onset_raw, 0, oenv.max(), label='Raw onsets') >>> plt.vlines(onset_bt, 0, oenv.max(), label='Backtracked', color='r') >>> plt.legend(frameon=True, framealpha=0.75) >>> plt.subplot(2,1,2) >>> plt.plot(rmse[0], label='RMSE') >>> plt.vlines(onset_bt_rmse, 0, rmse.max(), label='Backtracked (RMSE)', color='r') >>> plt.legend(frameon=True, framealpha=0.75) ''' # Find points where energy is non-increasing # all points: energy[i] <= energy[i-1] # tail points: energy[i] < energy[i+1] minima = np.flatnonzero((energy[1:-1] <= energy[:-2]) & (energy[1:-1] < energy[2:])) # Pad on a 0, just in case we have onsets with no preceding minimum # Shift by one to account for slicing in minima detection minima = util.fix_frames(1 + minima, x_min=0) # Only match going left from the detected events return minima[util.match_events(events, minima, right=False)]
def plot(self, column=None, plot_close=True, plot_zones=True, close_trace_kwargs=None, entry_trace_kwargs=None, exit_trace_kwargs=None, exit_profit_trace_kwargs=None, exit_loss_trace_kwargs=None, active_trace_kwargs=None, profit_shape_kwargs=None, loss_shape_kwargs=None, add_trace_kwargs=None, xref='x', yref='y', fig=None, **layout_kwargs): # pragma: no cover """Plot orders. Args: column (str): Name of the column to plot. plot_close (bool): Whether to plot `Trades.close`. plot_zones (bool): Whether to plot zones. Set to False if there are many trades within one position. close_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for `Trades.close`. entry_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Entry" markers. exit_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit" markers. exit_profit_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit - Profit" markers. exit_loss_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Exit - Loss" markers. active_trace_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Scatter` for "Active" markers. profit_shape_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Figure.add_shape` for profit zones. loss_shape_kwargs (dict): Keyword arguments passed to `plotly.graph_objects.Figure.add_shape` for loss zones. add_trace_kwargs (dict): Keyword arguments passed to `add_trace`. xref (str): X coordinate axis. yref (str): Y coordinate axis. fig (plotly.graph_objects.Figure): Figure to add traces to. **layout_kwargs: Keyword arguments for layout. ## Example ```python-repl >>> trades.plot() ``` ![](/vectorbt/docs/img/trades_plot.png)""" from vectorbt.settings import color_schema, contrast_color_schema self_col = self.select_series(column=column, group_by=False) if close_trace_kwargs is None: close_trace_kwargs = {} close_trace_kwargs = merge_dicts(dict( line_color=color_schema['blue'], name='Close' if self_col.wrapper.name is None else self_col.wrapper.name ), close_trace_kwargs) if entry_trace_kwargs is None: entry_trace_kwargs = {} if exit_trace_kwargs is None: exit_trace_kwargs = {} if exit_profit_trace_kwargs is None: exit_profit_trace_kwargs = {} if exit_loss_trace_kwargs is None: exit_loss_trace_kwargs = {} if active_trace_kwargs is None: active_trace_kwargs = {} if profit_shape_kwargs is None: profit_shape_kwargs = {} if loss_shape_kwargs is None: loss_shape_kwargs = {} if add_trace_kwargs is None: add_trace_kwargs = {} if fig is None: fig = FigureWidget() fig.update_layout(**layout_kwargs) # Plot close if plot_close: fig = self_col.close.vbt.plot(trace_kwargs=close_trace_kwargs, add_trace_kwargs=add_trace_kwargs, fig=fig) if len(self_col.values) > 0: # Extract information _id = self_col.values['id'] _id_str = 'Trade Id' if self.trade_type == TradeType.Trade else 'Position Id' size = self_col.values['size'] entry_idx = self_col.values['entry_idx'] entry_price = self_col.values['entry_price'] entry_fees = self_col.values['entry_fees'] exit_idx = self_col.values['exit_idx'] exit_price = self_col.values['exit_price'] exit_fees = self_col.values['exit_fees'] pnl = self_col.values['pnl'] ret = self_col.values['return'] direction_value_map = to_value_map(TradeDirection) direction = self_col.values['direction'] direction = np.vectorize(lambda x: str(direction_value_map[x]))(direction) status = self_col.values['status'] def get_duration_str(from_idx, to_idx): if isinstance(self_col.wrapper.index, DatetimeTypes): duration = self_col.wrapper.index[to_idx] - self_col.wrapper.index[from_idx] elif self_col.wrapper.freq is not None: duration = self_col.wrapper.to_time_units(to_idx - from_idx) else: duration = to_idx - from_idx return np.vectorize(str)(duration) duration = get_duration_str(entry_idx, exit_idx) if len(entry_idx) > 0: # Plot Entry markers entry_customdata = np.stack(( _id, size, entry_fees, direction, *((self_col.values['position_id'],) if self.trade_type == TradeType.Trade else ()) ), axis=1) entry_scatter = go.Scatter( x=self_col.wrapper.index[entry_idx], y=entry_price, mode='markers', marker=dict( symbol='square', color=contrast_color_schema['blue'], size=7, line=dict( width=1, color=adjust_lightness(contrast_color_schema['blue']) ) ), name='Entry', customdata=entry_customdata, hovertemplate=_id_str + ": %{customdata[0]}" "<br>Date: %{x}" "<br>Avg. Price: %{y}" "<br>Size: %{customdata[1]:.6f}" "<br>Fees: %{customdata[2]:.6f}" "<br>Direction: %{customdata[3]}" + ("<br>Position Id: %{customdata[4]}" if self.trade_type == TradeType.Trade else '') ) entry_scatter.update(**entry_trace_kwargs) fig.add_trace(entry_scatter, **add_trace_kwargs) # Plot end markers def _plot_end_markers(mask, name, color, kwargs): if np.any(mask): customdata = np.stack(( _id[mask], duration[mask], size[mask], exit_fees[mask], pnl[mask], ret[mask], direction[mask], *((self_col.values['position_id'][mask],) if self.trade_type == TradeType.Trade else ()) ), axis=1) scatter = go.Scatter( x=self_col.wrapper.index[exit_idx[mask]], y=exit_price[mask], mode='markers', marker=dict( symbol='square', color=color, size=7, line=dict( width=1, color=adjust_lightness(color) ) ), name=name, customdata=customdata, hovertemplate=_id_str + ": %{customdata[0]}" "<br>Date: %{x}" "<br>Duration: %{customdata[1]}" "<br>Avg. Price: %{y}" "<br>Size: %{customdata[2]:.6f}" "<br>Fees: %{customdata[3]:.6f}" "<br>PnL: %{customdata[4]:.6f}" "<br>Return: %{customdata[5]:.2%}" "<br>Direction: %{customdata[6]}" + ("<br>Position Id: %{customdata[7]}" if self.trade_type == TradeType.Trade else '') ) scatter.update(**kwargs) fig.add_trace(scatter, **add_trace_kwargs) # Plot Exit markers _plot_end_markers( (status == TradeStatus.Closed) & (pnl == 0.), 'Exit', contrast_color_schema['gray'], exit_trace_kwargs ) # Plot Exit - Profit markers _plot_end_markers( (status == TradeStatus.Closed) & (pnl > 0.), 'Exit - Profit', contrast_color_schema['green'], exit_profit_trace_kwargs ) # Plot Exit - Loss markers _plot_end_markers( (status == TradeStatus.Closed) & (pnl < 0.), 'Exit - Loss', contrast_color_schema['red'], exit_loss_trace_kwargs ) # Plot Active markers _plot_end_markers( status == TradeStatus.Open, 'Active', contrast_color_schema['orange'], active_trace_kwargs ) if plot_zones: profit_mask = pnl > 0. if np.any(profit_mask): # Plot profit zones for i in np.flatnonzero(profit_mask): fig.add_shape(**merge_dicts(dict( type="rect", xref=xref, yref=yref, x0=self_col.wrapper.index[entry_idx[i]], y0=entry_price[i], x1=self_col.wrapper.index[exit_idx[i]], y1=exit_price[i], fillcolor='green', opacity=0.2, layer="below", line_width=0, ), profit_shape_kwargs)) loss_mask = pnl < 0. if np.any(loss_mask): # Plot loss zones for i in np.flatnonzero(loss_mask): fig.add_shape(**merge_dicts(dict( type="rect", xref=xref, yref=yref, x0=self_col.wrapper.index[entry_idx[i]], y0=entry_price[i], x1=self_col.wrapper.index[exit_idx[i]], y1=exit_price[i], fillcolor='red', opacity=0.2, layer="below", line_width=0, ), loss_shape_kwargs)) return fig
def _send_procs_to_workers(self, updatehash=False, graph=None): """ Sends jobs to workers when system resources are available. """ # Check to see if a job is available (jobs with all dependencies run) # See https://github.com/nipy/nipype/pull/2200#discussion_r141605722 # See also https://github.com/nipy/nipype/issues/2372 jobids = np.flatnonzero(~self.proc_done & (self.depidx.sum(axis=0) == 0).__array__()) # Check available resources by summing all threads and memory used free_memory_gb, free_processors = self._check_resources( self.pending_tasks) stats = ( len(self.pending_tasks), len(jobids), free_memory_gb, self.memory_gb, free_processors, self.processors, ) if self._stats != stats: tasks_list_msg = "" if logger.level <= INFO: running_tasks = [ " * %s" % self.procs[jobid].fullname for _, jobid in self.pending_tasks ] if running_tasks: tasks_list_msg = "\nCurrently running:\n" tasks_list_msg += "\n".join(running_tasks) tasks_list_msg = indent(tasks_list_msg, " " * 21) logger.info( "[MultiProc] Running %d tasks, and %d jobs ready. Free " "memory (GB): %0.2f/%0.2f, Free processors: %d/%d.%s", len(self.pending_tasks), len(jobids), free_memory_gb, self.memory_gb, free_processors, self.processors, tasks_list_msg, ) self._stats = stats if free_memory_gb < 0.01 or free_processors == 0: logger.debug("No resources available") return if len(jobids) + len(self.pending_tasks) == 0: logger.debug("No tasks are being run, and no jobs can " "be submitted to the queue. Potential deadlock") return jobids = self._sort_jobs(jobids, scheduler=self.plugin_args.get("scheduler")) # Run garbage collector before potentially submitting jobs gc.collect() # Submit jobs for jobid in jobids: # First expand mapnodes if isinstance(self.procs[jobid], MapNode): try: num_subnodes = self.procs[jobid].num_subnodes() except Exception: traceback = format_exception(*sys.exc_info()) self._clean_queue(jobid, graph, result={ "result": None, "traceback": traceback }) self.proc_pending[jobid] = False continue if num_subnodes > 1: submit = self._submit_mapnode(jobid) if not submit: continue # Check requirements of this job next_job_gb = min(self.procs[jobid].mem_gb, self.memory_gb) next_job_th = min(self.procs[jobid].n_procs, self.processors) # If node does not fit, skip at this moment if next_job_th > free_processors or next_job_gb > free_memory_gb: logger.debug( "Cannot allocate job %d (%0.2fGB, %d threads).", jobid, next_job_gb, next_job_th, ) continue free_memory_gb -= next_job_gb free_processors -= next_job_th logger.debug( "Allocating %s ID=%d (%0.2fGB, %d threads). Free: " "%0.2fGB, %d threads.", self.procs[jobid].fullname, jobid, next_job_gb, next_job_th, free_memory_gb, free_processors, ) # change job status in appropriate queues self.proc_done[jobid] = True self.proc_pending[jobid] = True # If cached and up-to-date just retrieve it, don't run if self._local_hash_check(jobid, graph): continue # updatehash and run_without_submitting are also run locally if updatehash or self.procs[jobid].run_without_submitting: logger.debug("Running node %s on master thread", self.procs[jobid]) try: self.procs[jobid].run(updatehash=updatehash) except Exception: traceback = format_exception(*sys.exc_info()) self._clean_queue(jobid, graph, result={ "result": None, "traceback": traceback }) # Release resources self._task_finished_cb(jobid) self._remove_node_dirs() free_memory_gb += next_job_gb free_processors += next_job_th # Display stats next loop self._stats = None # Clean up any debris from running node in main process gc.collect() continue # Task should be submitted to workers # Send job to task manager and add to pending tasks if self._status_callback: self._status_callback(self.procs[jobid], "start") tid = self._submit_job(deepcopy(self.procs[jobid]), updatehash=updatehash) if tid is None: self.proc_done[jobid] = False self.proc_pending[jobid] = False else: self.pending_tasks.insert(0, (tid, jobid)) # Display stats next loop self._stats = None
if I_minute[C_i] == 0: #slice out BT images for the current basin C_BTemp = xr.open_dataset(IRDIR + "\\" + BTemp_filename)['Tb'].values[0][ DIM_BOUND[0]:DIM_BOUND[1] + 1, DIM_BOUND[2]:DIM_BOUND[3] + 1] C_lat = xr.open_dataset( IRDIR + "\\" + BTemp_filename)['lat'].values[DIM_BOUND[0]:DIM_BOUND[1] + 1] C_lon = xr.open_dataset( IRDIR + "\\" + BTemp_filename)['lon'].values[DIM_BOUND[2]:DIM_BOUND[3] + 1] #interpolate NaN values in BT images mask = np.isnan(C_BTemp) C_BTemp[mask] = np.interp(np.flatnonzero(mask), np.flatnonzero(~mask), C_BTemp[~mask]) elif I_minute[C_i] == 30: #slice out BT images for the current basin C_BTemp = xr.open_dataset(IRDIR + "\\" + BTemp_filename)['Tb'].values[1][ DIM_BOUND[0]:DIM_BOUND[1] + 1, DIM_BOUND[2]:DIM_BOUND[3] + 1] C_lat = xr.open_dataset( IRDIR + "\\" + BTemp_filename)['lat'].values[DIM_BOUND[0]:DIM_BOUND[1] + 1] C_lon = xr.open_dataset( IRDIR + "\\" + BTemp_filename)['lon'].values[DIM_BOUND[2]:DIM_BOUND[3] + 1] #interpolate NaN values in BT images
def digest_in_buckets(v, buckets_number): digested = set() for i in np.flatnonzero(v).flat: digested.add(i % buckets_number) return tuple(digested)
def unsaturated_case_3(self, dt, ia, case_3): """ Solve Green-Ampt model for second unsaturated case: 1) Soil is unsaturated 2) Available rainfall is greater than saturated hydraulic conductivity Inputs: ------- dt: float Time step (seconds) ia: np.ndarray (float) Available rainfall (m/s) case_3: np.ndarray (bool) Indicates whether case 3 applies to given element """ # 5. If availble rainfall rate exceeds saturated hydr. conductivity orig_ia = np.copy(ia) ia = ia[case_3] is_saturated = self.is_saturated[case_3] Ks = self.Ks[case_3] Lu = self.Lu[case_3] theta_dmax = self.theta_dmax[case_3] psi_f = self.psi_f[case_3] theta_d = self.theta_d[case_3] # Variables to write F = self.F[case_3] f = self.f[case_3] theta_du = self.theta_du[case_3] # Reset recovery time self.T[case_3] = self.Tr[case_3] # Compute volume needed to saturate surface layer Fs = Ks * psi_f * theta_d / (ia - Ks) cond_0 = (F >= Fs) cond_1 = (F + ia * dt < Fs) & (~cond_0) cond_2 = (~cond_0 & ~cond_1) # For first case if cond_0.any(): # Set to saturated is_saturated[cond_0] = True self.is_saturated[case_3] = is_saturated # Run saturated case # TODO: This is kind of confusing and probably not efficient sat_case = np.zeros(self.N, dtype=bool) sat_case[np.flatnonzero(case_3)[np.flatnonzero(cond_0)]] = True self.saturated_case(dt, orig_ia, sat_case) F[cond_0] = self.F[case_3][cond_0] f[cond_0] = self.f[case_3][cond_0] theta_du[cond_0] = self.theta_du[case_3][cond_0] # Run second case if cond_1.any(): f[cond_1] = ia[cond_1] dF = ia[cond_1] * dt F[cond_1] += dF theta_du[cond_1] -= dF / Lu[cond_1] theta_du[cond_1] = np.minimum(np.maximum(theta_du[cond_1], 0), theta_dmax[cond_1]) # Run third case # Solve integrated equation if cond_2.any(): sub_dt = dt - (Fs[cond_2] - F[cond_2]) / ia[cond_2] n = sub_dt.size F_2 = np.zeros(n, dtype=float) # Run green ampt F_2 = run_green_ampt_newton(F_2, F[cond_2], Fs[cond_2], sub_dt, Ks[cond_2], theta_d[cond_2], psi_f[cond_2], ia[cond_2]) dF = F_2 - F[cond_2] F[cond_2] = F_2 theta_du[cond_2] -= dF / Lu[cond_2] theta_du[cond_2] = np.minimum(np.maximum(theta_du[cond_2], 0), theta_dmax[cond_2]) f[cond_2] = dF / dt # Export instance variables self.F[case_3] = F self.f[case_3] = f self.theta_du[case_3] = theta_du
def convertNansToZeros(ma): nan_elements = np.flatnonzero(np.isnan(ma.data)) if len(nan_elements) > 0: ma.data[nan_elements] = 0.0 return ma
def _join(left, right, keys=None, join_type='inner', uniq_col_name='{col_name}_{table_name}', table_names=['1', '2'], col_name_map=None, metadata_conflicts='warn'): """ Perform a join of the left and right Tables on specified keys. Parameters ---------- left : Table Left side table in the join right : Table Right side table in the join keys : str or list of str Name(s) of column(s) used to match rows of left and right tables. Default is to use all columns which are common to both tables. join_type : str Join type ('inner' | 'outer' | 'left' | 'right'), default is 'inner' uniq_col_name : str or None String generate a unique output column name in case of a conflict. The default is '{col_name}_{table_name}'. table_names : list of str or None Two-element list of table names used when generating unique output column names. The default is ['1', '2']. col_name_map : empty dict or None If passed as a dict then it will be updated in-place with the mapping of output to input column names. Returns ------- joined_table : `~astropy.table.Table` object New table containing the result of the join operation. """ # Store user-provided col_name_map until the end _col_name_map = col_name_map if join_type not in ('inner', 'outer', 'left', 'right'): raise ValueError( "The 'join_type' argument should be in 'inner', " "'outer', 'left' or 'right' (got '{}' instead)".format(join_type)) # If we have a single key, put it in a tuple if keys is None: keys = tuple(name for name in left.colnames if name in right.colnames) if len(keys) == 0: raise TableMergeError( 'No keys in common between left and right tables') elif isinstance(keys, str): keys = (keys, ) # Check the key columns for arr, arr_label in ((left, 'Left'), (right, 'Right')): for name in keys: if name not in arr.colnames: raise TableMergeError( '{} table does not have key column {!r}'.format( arr_label, name)) if hasattr(arr[name], 'mask') and np.any(arr[name].mask): raise TableMergeError( '{} key column {!r} has missing values'.format( arr_label, name)) if not isinstance(arr[name], np.ndarray): raise ValueError( "non-ndarray column '{}' not allowed as a key column". format(name)) len_left, len_right = len(left), len(right) if len_left == 0 or len_right == 0: raise ValueError( 'input tables for join must both have at least one row') # Joined array dtype as a list of descr (name, type_str, shape) tuples col_name_map = get_col_name_map([left, right], keys, uniq_col_name, table_names) out_descrs = get_descrs([left, right], col_name_map) # Make an array with just the key columns. This uses a temporary # structured array for efficiency. out_keys_dtype = [descr for descr in out_descrs if descr[0] in keys] out_keys = np.empty(len_left + len_right, dtype=out_keys_dtype) for key in keys: out_keys[key][:len_left] = left[key] out_keys[key][len_left:] = right[key] idx_sort = out_keys.argsort(order=keys) out_keys = out_keys[idx_sort] # Get all keys diffs = np.concatenate(([True], out_keys[1:] != out_keys[:-1], [True])) idxs = np.flatnonzero(diffs) # Main inner loop in Cython to compute the cartesian product # indices for the given join type int_join_type = {'inner': 0, 'outer': 1, 'left': 2, 'right': 3}[join_type] masked, n_out, left_out, left_mask, right_out, right_mask = \ _np_utils.join_inner(idxs, idx_sort, len_left, int_join_type) out = _get_out_class([left, right])() for out_name, dtype, shape in out_descrs: left_name, right_name = col_name_map[out_name] if left_name and right_name: # this is a key which comes from left and right cols = [left[left_name], right[right_name]] col_cls = _get_out_class(cols) if not hasattr(col_cls.info, 'new_like'): raise NotImplementedError( 'join unavailable for mixin column type(s): {}'.format( col_cls.__name__)) out[out_name] = col_cls.info.new_like(cols, n_out, metadata_conflicts, out_name) if issubclass(col_cls, Column): out[out_name][:] = np.where(right_mask, left[left_name].take(left_out), right[right_name].take(right_out)) else: # np.where does not work for mixin columns (e.g. Quantity) so # use a slower workaround. left_mask = ~right_mask if np.any(left_mask): out[out_name][left_mask] = left[left_name].take(left_out) if np.any(right_mask): out[out_name][right_mask] = right[right_name].take( right_out) continue elif left_name: # out_name came from the left table name, array, array_out, array_mask = left_name, left, left_out, left_mask elif right_name: name, array, array_out, array_mask = right_name, right, right_out, right_mask else: raise TableMergeError('Unexpected column names (maybe one is ""?)') # Select the correct elements from the original table col = array[name][array_out] # If the output column is masked then set the output column masking # accordingly. Check for columns that don't support a mask attribute. if masked and np.any(array_mask): # If col is a Column but not MaskedColumn then upgrade at this point # because masking is required. if isinstance(col, Column) and not isinstance(col, MaskedColumn): col = out.MaskedColumn(col, copy=False) # array_mask is 1-d corresponding to length of output column. We need # make it have the correct shape for broadcasting, i.e. (length, 1, 1, ..). # Mixin columns might not have ndim attribute so use len(col.shape). array_mask.shape = (col.shape[0], ) + (1, ) * (len(col.shape) - 1) # Now broadcast to the correct final shape array_mask = np.broadcast_to(array_mask, col.shape) try: col[array_mask] = col.info.mask_val except Exception: # Not clear how different classes will fail here raise NotImplementedError( "join requires masking column '{}' but column" " type {} does not support masking".format( out_name, col.__class__.__name__)) # Set the output table column to the new joined column out[out_name] = col # If col_name_map supplied as a dict input, then update. if isinstance(_col_name_map, Mapping): _col_name_map.update(col_name_map) return out
# submission = submission.sort_values(by=['Patient_Week'], ascending=True).reset_index(drop=True) # for i in range(len(submission['Confidence'].values)): # if submission['Confidence'][i] < df_final['Confidence'][i]: # df_final['Confidence'][i] = 0.8*df_final['Confidence'][i] + 0.2*submission['Confidence'][i] # print(df_final) test = pd.read_csv(PATH+'test.csv') for i in range(len(test.values)): week = test['Weeks'][i] patient = test['Patient'][i] FVC = test['FVC'][i] mask = df_final['Patient_Week'] == str(patient+'_'+str(week)) pos = np.flatnonzero(mask) FVC_pre = df_final['FVC'][int(pos)] # print(pos) # print(FVC) # print(FVC_pre) if (FVC_pre - FVC) > 100: mask = df_final['Patient_Week'] == str(patient+'_'+'131') pos = np.flatnonzero(mask) df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5) df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7)) print('131_week',df_final['FVC'][int(pos)]) mask = df_final['Patient_Week'] == str(patient+'_'+'132') pos = np.flatnonzero(mask) df_final['FVC'][int(pos)] -= pow((FVC_pre - FVC),0.5) df_final['Confidence'][int(pos)] -= abs(pow((FVC_pre - FVC),0.7))
def convertNansToOnes(pArray): nan_elements = np.flatnonzero(np.isnan(pArray)) if len(nan_elements) > 0: pArray[nan_elements] = 1.0 return pArray
0, samples_eig_vecs, weight_type='linear', norm='L2', int_type='CI', bounds='upper') signal_weighted_adjacency_matrix = weighted_adjacency_matrix[ reject_dict['signal_inds']][:, reject_dict['signal_inds']] print(dt.datetime.now().isoformat() + ' INFO: ' + 'Constructing final signal network without leaves...') biggest_signal_comp, biggest_signal_inds, biggest_signal_assing, biggest_signal_size = nnr.getBiggestComponent( signal_weighted_adjacency_matrix) signal_comp_inds = reject_dict['signal_inds'][biggest_signal_inds] strength_distn = biggest_signal_comp.sum(axis=0) leaf_inds = np.flatnonzero(strength_distn == 1) keep_inds = np.flatnonzero(strength_distn > 1) signal_final_inds = signal_comp_inds[keep_inds] signal_leaf_inds = signal_comp_inds[leaf_inds] final_weighted_adjacency_matrix = biggest_signal_comp[keep_inds][:, keep_inds] saveVariablesForClustering( data_file_name.split('.')[0], npy_dir, reject_dict, weighted_adjacency_matrix, exceeding_space_dims, signal_final_inds, expected_wcm, final_weighted_adjacency_matrix, loadNodeLabels(data_file_name, data)) print(dt.datetime.now().isoformat() + ' INFO: ' + 'Plotting modularity matrix eigenspectrum against null eigenspectrum...') plt.figure() nnr.plotModEigValsVsNullEigHist(network_modularity_matrix, samples_eig_vals)
def chanvese(I, init_mask, max_its=200, alpha=0.2, thresh=0, color='r', display=False): I = I.astype(np.float) # Create a signed distance map (SDF) from mask phi = mask2phi(init_mask) if display: plt.ion() fig, axes = plt.subplots(ncols=2) show_curve_and_phi(fig, I, phi, color) #plt.savefig('levelset_start.png', bbox_inches='tight') # Main loop its = 0 stop = False prev_mask = init_mask c = 0 while (its < max_its and not stop): # Get the curve's narrow band idx = np.flatnonzero(np.logical_and(phi <= 1.2, phi >= -1.2)) if len(idx) > 0: # Intermediate output if display: if np.mod(its, 50) == 0: print('iteration: {0}'.format(its)) show_curve_and_phi(fig, I, phi, color) else: if np.mod(its, 10) == 0: print('iteration: {0}'.format(its)) # Find interior and exterior mean upts = np.flatnonzero(phi <= 0) # interior points vpts = np.flatnonzero(phi > 0) # exterior points u = np.sum(I.flat[upts]) / (len(upts) + eps) # interior mean v = np.sum(I.flat[vpts]) / (len(vpts) + eps) # exterior mean # Force from image information F = (I.flat[idx] - u)**2 - (I.flat[idx] - v)**2 # Force from curvature penalty curvature = get_curvature(phi, idx) # Gradient descent to minimize energy dphidt = F / np.max(np.abs(F)) + alpha * curvature # Maintain the CFL condition dt = 0.45 / (np.max(np.abs(dphidt)) + eps) # Evolve the curve phi.flat[idx] += dt * dphidt # Keep SDF smooth phi = sussman(phi, 0.5) new_mask = phi <= 0 c = convergence(prev_mask, new_mask, thresh, c) if c <= 5: its = its + 1 prev_mask = new_mask else: stop = True else: break # Final output if display: show_curve_and_phi(fig, I, phi, color) #plt.savefig('levelset_end.png', bbox_inches='tight') # Make mask from SDF seg = phi <= 0 # Get mask from levelset return seg, phi, its
def get_topup_inputs_from(dwi_file, bval_file, b0_threshold, topup_prefix, bids_origin_files, epi_fmaps=None, max_per_spec=3, topup_requested=False): """Create a datain spec and a slspec from a concatenated dwi series. Create inputs for TOPUP that come from data in ``dwi/`` and epi fieldmaps in ``fmap/``. The ``nii_file`` input may be the result of concatenating a number of scans with different distortions present. The original source of each volume in ``nii_file`` is listed in ``bids_origin_files``. The strategy is to select ``max_per_spec`` b=0 images from each distortion group. Here, distortion group uses the FSL definition of a phase encoding direction and total readout time, as specified in the datain file used by TOPUP (i.e. "0 -1 0 0.087"). Case: Two opposing PE direction dwi series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For example if the following b=0 images are found at the following indices into ``nii_file``: ============ ============================= ================== Image Index BIDS source file for a b=0 Distortion Group ------------ ----------------------------- ------------------ 0 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 15 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 30 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 45 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 60 sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087`` 75 sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087`` 90 sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087`` 105 sub-1_dir-AP_run-2_dwi.nii.gz ``0 -1 0 0.087`` 120 sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087`` 135 sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087`` 150 sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087`` 165 sub-1_dir-PA_run-1_dwi.nii.gz ``0 1 0 0.087`` ============ ============================= ================== This will select images 0, 45 and 105 to represent the distortion group ``0 -1 0 0.087`` and images 120, 135 and 165 to represent ``0 1 0 0.087``. The ``--datain`` file would then contain:: 0 -1 0 0.087 0 -1 0 0.087 0 -1 0 0.087 0 1 0 0.087 0 1 0 0.087 0 1 0 0.087 Case: one DWI series and an EPI fieldmap ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If a reverse-phase encoding fieldmap image (or images) are passed in through ``epi_fmaps``, these will undergo the same selection process using ``max_per_spec``. The images will be added to the *end* of the image series, though, to ensure that the fieldmap correction will be aligned to the first b=0 image in ``nii_file``. For example if ``nii_file`` contains ============ ============================= ================== Image Index BIDS source file for a b=0 Distortion Group ------------ ----------------------------- ------------------ 0 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 15 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 30 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` 45 sub-1_dir-AP_run-1_dwi.nii.gz ``0 -1 0 0.087`` ============ ============================= ================== and the file from fmaps contains ============ ============================= ================== Image Index BIDS source file for a b=0 Distortion Group ------------ ----------------------------- ------------------ 0 sub-1_dir-PA_epi.nii.gz ``0 1 0 0.087`` 1 sub-1_dir-PA_epi.nii.gz ``0 1 0 0.087`` ============ ============================= ================== images 0, 15 and 45 would be selected to represent ``0 -1 0 0.087`` and images 0 and 1 would be selected to represent ``0 1 0 0.087``, resulting in a ``--datain`` file that contains:: 0 -1 0 0.087 0 -1 0 0.087 0 -1 0 0.087 0 1 0 0.087 0 1 0 0.087 Parameters: =========== nii_file : str A 4D DWI Series b0_indices: array-like indices into nii_file that can be used by topup topup_prefix: str file prefix for topup inputs bids_origin_files: list A list with the original bids file of each image in ``nii_file``. This is necessary because merging may have happened earlier in the pipeline epi_fmaps: A list of b=0 images from the fmaps/ directory. max_per_spec: int The maximum number of b=0 images to extract from a PE direction / image set """ # Start with the DWI file. Determine which images are b=0 bvals = np.loadtxt(bval_file) b0_indices = np.flatnonzero(bvals < b0_threshold) if not b0_indices.size: raise RuntimeError("No b=0 images available for TOPUP from the dwi.") dwi_nii = load_img(dwi_file) # Gather images from just the dwi series dwi_spec_lines, dwi_imain, dwi_report, _ = topup_inputs_from_4d_file( dwi_nii, b0_indices, bids_origin_files, image_source="combined DWI series", max_per_spec=max_per_spec) # If there are EPI fieldmaps, add them to the END of the topup spec if epi_fmaps and isdefined(epi_fmaps): topup_imain, topup_spec_lines, fmap_report = add_epi_fmaps_to_dwi_b0s( epi_fmaps, b0_threshold, max_per_spec, dwi_spec_lines, dwi_imain) topup_text = dwi_report + fmap_report else: topup_imain = dwi_imain topup_spec_lines = dwi_spec_lines topup_text = dwi_report imain_output = topup_prefix + "imain.nii.gz" imain_img = to_lps(topup_imain, new_axcodes=('L', 'A', 'S')) assert imain_img.shape[3] == len(topup_spec_lines) imain_img.to_filename(imain_output) # Write the datain text file and make sure it's usable if it's needed if len(set(topup_spec_lines)) < 2 and topup_requested: print(topup_spec_lines) raise Exception( "Unable to run TOPUP: not enough distortion groups. " "Check \"IntendedFor\" fields or consider using --ignore fieldmaps." ) datain_file = topup_prefix + "datain.txt" with open(datain_file, "w") as f: f.write("\n".join(topup_spec_lines)) return datain_file, imain_output, topup_text
def compute_reho(in_file, mask_file, cluster_size): """ Computes the ReHo Map, by computing tied ranks of the timepoints, followed by computing Kendall's coefficient concordance(KCC) of a timeseries with its neighbours Parameters ---------- in_file : nifti file 4D EPI File mask_file : nifti file Mask of the EPI File(Only Compute ReHo of voxels in the mask) cluster_size : integer for a brain voxel the number of neighbouring brain voxels to use for KCC. Returns ------- out_file : nifti file ReHo map of the input EPI image """ out_file = None res_fname = (in_file) res_mask_fname = (mask_file) CUTNUMBER = 10 if not (cluster_size == 27 or cluster_size == 19 or cluster_size == 7): cluster_size = 27 nvoxel = cluster_size res_img = nb.load(res_fname) res_mask_img = nb.load(res_mask_fname) res_data = res_img.get_data() res_mask_data = res_mask_img.get_data() print(res_data.shape) (n_x, n_y, n_z, n_t) = res_data.shape # "flatten" each volume of the timeseries into one big array instead of # x,y,z - produces (timepoints, N voxels) shaped data array res_data = np.reshape(res_data, (n_x * n_y * n_z, n_t), order='F').T # create a blank array of zeroes of size n_voxels, one for each time point Ranks_res_data = np.tile((np.zeros((1, (res_data.shape)[1]))), [(res_data.shape)[0], 1]) # divide the number of total voxels by the cutnumber (set to 10) # ex. end up with a number in the thousands if there are tens of thousands # of voxels segment_length = np.ceil(float((res_data.shape)[1]) / float(CUTNUMBER)) for icut in range(0, CUTNUMBER): segment = None # create a Numpy array of evenly spaced values from the segment # starting point up until the segment_length integer if not (icut == (CUTNUMBER - 1)): segment = np.array( np.arange(icut * segment_length, (icut + 1) * segment_length)) else: segment = np.array( np.arange(icut * segment_length, (res_data.shape[1]))) segment = np.int64(segment[np.newaxis]) # res_data_piece is a chunk of the original timeseries in_file, but # aligned with the current segment index spacing res_data_piece = res_data[:, segment[0]] nvoxels_piece = res_data_piece.shape[1] # run a merge sort across the time axis, re-ordering the flattened # volume voxel arrays res_data_sorted = np.sort(res_data_piece, 0, kind='mergesort') sort_index = np.argsort(res_data_piece, axis=0, kind='mergesort') # subtract each volume from each other db = np.diff(res_data_sorted, 1, 0) # convert any zero voxels into "True" flag db = db == 0 # return an n_voxel (n voxels within the current segment) sized array # of values, each value being the sum total of TRUE values in "db" sumdb = np.sum(db, 0) temp_array = np.array(np.arange(0, n_t)) temp_array = temp_array[:, np.newaxis] sorted_ranks = np.tile(temp_array, [1, nvoxels_piece]) if np.any(sumdb[:]): tie_adjust_index = np.flatnonzero(sumdb) for i in range(0, len(tie_adjust_index)): ranks = sorted_ranks[:, tie_adjust_index[i]] ties = db[:, tie_adjust_index[i]] tieloc = np.append(np.flatnonzero(ties), n_t + 2) maxties = len(tieloc) tiecount = 0 while (tiecount < maxties - 1): tiestart = tieloc[tiecount] ntied = 2 while (tieloc[tiecount + 1] == (tieloc[tiecount] + 1)): tiecount += 1 ntied += 1 ranks[tiestart:tiestart + ntied] = np.ceil( np.float32(np.sum(ranks[tiestart:tiestart + ntied])) / np.float32(ntied)) tiecount += 1 sorted_ranks[:, tie_adjust_index[i]] = ranks del db, sumdb sort_index_base = np.tile( np.multiply(np.arange(0, nvoxels_piece), n_t), [n_t, 1]) sort_index += sort_index_base del sort_index_base ranks_piece = np.zeros((n_t, nvoxels_piece)) ranks_piece = ranks_piece.flatten(order='F') sort_index = sort_index.flatten(order='F') sorted_ranks = sorted_ranks.flatten(order='F') ranks_piece[sort_index] = np.array(sorted_ranks) ranks_piece = np.reshape(ranks_piece, (n_t, nvoxels_piece), order='F') del sort_index, sorted_ranks Ranks_res_data[:, segment[0]] = ranks_piece sys.stdout.write('.') Ranks_res_data = np.reshape(Ranks_res_data, (n_t, n_x, n_y, n_z), order='F') K = np.zeros((n_x, n_y, n_z)) mask_cluster = np.ones((3, 3, 3)) if nvoxel == 19: mask_cluster[0, 0, 0] = 0 mask_cluster[0, 2, 0] = 0 mask_cluster[2, 0, 0] = 0 mask_cluster[2, 2, 0] = 0 mask_cluster[0, 0, 2] = 0 mask_cluster[0, 2, 2] = 0 mask_cluster[2, 0, 2] = 0 mask_cluster[2, 2, 2] = 0 elif nvoxel == 7: mask_cluster[0, 0, 0] = 0 mask_cluster[0, 1, 0] = 0 mask_cluster[0, 2, 0] = 0 mask_cluster[0, 0, 1] = 0 mask_cluster[0, 2, 1] = 0 mask_cluster[0, 0, 2] = 0 mask_cluster[0, 1, 2] = 0 mask_cluster[0, 2, 2] = 0 mask_cluster[1, 0, 0] = 0 mask_cluster[1, 2, 0] = 0 mask_cluster[1, 0, 2] = 0 mask_cluster[1, 2, 2] = 0 mask_cluster[2, 0, 0] = 0 mask_cluster[2, 1, 0] = 0 mask_cluster[2, 2, 0] = 0 mask_cluster[2, 0, 1] = 0 mask_cluster[2, 2, 1] = 0 mask_cluster[2, 0, 2] = 0 mask_cluster[2, 1, 2] = 0 mask_cluster[2, 2, 2] = 0 for i in range(1, n_x - 1): for j in range(1, n_y - 1): for k in range(1, n_z - 1): block = Ranks_res_data[:, i - 1:i + 2, j - 1:j + 2, k - 1:k + 2] mask_block = res_mask_data[i - 1:i + 2, j - 1:j + 2, k - 1:k + 2] if not (int(mask_block[1, 1, 1]) == 0): if nvoxel == 19 or nvoxel == 7: mask_block = np.multiply(mask_block, mask_cluster) R_block = np.reshape(block, (block.shape[0], 27), order='F') mask_R_block = R_block[:, np.argwhere( np.reshape(mask_block, (1, 27), order='F') > 0)[:, 1]] K[i, j, k] = f_kendall(mask_R_block) img = nb.Nifti1Image(K, header=res_img.get_header(), affine=res_img.get_affine()) reho_file = os.path.join(os.getcwd(), 'ReHo.nii.gz') img.to_filename(reho_file) out_file = reho_file return out_file
def hot_to_indices(bool_array): """ Only works for 1D-vector. """ index_array = np.flatnonzero(bool_array) return index_array
def __call__(self, atts): i = numpy.random.choice(numpy.flatnonzero(atts == atts.max())) dist = numpy.zeros(len(atts)) dist[i] = 1 return dist
def cut_prime_meridian(vertices): """Cut a polygon across the prime meridian, possibly splitting it into multiple polygons. Vertices consist of (longitude, latitude) pairs where longitude is always given in terms of a wrapped angle (between 0 and 2π). This routine is not meant to cover all possible cases; it will only work for convex polygons that extend over less than a hemisphere.""" # Ensure that the list of vertices does not contain a repeated endpoint. if (vertices[0] == vertices[-1]).all(): vertices = vertices[:-1] # Ensure that the longitudes are wrapped from 0 to 2π. vertices = np.column_stack((wrapped_angle(vertices[:, 0]), vertices[:, 1])) # Test if the segment consisting of points i-1 and i croses the meridian. # # If the two longitudes are in [0, 2π), then the shortest arc connecting # them crosses the meridian if the difference of the angles is greater # than π. phis = vertices[:, 0] phi0, phi1 = np.sort(np.row_stack((np.roll(phis, 1), phis)), axis=0) crosses_meridian = (phi1 - phi0 > np.pi) # Count the number of times that the polygon crosses the meridian. meridian_crossings = np.sum(crosses_meridian) if meridian_crossings == 0: # There were zero meridian crossings, so we can use the # original vertices as is. out_vertices = [vertices] elif meridian_crossings == 1: # There was one meridian crossing, so the polygon encloses the pole. # Any meridian-crossing edge has to be extended # into a curve following the nearest polar edge of the map. i, = np.flatnonzero(crosses_meridian) v0 = vertices[i - 1] v1 = vertices[i] # Find the latitude at which the meridian crossing occurs by # linear interpolation. delta_lon = abs(reference_angle(v1[0] - v0[0])) lat = (abs(reference_angle(v0[0])) / delta_lon * v0[1] + abs(reference_angle(v1[0])) / delta_lon * v1[1]) # FIXME: Use this simple heuristic to decide which pole to enclose. sign_lat = np.sign(np.sum(vertices[:, 1])) # Find the closer of the left or the right map boundary for # each vertex in the line segment. lon_0 = 0. if v0[0] < np.pi else 2 * np.pi lon_1 = 0. if v1[0] < np.pi else 2 * np.pi # Set the output vertices to the polar cap plus the original # vertices. out_vertices = [ np.vstack((vertices[:i], [[lon_0, lat], [lon_0, sign_lat * np.pi / 2], [lon_1, sign_lat * np.pi / 2], [lon_1, lat]], vertices[i:])) ] elif meridian_crossings == 2: # Since the polygon is assumed to be convex, if there is an even number # of meridian crossings, we know that the polygon does not enclose # either pole. Then we can use ordinary Euclidean polygon intersection # algorithms. out_vertices = [] # Construct polygon representing map boundaries. frame_poly = geometry.Polygon( np.asarray([[0., np.pi / 2], [0., -np.pi / 2], [2 * np.pi, -np.pi / 2], [2 * np.pi, np.pi / 2]])) # Intersect with polygon re-wrapped to lie in [-π, π) or [π, 3π). for shift in [0, 2 * np.pi]: poly = geometry.Polygon( np.column_stack( (reference_angle(vertices[:, 0]) + shift, vertices[:, 1]))) intersection = poly.intersection(frame_poly) if intersection: assert isinstance(intersection, geometry.Polygon) assert intersection.is_simple out_vertices += [np.asarray(intersection.exterior)] else: # There were more than two intersections. Not implemented! raise NotImplemented('The polygon intersected the map boundaries two ' 'or more times, so it is probably not simple and ' 'convex.') # Done! return out_vertices
# In[ ]: model_object = CNN_Model(data,len(data.all_encoded.keys())) # In[ ]: classifier = model_object.fit_generator(num_epochs=10, batch_size=16) # # MRR metric classifier = load_model("Code/Models/RCNN_ResNext.h5") test_data = ImageDataGenerator(data.x_test[:10000], data.y_test[:10000], 32, 16)predictions = classifier.predict_generator(test_data)species_embedding = data.embed_vectors2['species_glc_id']y_test = np.array(data.y_test)[:,-1,:]preds = predictions[:,-1,:] trues = [np.flatnonzero((species_embedding == i).all(1)) for i in y_test]from sklearn.metrics.pairwise import euclidean_distances def prediction_to_species(ypred, unique_species_id, max_ranks=5): e2 = pkl.load(open("Data/Embed2.pkl", "rb")) species_embedding = e2['species_glc_id'][-3336:] embedding_distance = euclidean_distances(species_embedding, ypred.reshape(-1,10)) top_indices = embedding_distance.flatten().argsort()[:max_ranks] species_glc_id = pd.Series(unique_species_id)[top_indices].reshape(-1,1) return species_glc_ids = 0 for i in range(len(trues)): if(i%100 == 1): print(i, s/i) try: ypred = prediction_to_species(preds[i],data.species, 100) r = ypred.index(trues[i]) except:
T = fits_table(fn) #T.about() events = fits_table(fn, ext=2) steps, Istep = np.unique(T.step, return_index=True) stepmap = dict([(s, i) for i, s in enumerate(steps)]) print('Steps:', len(steps)) unixtimes = T.unixtime[Istep] t0 = min(unixtimes) xaxis = unixtimes - t0 parent_pid = T._header['PPID'] T.mine = np.logical_or(T.pid == parent_pid, T.ppid == parent_pid) T.main = (T.pid == parent_pid) I = np.flatnonzero(T.mine) mypids = np.unique(T.pid[I]) print(len(I), 'measurements of my PIDs') print(len(mypids), 'PIDs') plt.clf() ps_icpu = np.zeros(len(steps), np.float32) plotted_worker = False for pid in mypids: II = I[T.pid[I] == pid] # Fill in the non-empty steps J = np.array([stepmap[s] for s in T.step[II]]) cmds = np.unique(T.command[II]) #print('PID', pid, ':', cmds)
def fit(self, X, y, max_features=None, overwrite_X=False, **params): """Fit the model using X, y as training data. Parameters ---------- X : array-like, shape = [n_samples, n_features] Training data. y : array-like, shape = [n_samples] Target values. Returns ------- self : object returns an instance of self. """ self._set_params(**params) X = np.atleast_2d(X) y = np.atleast_1d(y) X, y, Xmean, ymean = LinearModel._center_data(X, y, self.fit_intercept) alpha = getattr(self, 'alpha', 0.) if self.normalize: norms = np.sqrt(np.sum(X**2, axis=0)) nonzeros = np.flatnonzero(norms) if not overwrite_X: X = X.copy() overwrite_X = True X[:, nonzeros] /= norms[nonzeros] # precompute if n_samples > n_features precompute = self.precompute if hasattr(precompute, '__array__'): # copy as it's going to be modified Gram = precompute.copy() elif precompute == 'auto': Gram = 'auto' else: Gram = None self.alphas_, self.active_, self.coef_path_ = lars_path( X, y, Gram=Gram, overwrite_X=overwrite_X, overwrite_Gram=True, alpha_min=alpha, method=self.method, verbose=self.verbose, max_features=max_features, max_iter=self.max_iter) if self.normalize: self.coef_path_ /= norms[:, np.newaxis] self.coef_ = self.coef_path_[:, -1] self._set_intercept(Xmean, ymean) return self
def samp_entropy(a, m, r, tau=1, relative_r=True): """ Compute the sample entropy [RIC00]_ of a signal with embedding dimension `de` and delay `tau` [PYEEG]_. Vectorised version of the eponymous PyEEG function. In addition, this function can also be used to vary tau and therefore compute Multi-Scale Entropy(MSE) [COS05]_ by coarse grainning the time series (see example bellow). By default, r is expressed as relatively to the standard deviation of the signal. :param a: a one dimensional floating-point array representing a time series. :type a: :class:`~numpy.ndarray` or :class:`~pyrem.time_series.Signal` :param m: the scale :type m: int :param r: The tolerance :type r: float :param tau: The scale for coarse grainning. :type tau: int :param relative_r: whether the argument r is relative to the standard deviation. If false, an absolute value should be given for r. :type relative_r: true :return: the approximate entropy, a scalar :rtype: float Example: """ if len(a) <= 2: return np.nan coarse_a = _coarse_grainning(a, tau) if relative_r: coarse_a /= np.std(coarse_a) embsp = _embed_seq(coarse_a, 1, m + 1) embsp_last = embsp[:, -1] embs_mini = embsp[:, :-1] # Buffers are preallocated chunks of memory storing temporary results. # see the `out` argument in numpy *ufun* documentation dist_buffer = np.zeros(embsp.shape[0] - 1, dtype=np.float32) subtract_buffer = np.zeros((dist_buffer.size, m), dtype=np.float32) in_range_buffer = np.zeros_like(dist_buffer, dtype=np.bool) sum_cm, sum_cmp = 0.0, 0.0 # we iterate through all templates (rows), except last one. for i, template in enumerate(embs_mini[:-1]): # these are just views to the buffer arrays. to store intermediary matrices dist_b_view = dist_buffer[i:] sub_b_view = subtract_buffer[i:] range_b_view = in_range_buffer[i:] embsp_view = embsp_last[i+1:] # substract the template from each subsequent row of the embedded matrix np.subtract(embs_mini[i+1:], template, out=sub_b_view) # Absolute distance np.abs(sub_b_view, out=sub_b_view) # Maximal absolute difference between a scroll and a template is the distance np.max(sub_b_view, axis=1, out=dist_b_view) # we compare this distance to a tolerance r np.less_equal(dist_b_view, r, out=range_b_view) # score one for this template for each match in_range_sum = np.sum(range_b_view) sum_cm += in_range_sum # reuse the buffers for last column dist_b_view = dist_buffer[:in_range_sum] where = np.flatnonzero(range_b_view) dist_b_view = np.take(embsp_view, where, out=dist_b_view) range_b_view = in_range_buffer[range_b_view] # score one to TODO for each match of the last element dist_b_view -= embsp_last[i] np.abs(dist_b_view, out=dist_b_view) np.less_equal(dist_b_view, r, out=range_b_view) sum_cmp += np.sum(range_b_view) if sum_cm == 0 or sum_cmp ==0: return np.NaN return np.log(sum_cm/sum_cmp)
z_test = z_m_test print 'z_test_shape: ' ,z_test.shape print 's_test: ' ,s_test.shape X_test_dec = net.Dec.predict([z_test,s_test],batch_size=params['batch_size']) sdim = s_test.shape[1] n=10 full_inds = np.zeros((n,)) y_vals = np.unique(y_test) if len(y_vals)>n: y_vals = np.random.permutation(y_vals) for i in range(n): dig = y_vals[i%len(y_vals)] #print dig cur_inds = np.flatnonzero(y_test==dig) perm_inds = np.random.permutation(range(len(cur_inds))) full_inds[i] = cur_inds[perm_inds[i]] #print full_inds full_inds = full_inds.astype(np.int) z_rand = np.zeros((n*n,z_test.shape[1])) #z_rand = z_test[:n*n,:] s_rand = np.zeros((n*n,sdim)) for i in range(n): for jj in range(n): z_rand[i*n+jj,:] = z_test[full_inds[i],:] if sdim>1: s_rand[i*n+jj,:] = s_test[full_inds[jj],:] else: s_rand[i*n+jj] = s_test[full_inds[jj]]
def __init__(self, zscores, null_lb=-1, null_ub=1, estimate_mean=True, estimate_scale=True, estimate_null_proportion=False): # Extract the null z-scores ii = np.flatnonzero((zscores >= null_lb) & (zscores <= null_ub)) if len(ii) == 0: raise RunTimeError("No Z-scores fall between null_lb and null_ub") zscores0 = zscores[ii] # Number of Z-scores, and null Z-scores n_zs, n_zs0 = len(zscores), len(zscores0) # Unpack and transform the parameters to the natural scale, hold # parameters fixed as specified. def xform(params): mean = 0. sd = 1. prob = 1. ii = 0 if estimate_mean: mean = params[ii] ii += 1 if estimate_scale: sd = np.exp(params[ii]) ii += 1 if estimate_null_proportion: prob = 1 / (1 + np.exp(-params[ii])) return mean, sd, prob from scipy.stats.distributions import norm def fun(params): """ Negative log-likelihood of z-scores. The function has three arguments, packed into a vector: mean : location parameter logscale : log of the scale parameter logitprop : logit of the proportion of true nulls The implementation follows section 4 from Efron 2008. """ d, s, p = xform(params) # Mass within the central region central_mass = (norm.cdf((null_ub - d) / s) - norm.cdf( (null_lb - d) / s)) # Probability that a Z-score is null and is in the central region cp = p * central_mass # Binomial term rval = n_zs0 * np.log(cp) + (n_zs - n_zs0) * np.log(1 - cp) # Truncated Gaussian term for null Z-scores zv = (zscores0 - d) / s rval += np.sum(-zv**2 / 2) - n_zs0 * np.log(s) rval -= n_zs0 * np.log(central_mass) return -rval # Estimate the parameters from scipy.optimize import minimize # starting values are mean = 0, scale = 1, p0 ~ 1 mz = minimize(fun, np.r_[0., 0, 3], method="Nelder-Mead") mean, sd, prob = xform(mz['x']) self.mean = mean self.sd = sd self.null_proportion = prob
def true_choice_nb(col, from_i, to_i, a): """`choice_func_nb` that returns the indices of signals in `a`.""" return from_i + np.flatnonzero(a[from_i:to_i, col])
LFtous = numpy.zeros(Npoints) LFine = numpy.zeros(Npoints) LFelas = numpy.zeros(Npoints) #Define number of particles/bunch (or filling pattern array) Np = I0 * mA #Calculates the bunch length and saves data in the file BunchLength.txt #Also prints some statistics at the end Calc_Sigma(Vh, ph, Vm, pm, I0, Npoints, param) #Average pressure in the machine Pmed = 1 #[nTorr] #Check places where I0 differnte then and get the indexes not_zero = numpy.flatnonzero(I0) print '-----------------------------------------------' print 'Calculates IBS effects and Lifetime results' for j in range(len(not_zero)): #Redefines parameter Np param['Np'] = I0[not_zero[j]] * mA #Calculates IBS effects on the emittances (exi[j], eyi[j], spi[j], ssi[j]) = Iterate_emittances3HC(twiss, param, pm[not_zero[j]], Vm[not_zero[j]], ph[not_zero[j]], Vh[not_zero[j]]) #Uses the IBS results to calculates lifetimes (LFtous[j], LFine[j], LFelas[j]) = Calc_Lifetime(Pmed, param, I0[not_zero[j]], twiss, exi[j], eyi[j], spi[j], ssi[j])
ax.set_xlabel('Threshold [Filter output sdev]') ax.set_ylabel('Rate [cps]') for ifile, file in enumerate(files): with np.load(file) as arch: thr = arch['thr'] thrcounts = arch['thrcounts'] thr_theory = arch['thr_theory'] thrcounts_theory = arch['thrcounts_theory'] effnsamples = arch['effnsamples'] nevents = arch['nevents'] freq = arch['freq'] sdev = arch['sdev'] nz = np.flatnonzero(thrcounts) start = max(0, nz[0] - 1) end = min(len(thr), nz[-1] + 2) s = slice(start, end) ratefactor = freq / (nevents * effnsamples) cond = thr_theory >= np.min(thr) cond &= thr_theory <= np.max(thr) cond &= thrcounts_theory >= np.min(thrcounts[thrcounts > 0]) label = os.path.split(file)[1].replace('.npz', '') kw = dict(color=f'C{ifile}') ax.plot(thr_theory[cond] / sdev, ratefactor * thrcounts_theory[cond], '-', **kw) ax.plot(thr[s] / sdev,