def getgeodesicpts(m): """ computes the lat/lon values of the points on the surface of the sphere corresponding to a twenty-sided (icosahedral) geodesic. @param m: the number of points on the edge of a single geodesic triangle. There are 10*(m-1)**2+2 total geodesic points, including the poles. @return: C{B{lats, lons}} - rank 1 numpy float32 arrays containing the latitudes and longitudes of the geodesic points (in degrees). These points are nearly evenly distributed on the surface of the sphere. """ x,y,z = _spherepack.ihgeod(m) # convert cartesian coords to lat/lon. rad2dg = 180./math.pi r1 = x*x+y*y r = numpy.sqrt(r1+z*z) r1 = numpy.sqrt(r1) xtmp = numpy.where(numpy.logical_or(x,y),x,numpy.ones(x.shape,numpy.float32)) ztmp = numpy.where(numpy.logical_or(r1,z),z,numpy.ones(z.shape,numpy.float32)) lons = rad2dg*numpy.arctan2(y,xtmp)+180. lats = rad2dg*numpy.arctan2(r1,ztmp)-90. lat = numpy.zeros(10*(m-1)**2+2,numpy.float32) lon = numpy.zeros(10*(m-1)**2+2,numpy.float32) # first two points are poles. lat[0] = 90; lat[1] = -90. lon[0] = 0.; lon[1] = 0. lat[2:] = lats[0:2*(m-1),0:m-1,:].flatten() lon[2:] = lons[0:2*(m-1),0:m-1,:].flatten() return lat,lon
def __init__(self, strFilename, reconst=False, mask=None, pointfile=None, dSpacing=None, wavelength=None): """ @param: input image filename @param reconst: shall mased part or negative values be reconstructed (wipe out problems with pilatus gaps) """ self.strFilename = strFilename self.data = fabio.open(strFilename).data.astype("float32") if mask is not None: mask = mask.astype(bool) view = self.data.ravel() flat_mask = mask.ravel() min_valid = view[numpy.where(flat_mask == False)].min() view[numpy.where(flat_mask)] = min_valid self.shape = self.data.shape self.points = ControlPoints(pointfile, dSpacing=dSpacing, wavelength=wavelength) # self.lstPoints = [] self.fig = None self.fig2 = None self.fig2sp = None self.ax = None self.ct = None self.msp = None if reconstruct and (reconst is not False): if mask is None: mask = self.data < 0 self.massif = Massif(reconstruct(self.data, mask)) else: self.massif = Massif(self.data) self._sem = threading.Semaphore() self._semGui = threading.Semaphore() self.defaultNbPoints = 100
def vis_result(image, seg, gt, title1='Segmentation', title2='Ground truth', savefile=None): indices = np.where(seg >= 0.5) indices_gt = np.where(gt >= 0.5) im_norm = image / image.max() rgb_image = color.gray2rgb(im_norm) multiplier = [0., 1., 1.] multiplier_gt = [1., 1., 0.] im_seg = rgb_image.copy() im_gt = rgb_image.copy() im_seg[indices[0], indices[1], :] *= multiplier im_gt[indices_gt[0], indices_gt[1], :] *= multiplier_gt fig = plt.figure() a = fig.add_subplot(1, 2, 1) plt.imshow(im_seg) a.set_title(title1) a = fig.add_subplot(1, 2, 2) plt.imshow(im_gt) a.set_title(title2) if savefile is None: plt.show() else: plt.savefig(savefile) plt.close()
def test_float_modulus_exact(self): # test that float results are exact for small integers. This also # holds for the same integers scaled by powers of two. nlst = list(range(-127, 0)) plst = list(range(1, 128)) dividend = nlst + [0] + plst divisor = nlst + plst arg = list(itertools.product(dividend, divisor)) tgt = list(divmod(*t) for t in arg) a, b = np.array(arg, dtype=int).T # convert exact integer results from Python to float so that # signed zero can be used, it is checked. tgtdiv, tgtrem = np.array(tgt, dtype=float).T tgtdiv = np.where((tgtdiv == 0.0) & ((b < 0) ^ (a < 0)), -0.0, tgtdiv) tgtrem = np.where((tgtrem == 0.0) & (b < 0), -0.0, tgtrem) for dt in np.typecodes['Float']: msg = 'dtype: %s' % (dt,) fa = a.astype(dt) fb = b.astype(dt) # use list comprehension so a_ and b_ are scalars div = [self.floordiv(a_, b_) for a_, b_ in zip(fa, fb)] rem = [self.mod(a_, b_) for a_, b_ in zip(fa, fb)] assert_equal(div, tgtdiv, err_msg=msg) assert_equal(rem, tgtrem, err_msg=msg)
def plotResult(self, nn): cmask = np.where(self.y==1); plot(self.X[cmask,0], self.X[cmask,1], 'or', markersize=4) cmask = np.where(self.y==2); plot(self.X[cmask,0], self.X[cmask,1], 'ob', markersize=4) cmask = np.where(self.y==3); plot(self.X[cmask,0], self.X[cmask,1], 'og', markersize=4) minX = min(self.X[:,0]) minY = min(self.X[:,1]) maxX = max(self.X[:,0]) maxY = max(self.X[:,1]) grid_range = [minX, maxX, minY, maxY]; delta = 0.05; levels = 100 a = arange(grid_range[0],grid_range[1],delta) b = arange(grid_range[2],grid_range[3],delta) A, B = meshgrid(a, b) values = np.zeros(A.shape) for i in range(len(a)): for j in range(len(b)): values[j,i] = nn.getNetworkOutput( [ a[i], b[j] ] ) contour(A, B, values, levels=[1], colors=['k'], linestyles='dashed') contourf(A, B, values, levels=linspace(values.min(),values.max(),levels), cmap=cm.RdBu)
def do_cluster(raw_data, mode): cluster_labels=[] filter_zeros=np.where(raw_data[:, 1] != 0)[0] #% OPTIMIZE DATA QUALITY clear_data = raw_data[filter_zeros,:] # ignore the zeros on X filter_zeros=np.where(clear_data[:, 2] != 0)[0] clear_data = clear_data[filter_zeros,:] # ignore the zeros on Y rospy.loginfo('filtered data') #filter2=np.where(clear_data[:, 1] > 0.5)[0] #clear_data = clear_data[filter2, :] # ignore mishits #% TRAINING if mode == 0: cluster_labels =np.zeros((len(clear_data),1),int) eps = 0.5 min_points = 3 rospy.loginfo('call DBscan ') [core_samples,cluster_labels, n_clusters, human]=scikit_dbscan.dbscan(clear_data, eps, min_points,mode) #% TESTING if mode == 1: rospy.loginfo('call DBscan ') [core_samples,cluster_labels,n_clusters, human]=scikit_dbscan.dbscan(clear_data,eps, min_points) return core_samples,cluster_labels,human
def soft_threshold(lamda,b): th = float(lamda)/2.0 print ("(lamda,Threshold)",lamda,th) print("The type of b is ..., its len is ",type(b),b.shape,len(b[0])) if(lamda == 0): return b m,n = b.shape x = np.zeros((m,n)) k = np.where(b > th) # print("(b > th)",k) #print("Number of elements -->(b > th) ",type(k)) x[k] = b[k] - th k = np.where(np.absolute(b) <= th) # print("abs(b) <= th",k) # print("Number of elements -->abs(b) <= th ",len(k)) x[k] = 0 k = np.where(b < -th ) # print("(b < -th )",k) # print("Number of elements -->(b < -th ) <= th",len(k)) x[k] = b[k] + th x = x[:] return x
def factorize(self): """ factorize the filter into a minimal phase and maximal phase Returns ------- (Fmin,Fmax) : tuple of filter Fmin : minimal phase filter Fmax : maximal phase filter """ if self.fir: # selectionne les zeros interne et externe internal = np.where(abs(self.z) < 1)[0] external = np.where(abs(self.z) >= 1)[0] zi = self.z[internal] ze = self.z[external] Fmin = DF() Fmin.z = zi Fmax = DF() Fmax.z = ze return (Fmin, Fmax)
def compute_cost( X, y, theta, lam ): '''Compute cost for logistic regression.''' # Number of training examples m = y.shape[0] # Compute the prediction based on theta and X predictions = X.dot( theta ) # Preprocessing values before sending to sigmoid function. # If the argument to sigmoid function >= 0, we know that the # sigmoid value is 1. Similarly for the negative values. predictions[ where( predictions >= 20 ) ] = 20 predictions[ where( predictions <= -500 ) ] = -500 hypothesis = sigmoid( predictions ) hypothesis[ where( hypothesis == 1.0 ) ] = 0.99999 # Part of the cost function without regularization J1 = ( -1.0 / m ) * sum( ( y * np.log( hypothesis ) ) + ( ( 1.0 - y ) * np.log( 1.0 - hypothesis ) ) ) # Computing the regularization term J2 = lam / ( 2.0 * m ) * sum( theta[ 1:, ] * theta[ 1:, ] ) error = hypothesis - y return J1 + J2
def select_id(self, id, roi=True): """Convert a ROI id into an index to be used to index features safely. Parameters ---------- id : any hashable type, must be in self.get_id() The id of the region one wants to access. roi : bool If True (default), return the ROI index in the ROI list. If False, return the indices of the voxels of the ROI with the given id. That way, internal access to self.label can be made. Returns ------- index : int or np.array of shape (roi.size, ) Either the position of the ROI in the ROI list (if roi == True), or the positions of the voxels of the ROI with id `id` with respect to the self.label array. """ if id not in self.get_id(): raise ValueError("Unexisting `id` provided") if roi: index = int(np.where(self.get_id() == id)[0]) else: index = np.where(self.label == np.where(self.get_id() == id)[0])[0] return index
def infos() : '''Some information about the neuronal populations for each structures''' print "Striatal populations: %d" % len(STR) print "Pallidal populations: %d" % len(GP) print C = (W_STR_STR > 0).sum(axis=1) + (W_STR_STR < 0).sum(axis=1) L = W_STR_STR[np.where(W_STR_STR != 0)] print "Collateral striatal connections" print "Mean number: %g (+/- %g)" % (C.mean(), C.std()) print "Mean length: %g (+/- %g)" % (L.mean(), L.std()) print C = (W_GP_GP > 0).sum(axis=1) + (W_GP_GP < 0).sum(axis=1) L = W_GP_GP[np.where(W_GP_GP != 0)] print "Collateral pallidal connections" print "Mean number: %g (+/- %g)" % (C.mean(), C.std()) print "Mean length: %g (+/- %g)" % (L.mean(), L.std()) print C = (W_STR_GP > 0).sum(axis=1) + (W_STR_GP < 0).sum(axis=1) L = W_STR_GP[np.where(W_STR_GP != 0)] print "Striato-pallidal connections" print "Mean number: %g (+/- %g)" % (C.mean(), C.std()) print "Mean length: %g (+/- %g)" % (L.mean(), L.std()) print print "Mean # collateral striato-pallidal connections: %g (+/- %g)" % (C.mean(), C.std()) C = (W_GP_STR > 0).sum(axis=1) + (W_GP_STR < 0).sum(axis=1) L = W_GP_STR[np.where(W_GP_STR != 0)] print "Pallido-striatal connections" print "Mean number: %g (+/- %g)" % (C.mean(), C.std()) print "Mean length: %g (+/- %g)" % (L.mean(), L.std()) print
def prime_to_pixel(self, xprime, yprime, color=0): color0 = self._get_ricut() g0, g1, g2, g3 = self._get_drow() h0, h1, h2, h3 = self._get_dcol() px, py, qx, qy = self._get_cscc() # #$(%*&^(%$%*& bad documentation. (px,py) = (py,px) (qx,qy) = (qy,qx) qx = qx * np.ones_like(xprime) qy = qy * np.ones_like(yprime) xprime -= np.where(color < color0, px * color, qx) yprime -= np.where(color < color0, py * color, qy) # Now invert: # yprime = y + g0 + g1 * x + g2 * x**2 + g3 * x**3 # xprime = x + h0 + h1 * x + h2 * x**2 + h3 * x**3 x = xprime - h0 # dumb-ass Newton's method dx = 1. # FIXME -- should just update the ones that aren't zero # FIXME -- should put in some failsafe... while np.max(np.abs(np.atleast_1d(dx))) > 1e-10: xp = x + h0 + h1 * x + h2 * x**2 + h3 * x**3 dxpdx = 1 + h1 + h2 * 2*x + h3 * 3*x**2 dx = (xprime - xp) / dxpdx x += dx y = yprime - (g0 + g1 * x + g2 * x**2 + g3 * x**3) return (x, y)
def processTrafficData(self): for index, row in self.traffic_data.iterrows(): adjacent_list = [] if index > 1 and index < 18467: if self.traffic_data.ix[index - 1]['inter1'] == row[0]: key1 = self.traffic_data.ix[index - 1]['inter1'] + ', ' + self.traffic_data.ix[index - 1]['inter2'] adjacent_list.append(key1) if self.traffic_data.ix[index + 1]['inter1'] == row[0]: key2 = self.traffic_data.ix[index + 1]['inter1'] + ', ' + self.traffic_data.ix[index + 1]['inter2'] adjacent_list.append(key2) keylist = np.where(self.traffic_data['inter1'] == row[1])[0] keylist1 = np.where(self.traffic_data['inter2'] == row[0])[0] ind_list = np.intersect1d(keylist, keylist1) if len(ind_list) >= 1: ind = ind_list[0] if ind > 1 and ind < 18467: if self.traffic_data.ix[ind - 1]['inter1'] == row[1]: key3 = self.traffic_data.ix[ind - 1]['inter1'] + ', ' + self.traffic_data.ix[ind - 1]['inter2'] adjacent_list.append(key3) if self.traffic_data.ix[ind + 1]['inter1'] == row[1]: key4 = self.traffic_data.ix[ind + 1]['inter1'] + ', ' + self.traffic_data.ix[ind + 1]['inter2'] adjacent_list.append(key4) node = row['node'] node.setAdjacents(adjacent_list)
def stats(t, snp=None): '''Return a record array with imputation statistics.''' T = t.sample_index_to_impute imputed = t.imputed_data[:, T, :] tot_to_impute = 2 * imputed.shape[1] snp = snp if snp is not None else np.arange(t.num_snps) stats = np.zeros((len(snp),), dtype=[ ('dist_cm', 'f4'), # Genetic distance from beginning of chromosome ('count', '(2,)i4'), # Allele count ('frequency', '(2,)f4'), # Allele frequency ('call_rate', 'f4'), # Imputation Call rate ('call_rate_training', 'f4') # Imputation Call rate ]) call_rate_training = 1.0 * np.sum(np.sum(t.imputed_data[:, t.sample_index, :] != 0, axis=2), axis=1)# / (2 * len(t.sample_index)) for row, snp_index in enumerate(snp): # TODO: replace by a bulk group-by/hist? # g = t.__t.training_data[snp_index, :, :] i = imputed[snp_index, :] (c1, c2) = (len(np.where(i == 1)[0]), len(np.where(i == 2)[0])) c = c1 + c2 + SMALL_FLOAT f1, f2 = (1.0 * c1) / c, (1.0 * c2) / c call_rate = 1.0 * len(i.nonzero()[0]) / tot_to_impute # print 'c1 %4d c2 %4d f1 %.2f f2 %.2f call rate %5.2f' % (c1, c2, f1, f2, call_rate) stats[row] = (t.snp['dist_cm'][snp_index], [c1, c2], [f1, f2], call_rate, call_rate_training[snp_index]) return stats
def pixel_to_prime(self, x, y, color=0): # Secret decoder ring: # http://www.sdss.org/dr7/products/general/astrometry.html # (color)0 is called riCut; # g0, g1, g2, and g3 are called # dRow0, dRow1, dRow2, and dRow3, respectively; # h0, h1, h2, and h3 are called # dCol0, dCol1, dCol2, and dCol3, respectively; # px and py are called csRow and csCol, respectively; # and qx and qy are called ccRow and ccCol, respectively. color0 = self._get_ricut() g0, g1, g2, g3 = self._get_drow() h0, h1, h2, h3 = self._get_dcol() px, py, qx, qy = self._get_cscc() # #$(%*&^(%$%*& bad documentation. (px,py) = (py,px) (qx,qy) = (qy,qx) yprime = y + g0 + g1 * x + g2 * x**2 + g3 * x**3 xprime = x + h0 + h1 * x + h2 * x**2 + h3 * x**3 # The code below implements this, vectorized: # if color < color0: # xprime += px * color # yprime += py * color # else: # xprime += qx # yprime += qy qx = qx * np.ones_like(x) qy = qy * np.ones_like(y) xprime += np.where(color < color0, px * color, qx) yprime += np.where(color < color0, py * color, qy) return (xprime, yprime)
def prep_data(dataset): df = dataset.copy() latlon = list(zip(df.lat, df.lon)) dist = np.array([distance(latlon[i + 1], latlon[i]) for i in range(len((latlon[:-1])))]) df["dist"] = np.concatenate(([0], np.cumsum(dist))) slope = np.abs(100 * np.diff(df.alt) / (1000 * dist)) slope[np.where( slope < 4) ] = 0 # "green" slope[np.where((slope >= 4) & (slope < 6))] = 1 # "yellow" slope[np.where((slope >= 6) & (slope < 10))] = 2 # "pink" slope[np.where((slope >= 10) & (slope < 15))] = 3 # "orange" slope[np.where( slope >= 15 )] = 4 # "red" slope = im.median_filter(slope, 6) colors = np.empty_like(slope, dtype=object) colors[np.where(slope == 0)] = "green" colors[np.where(slope == 1)] = "yellow" colors[np.where(slope == 2)] = "pink" colors[np.where(slope == 3)] = "orange" colors[np.where(slope == 4)] = "red" df["colors"] = list(colors) + [None] # NOTE: add [None] just make pandas happy return df
def myTradingSystem(DATE, CLOSE, settings): ''' This system uses mean reversion techniques to allocate capital into the desired equities ''' # This strategy evaluates two averages over time of the close over a long/short # scale and builds the ratio. For each day, "smaQuot" is an array of "nMarkets" # size. nMarkets = numpy.shape(CLOSE)[1] periodLong = 200 periodShort = 40 smaLong = numpy.sum(CLOSE[-periodLong:, :], axis=0)/periodLong smaRecent = numpy.sum(CLOSE[-periodShort:, :], axis=0)/periodShort smaQuot = smaRecent / smaLong # For each day, scan the ratio of moving averages over the markets and find the # market with the maximum ratio and the market with the minimum ratio: longEquity = numpy.where(smaQuot == numpy.nanmin(smaQuot)) shortEquity = numpy.where(smaQuot == numpy.nanmax(smaQuot)) # Take a contrarian view, going long the market with the minimum ratio and # going short the market with the maximum ratio. The array "pos" will contain # all zero entries except for those cases where we go long (1) and short (-1): pos = numpy.zeros((1, nMarkets)) pos[0, longEquity[0][0]] = 1 pos[0, shortEquity[0][0]] = -1 # For the position sizing, we supply a vector of weights defining our # exposure to the markets in settings['markets']. This vector should be # normalized. pos = pos/numpy.nansum(abs(pos)) return pos, settings
def getFracMelt(self, time=None, scale=1.0): """Get fractional area where basal melting occurs. time: if None, return data for all time slices if list/etc of size two, interpret as array selection if single value, get only this time slice""" (tarray, t) = CFchecklist(time, self.file.variables["time"]) values = [] fact = self.deltax * self.deltay * scale if tarray: for i in range(t[0], t[1] + 1): ih = self.getIceArea(time=i, scale=scale) if ih > 0: mlt = numpy.where(self.file.variables["bmlt"][i, :, :] > 0.0, 1, 0).flat values.append(sum(mlt) * fact / ih) else: values.append(0.0) return values ih = self.getIceArea(time=t, scale=scale) if ih > 0: mlt = numpy.where(self.file.variables["bmlt"][t, :, :] > 0.0, 1, 0).flat return sum(mlt) * fact / ih else: return 0.0
def get_absline(self, inp): """ Returns an AbsLine from the AbsSystem Parameters ---------- inp : str or Quantity str -- Name of the transition, e.g. 'CII 1334' Quantity -- Rest wavelength of the transition, e.g. 1334.53*u.AA to 0.01 precision Returns ------- absline -- AbsLine object or list of Abslines More than one will be returned if this line exists in multiple components. The returned quantity will then be a list instead of a single object """ # Generate the lines abslines = self.list_of_abslines() if isinstance(inp,basestring): names = np.array([absline.name for absline in abslines]) mt = np.where(names == inp)[0] elif isinstance(inp,Quantity): wrest = Quantity([absline.wrest for absline in abslines]) mt = np.where(np.abs(wrest-inp) < 0.01*u.AA)[0] else: raise IOError("Bad input to absline") # Finish if len(mt) == 0: warnings.warn("No absline with input={}".format(inp)) return None elif len(mt) == 1: return abslines[mt] else: return [abslines[ii] for ii in mt]
def multi_where(vec1, vec2): '''Given two vectors, multi_where returns a tuple of indices where those two vectors overlap. ****THIS FUNCTION HAS NOT BEEN TESTED ON N-DIMENSIONAL ARRAYS******* Inputs: 2 numpy vectors Output: (xy, yx) where xy is a numpy vector containing the indices of the elements in vector 1 that are also in vector 2. yx is a vector containing the indices of the elements in vector 2 that are also in vector 1. Example: >> x = np.array([1,2,3,4,5]) >> y = np.array([3,4,5,6,7]) >> (xy,yx) = multi_where(x,y) >> xy array([2,3,4]) >> yx array([0,1,2]) ''' OneInTwo = np.array([]) TwoInOne = np.array([]) for i in range(vec1.shape[0]): if np.where(vec2 == vec1[i])[0].shape[0]: OneInTwo = np.append(OneInTwo,i) TwoInOne = np.append(TwoInOne, np.where(vec2 == vec1[i])[0][0]) return (np.int8(OneInTwo), np.int8(TwoInOne))
def hist_mask(data, threshold=.95, keep='lower'): """ Returns boolean mask of values below a frequency percentage threshold (0-1). Args: -data (1D array) -threshold (float): 0-1 -keep (str): lower, greater, middle. If middle, threshold is ignored, and a single cluster is searched out. """ bins = len(data)/100 if keep.lower() == 'middle' else len(data) / 2 freq, val = np.histogram(data, bins=bins) freq = freq / np.sum(freq).astype(float) # Normalize frequency data if keep.lower() in ('lower', 'upper'): cutoff_value = val[np.where(np.diff(np.cumsum(freq) < threshold))[0] + 1] cutoff_value = val[1] if len(cutoff_value)==0 else cutoff_value if keep.lower() == 'lower': return data < cutoff_value else: return data > cutoff_value else: histmask = np.ones(data.shape[0], dtype=bool) # Initializing mask with all True values # Slowly increment the parameter until a strong single central cluster is found for param in np.arange(0.0005, .02, .0003): cutoff_values = val[np.where(np.diff(freq < param))[0]] if len(cutoff_values) == 2: histmask &= data > cutoff_values[0] histmask &= data < cutoff_values[1] return histmask else: return data > -100000. # Return an all-true mask print("Warning: Histogram filter not finding a good parameter to form a central cluster. Please try again.")
def implicit_black_box(propensities, V, X, w, h, deter_vector, stoc_positions, positions, valid, deriv): # Adjustment for systems reaching steady state temp = derivative_G(propensities, V, X, w, deter_vector, stoc_positions, positions, valid) # pdb.set_trace() valid_adjust_pos = np.where(np.sum(np.abs(temp), axis=0) < 1e-10, True, False) valid_adjust = valid[:, :] valid_adjust[valid_adjust_pos, :] = False # print(" Reached Steady State %d"%(np.sum(valid_adjust_pos))) from scipy.integrate import ode # pdb.set_trace() deter_ode = ode(f).set_integrator("lsoda", method="adams", with_jacobian=False) deter_ode.set_initial_value(X[deter_vector, :].flatten(), 0).set_f_params( [propensities, V, X, deter_vector, stoc_positions, positions, valid_adjust, w] ) # pdb.set_trace() while deter_ode.successful() and deter_ode.t < h: deter_ode.integrate(h) # print("Black Box: \n"+ str(deter_ode.y)) # print("iterator : \n:"+str(next_X[deter_vector,:])) X[deter_vector, :] = deter_ode.y.reshape((np.sum(deter_vector), X.shape[1])) # Another adjust to compensate for non negative X = np.where(X < 0.0, 0.0, X) return X
def word2ind(word, vocab, utok_ind=None): ind = np.where(vocab == unicode(word)) if len(ind[0]) == 0: if not utok_ind: utok_ind = np.where(vocab == UTOK) ind = utok_ind return ind[0][0]
def _lininterp(self,x,X,Y): if hasattr(x,'__len__'): xtype = 'array' xx=np.asarray(x).astype(np.float) else: xtype = 'scalar' xx=np.asarray([x]).astype(np.float) idx = X.searchsorted(xx) yy = xx*0 yy[idx>len(X)-1] = Y[-1] # over yy[idx<=0] = Y[0] # under wok = np.where((idx>0) & (idx<len(X))) # the good ones iok=idx[wok] yywok = Y[iok-1] + ( (Y[iok]-Y[iok-1])/(X[iok]-X[iok-1]) * (xx[wok]-X[iok-1]) ) w = np.where( ((X[iok]-X[iok-1]) == 0) ) # where are the nan ? yywok[w] = Y[iok[w]-1] # replace by previous value wl = np.where(xx[wok] == X[0]) yywok[wl] = Y[0] wh = np.where(xx[wok] == X[-1]) yywok[wh] = Y[-1] yy[wok] = yywok if xtype == 'scalar': yy = yy[0] return yy
def corrtag_image(in_data,xtype='XCORR',ytype='YCORR',pha=(2,30),bins=(1024,16384),times=None,ranges=((0,1023),(0,16384)),binning=(1,1),NUV=False): try: histogram2d except NameError: from numpy import histogram2d,where,zeros try: getdata except NameError: from pyfits import getdata try: events=getdata(in_data,1) except: events=in_data xlength = (ranges[1][1]+1) ylength = (ranges[0][1]+1) xbinning = binning[1] ybinning = binning[0] if NUV: bins = (1024,1024) pha = (-1,1) ranges = ( (0,1023), (0,1023) ) if times != None: index = where( (events['TIME']>=times[0]) & (events['TIME'] <= times[1]) ) events= events[index] index = where((events['PHA']>=pha[0])&(events['PHA']<=pha[1])) if len(index[0]): image,y_r,x_r = histogram2d(events[ytype][index],events[xtype][index],bins=bins,range=ranges) else: image = zeros( (bins[0]//binning[0],bins[1]//binning[1]) ) return image
def mutual_information(self, x_index, y_index, log_base, debug=False): """ Calculate and return Mutual information between two random variables """ # Check if index are into the bounds assert (0 <= x_index <= self.n_rows) assert (0 <= y_index <= self.n_rows) # Variable to return MI summation = 0.0 # Get uniques values of random variables values_x = set(self.data[x_index]) values_y = set(self.data[y_index]) # Print debug info if debug: print 'MI between' print self.data[x_index] print self.data[y_index] # For each random for value_x in values_x: for value_y in values_y: px = shape(where(self.data[x_index] == value_x))[1] / self.n_cols py = shape(where(self.data[y_index] == value_y))[1] / self.n_cols pxy = len(where(in1d(where(self.data[x_index] == value_x)[0], where(self.data[y_index] == value_y)[0]) == True)[0]) / self.n_cols if pxy > 0.0: summation += pxy * math.log((pxy / (px * py)), log_base) if debug: print '(%d,%d) px:%f py:%f pxy:%f' % (value_x, value_y, px, py, pxy) return summation
def __init__(self,turn,elem,single,name,s,x,xp,y,yp,pc,de,tau,**args): apc=float(pc[0])*1e9 ade=float(de[0]) self.m0=self.pmass en=np.sqrt(apc**2+self.pmass**2) self.e0=en-ade self.p0c=np.sqrt(self.e0**2-self.m0**2) # structure self.elem=np.array(elem,dtype=int) self.turn=np.array(turn,dtype=int) d0=np.where(np.diff(self.elem)!=0)[0][0]+1 d1=(np.where(np.diff(self.turn)!=0)[0][0]+1)/d0 d2=len(self.turn)/d1/d0 self.single=np.array(single,dtype=int) self.name=np.array(name,dtype=str) self.s =np.array(s ,dtype=float) self.x =np.array(x ,dtype=float) self.y =np.array(y ,dtype=float) self.tau=-np.array(tau,dtype=float)*self.clight opd=np.array(pc,dtype=float)*(1e9/self.p0c) self.delta=opd-1 self.pt=np.array(de,dtype=float)/self.p0c self.px=np.array(xp,dtype=float)*opd self.py=np.array(yp,dtype=float)*opd for nn,vv in self.__dict__.items(): if hasattr(vv,'__len__') and len(vv)==d0*d1*d2: setattr(self,nn,vv.reshape(d2,d1,d0))
def test_background_model(tmpdir): data_store = DataStore.from_dir('$GAMMAPY_EXTRA/datasets/hess-crab4-hd-hap-prod2/') bgmaker = OffDataBackgroundMaker(data_store, outdir=str(tmpdir)) bgmaker.select_observations(selection='all') table = Table.read('run.lis', format='ascii.csv') assert table['OBS_ID'][1] == 23526 bgmaker.group_observations() table = ObservationTable.read(str(tmpdir / 'obs.fits')) assert list(table['GROUP_ID']) == [0, 0, 0, 1] table = ObservationTable.read(str(tmpdir / 'group-def.fits')) assert list(table['ZEN_PNT_MAX']) == [49, 90] # TODO: Fix 3D code # bgmaker.make_model("3D") # bgmaker.save_models("3D") # model = CubeBackgroundModel.read(str(tmpdir / 'background_3D_group_001_table.fits.gz')) # assert model.counts_cube.data.sum() == 1527 bgmaker.make_model("2D") bgmaker.save_models("2D") model = EnergyOffsetBackgroundModel.read(str(tmpdir / 'background_2D_group_001_table.fits.gz')) assert model.counts.data.value.sum() == 1398 index_table_new = bgmaker.make_total_index_table(data_store, "2D", None, None) table_bkg = index_table_new[np.where(index_table_new["HDU_NAME"] == "bkg_2d")] name_bkg_run023523 = table_bkg[np.where(table_bkg["OBS_ID"] == 23523)]["FILE_NAME"] assert str(tmpdir) + "/" + name_bkg_run023523[0] == str(tmpdir) + '/background_2D_group_001_table.fits.gz' name_bkg_run023526 = table_bkg[np.where(table_bkg["OBS_ID"] == 23526)]["FILE_NAME"] assert str(tmpdir) + "/" + name_bkg_run023526[0] == str(tmpdir) + '/background_2D_group_000_table.fits.gz'
def entropy(self, x_index, y_index, log_base, debug=False): """ Calculate the entropy between two random variable """ assert (0 <= x_index <= self.n_rows) assert (0 <= y_index <= self.n_rows) # Variable to return MI summation = 0.0 # Get uniques values of random variables values_x = set(self.data[x_index]) values_y = set(self.data[y_index]) # Print debug info if debug: print 'Entropy between' print self.data[x_index] print self.data[y_index] # For each random for value_x in values_x: for value_y in values_y: pxy = len(where(in1d(where(self.data[x_index] == value_x)[0], where(self.data[y_index] == value_y)[0]) == True)[0]) / self.n_cols if pxy > 0.0: summation += pxy * math.log(pxy, log_base) if debug: print '(%d,%d) pxy:%f' % (value_x, value_y, pxy) if summation == 0.0: return summation else: return - summation
def CalcErange(inWS,ns,erange,binWidth): #length of array in Fortran array_len = 4096 binWidth = int(binWidth) bnorm = 1.0/binWidth #get data from input workspace _,X,Y,E = GetXYE(inWS,ns,array_len) Xdata = mtd[inWS].readX(0) #get all x values within the energy range rangeMask = (Xdata >= erange[0]) & (Xdata <= erange[1]) Xin = Xdata[rangeMask] #get indicies of the bounds of our energy range minIndex = np.where(Xdata==Xin[0])[0][0]+1 maxIndex = np.where(Xdata==Xin[-1])[0][0] #reshape array into sublists of bins Xin = Xin.reshape(len(Xin)/binWidth, binWidth) #sum and normalise values in bins Xout = [sum(bin_val) * bnorm for bin_val in Xin] #count number of bins nbins = len(Xout) nout = [nbins, minIndex, maxIndex] #pad array for use in Fortran code Xout = PadArray(Xout,array_len) return nout,bnorm,Xout,X,Y,E
def load_hdf5(self, restartFolder, timestep, tXY): """ This function reads and allocates **badlands** parameters & variables from a previous simulation. These parameters are obtained from a specific time HDF5 file. Important: This function is only called when a **restart simulation** is ran... |:bomb:| Args: restartFolder: restart folder name as defined in tge XML input file. timestep: time step to load defined based on output interval number. tXY: 2D numpy array TIN grid local coordinates. Returns ------- elev numpy 1D array containing the updated elevation from the restart model. cum numpy 1D array containing the updated erosion/deposition values from the restart model. """ if os.path.exists(restartFolder): folder = restartFolder + "/h5/" fileCPU = "tin.time%s.hdf5" % timestep restartncpus = len(glob.glob1(folder, fileCPU)) if restartncpus == 0: raise ValueError( "The requested time step for the restart simulation cannot be found in the restart folder." ) else: raise ValueError( "The restart folder is missing or the given path is incorrect." ) df = h5py.File("%s/h5/tin.time%s.hdf5" % (restartFolder, timestep), "r") coords = numpy.array((df["/coords"])) cumdiff = numpy.array((df["/cumdiff"])) cumhill = numpy.array((df["/cumhill"])) cumfail = numpy.array((df["/cumfail"])) x, y, z = numpy.hsplit(coords, 3) c = cumdiff h = cumhill f = cumfail XY = numpy.column_stack((x, y)) tree = cKDTree(XY) distances, indices = tree.query(tXY, k=3) if len(z[indices].shape) == 3: z_vals = z[indices][:, :, 0] c_vals = c[indices][:, :, 0] h_vals = h[indices][:, :, 0] f_vals = f[indices][:, :, 0] else: z_vals = z[indices] c_vals = c[indices] h_vals = h[indices] f_vals = f[indices] with numpy.errstate(divide="ignore"): elev = numpy.average(z_vals, weights=(1.0 / distances), axis=1) cum = numpy.average(c_vals, weights=(1.0 / distances), axis=1) hcum = numpy.average(h_vals, weights=(1.0 / distances), axis=1) fcum = numpy.average(f_vals, weights=(1.0 / distances), axis=1) onIDs = numpy.where(distances[:, 0] == 0)[0] if len(onIDs) > 0: if len(z[indices].shape) == 3: elev[onIDs] = z[indices[onIDs, 0], 0] cum[onIDs] = c[indices[onIDs, 0], 0] hcum[onIDs] = h[indices[onIDs, 0], 0] fcum[onIDs] = f[indices[onIDs, 0], 0] else: elev[onIDs] = z[indices[onIDs, 0]] cum[onIDs] = c[indices[onIDs, 0]] hcum[onIDs] = h[indices[onIDs, 0]] fcum[onIDs] = f[indices[onIDs, 0]] return elev, cum, hcum, fcum
def sample_homography(shape, perspective=True, scaling=True, rotation=True, translation=True, n_scales=5, n_angles=25, scaling_amplitude=0.1, perspective_amplitude_x=0.1, perspective_amplitude_y=0.1, patch_ratio=0.5, max_angle=np.pi / 2, allow_artifacts=False, translation_overflow=0.): """Sample a random valid homography. Computes the homography transformation between a random patch in the original image and a warped projection with the same image size. As in `tf.contrib.image.transform`, it maps the output point (warped patch) to a transformed input point (original patch). The original patch, which is initialized with a simple half-size centered crop, is iteratively projected, scaled, rotated and translated. Arguments: shape: A rank-2 `Tensor` specifying the height and width of the original image. perspective: A boolean that enables the perspective and affine transformations. scaling: A boolean that enables the random scaling of the patch. rotation: A boolean that enables the random rotation of the patch. translation: A boolean that enables the random translation of the patch. n_scales: The number of tentative scales that are sampled when scaling. n_angles: The number of tentatives angles that are sampled when rotating. scaling_amplitude: Controls the amount of scale. perspective_amplitude_x: Controls the perspective effect in x direction. perspective_amplitude_y: Controls the perspective effect in y direction. patch_ratio: Controls the size of the patches used to create the homography. max_angle: Maximum angle used in rotations. allow_artifacts: A boolean that enables artifacts when applying the homography. translation_overflow: Amount of border artifacts caused by translation. Returns: A `Tensor` of shape `[1, 8]` corresponding to the flattened homography transform. """ # Corners of the output image pts1 = np.stack([[0., 0.], [0., 1.], [1., 1.], [1., 0.]], axis=0) # Corners of the input patch margin = (1 - patch_ratio) / 2 pts2 = margin + np.array([[0, 0], [0, patch_ratio], [patch_ratio, patch_ratio], [patch_ratio, 0]], dtype=np.float32) # Random perspective and affine perturbations if perspective: if not allow_artifacts: perspective_amplitude_x = min(perspective_amplitude_x, margin) perspective_amplitude_y = min(perspective_amplitude_y, margin) perspective_displacement = truncated_normal( [1], 0., perspective_amplitude_y / 2) h_displacement_left = truncated_normal([1], 0., perspective_amplitude_x / 2) h_displacement_right = truncated_normal([1], 0., perspective_amplitude_x / 2) pts2 += np.stack([ np.concatenate([h_displacement_left, perspective_displacement], 0), np.concatenate([h_displacement_left, -perspective_displacement], 0), np.concatenate([h_displacement_right, perspective_displacement], 0), np.concatenate([h_displacement_right, -perspective_displacement], 0) ]) # Random scaling # sample several scales, check collision with borders, randomly pick a valid one if scaling: scales = np.concatenate( [[1.], truncated_normal([n_scales], 1, scaling_amplitude / 2)], 0) center = np.mean(pts2, axis=0, keepdims=True) scaled = np.expand_dims(pts2 - center, axis=0) * np.expand_dims( np.expand_dims(scales, 1), 1) + center if allow_artifacts: valid = np.arange(n_scales) # all scales are valid except scale=1 else: a = np.all((scaled >= 0.) & (scaled < 1.), (1, 2)) valid = np.where(np.all((scaled >= 0.) & (scaled < 1.), (1, 2)))[:][0] idx = valid[np.random.random_integers(0, high=np.shape(valid)[0])] pts2 = scaled[idx] # Random translation if translation: t_min, t_max = np.min(pts2, axis=0), np.min(1 - pts2, axis=0) if allow_artifacts: t_min += translation_overflow t_max += translation_overflow pts2 += np.expand_dims(np.stack([ np.random.uniform(-t_min[0], t_max[0]), np.random.uniform(-t_min[1], t_max[1]) ]), axis=0) # Random rotation # sample several rotations, check collision with borders, randomly pick a valid one if rotation: angles = np.linspace(-max_angle, max_angle, n_angles) angles = np.concatenate([[0.], angles], axis=0) # in case no rotation is valid center = np.mean(pts2, axis=0, keepdims=True) rot_mat = np.reshape( np.stack([ np.cos(angles), -np.sin(angles), np.sin(angles), np.cos(angles) ], axis=1), [-1, 2, 2]) rotated = np.matmul( np.tile(np.expand_dims(pts2 - center, axis=0), [n_angles + 1, 1, 1]), rot_mat) + center if allow_artifacts: valid = np.arange(n_angles) # all angles are valid, except angle=0 else: valid = np.where( np.all((rotated >= 0.) & (rotated < 1.), axis=(1, 2)))[:][0] idx = valid[np.random.random_integers(0, high=np.shape(valid)[0])] pts2 = rotated[idx] # Rescale to actual size shape = shape[::-1] # different convention [y, x] pts1 *= np.expand_dims(shape, axis=0).astype(np.float32) pts2 *= np.expand_dims(shape, axis=0).astype(np.float32) def ax(p, q): return [p[0], p[1], 1, 0, 0, 0, -p[0] * q[0], -p[1] * q[0]] def ay(p, q): return [0, 0, 0, p[0], p[1], 1, -p[0] * q[1], -p[1] * q[1]] a_mat = np.stack([f(pts1[i], pts2[i]) for i in range(4) for f in (ax, ay)], axis=0) p_mat = np.transpose( np.stack([[pts2[i][j] for i in range(4) for j in range(2)]], axis=0)) homography = np.transpose(np.linalg.solve(a_mat, p_mat)) return homography
def get_current_frame(self, draw_options={}): with self.current_frame_lock: frame_copy = np.copy(self._current_frame) frame_time = self.current_frame_time tracked_objects = { k: v.to_dict() for k, v in self.tracked_objects.items() } motion_boxes = self.motion_boxes.copy() regions = self.regions.copy() frame_copy = cv2.cvtColor(frame_copy, cv2.COLOR_YUV2BGR_I420) # draw on the frame if draw_options.get('bounding_boxes'): # draw the bounding boxes on the frame for obj in tracked_objects.values(): thickness = 2 color = COLOR_MAP[obj['label']] if obj['frame_time'] != frame_time: thickness = 1 color = (255, 0, 0) # draw the bounding boxes on the frame box = obj['box'] draw_box_with_label( frame_copy, box[0], box[1], box[2], box[3], obj['label'], f"{int(obj['score']*100)}% {int(obj['area'])}", thickness=thickness, color=color) if draw_options.get('regions'): for region in regions: cv2.rectangle(frame_copy, (region[0], region[1]), (region[2], region[3]), (0, 255, 0), 2) if draw_options.get('zones'): for name, zone in self.camera_config.zones.items(): thickness = 8 if any([ name in obj['current_zones'] for obj in tracked_objects.values() ]) else 2 cv2.drawContours(frame_copy, [zone.contour], -1, zone.color, thickness) if draw_options.get('mask'): mask_overlay = np.where(self.camera_config.motion.mask == [0]) frame_copy[mask_overlay] = [0, 0, 0] if draw_options.get('motion_boxes'): for m_box in motion_boxes: cv2.rectangle(frame_copy, (m_box[0], m_box[1]), (m_box[2], m_box[3]), (0, 0, 255), 2) if draw_options.get('timestamp'): time_to_show = datetime.datetime.fromtimestamp( frame_time).strftime("%m/%d/%Y %H:%M:%S") cv2.putText(frame_copy, time_to_show, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, fontScale=.8, color=(255, 255, 255), thickness=2) return frame_copy
def _diff(self): # Much of the code for comparing columns is similar to the code for # comparing headers--consider refactoring colsa = self.a.columns colsb = self.b.columns if len(colsa) != len(colsb): self.diff_column_count = (len(colsa), len(colsb)) # Even if the number of columns are unequal, we still do comparison of # any common columns colsa = dict((c.name.lower(), c) for c in colsa) colsb = dict((c.name.lower(), c) for c in colsb) if '*' in self.ignore_fields: # If all columns are to be ignored, ignore any further differences # between the columns return # Keep the user's original ignore_fields list for reporting purposes, # but internally use a case-insensitive version ignore_fields = set([f.lower() for f in self.ignore_fields]) # It might be nice if there were a cleaner way to do this, but for now # it'll do for fieldname in ignore_fields: fieldname = fieldname.lower() if fieldname in colsa: del colsa[fieldname] if fieldname in colsb: del colsb[fieldname] colsa_set = set(colsa.values()) colsb_set = set(colsb.values()) self.common_columns = sorted(colsa_set.intersection(colsb_set), key=lambda c: c.name) self.common_column_names = set([col.name.lower() for col in self.common_columns]) left_only_columns = dict((col.name.lower(), col) for col in colsa_set.difference(colsb_set)) right_only_columns = dict((col.name.lower(), col) for col in colsb_set.difference(colsa_set)) if left_only_columns or right_only_columns: self.diff_columns = (left_only_columns, right_only_columns) self.diff_column_names = ([], []) if left_only_columns: for col in self.a.columns: if col.name.lower() in left_only_columns: self.diff_column_names[0].append(col.name) if right_only_columns: for col in self.b.columns: if col.name.lower() in right_only_columns: self.diff_column_names[1].append(col.name) # If the tables have a different number of rows, we don't compare the # columns right now. # TODO: It might be nice to optionally compare the first n rows where n # is the minimum of the row counts between the two tables. if len(self.a) != len(self.b): self.diff_rows = (len(self.a), len(self.b)) return # If the tables contain no rows there's no data to compare, so we're # done at this point. (See ticket #178) if len(self.a) == len(self.b) == 0: return # Like in the old fitsdiff, compare tables on a column by column basis # The difficulty here is that, while FITS column names are meant to be # case-insensitive, PyFITS still allows, for the sake of flexibility, # two columns with the same name but different case. When columns are # accessed in FITS tables, a case-sensitive is tried first, and failing # that a case-insensitive match is made. # It's conceivable that the same column could appear in both tables # being compared, but with different case. # Though it *may* lead to inconsistencies in these rare cases, this # just assumes that there are no duplicated column names in either # table, and that the column names can be treated case-insensitively. for col in self.common_columns: name_lower = col.name.lower() if name_lower in ignore_fields: continue cola = colsa[name_lower] colb = colsb[name_lower] for attr, _ in _COL_ATTRS: vala = getattr(cola, attr, None) valb = getattr(colb, attr, None) if diff_values(vala, valb): self.diff_column_attributes.append( ((col.name.upper(), attr), (vala, valb))) arra = self.a[col.name] arrb = self.b[col.name] if (np.issubdtype(arra.dtype, float) and np.issubdtype(arrb.dtype, float)): diffs = where_not_allclose(arra, arrb, atol=0.0, rtol=self.tolerance) elif 'P' in col.format: diffs = ([idx for idx in xrange(len(arra)) if not np.allclose(arra[idx], arrb[idx], atol=0.0, rtol=self.tolerance)],) else: diffs = np.where(arra != arrb) self.diff_total += len(set(diffs[0])) if self.numdiffs >= 0: if len(self.diff_values) >= self.numdiffs: # Don't save any more diff values continue # Add no more diff'd values than this max_diffs = self.numdiffs - len(self.diff_values) else: max_diffs = len(diffs[0]) last_seen_idx = None for idx in islice(diffs[0], 0, max_diffs): if idx == last_seen_idx: # Skip duplicate indices, which my occur when the column # data contains multi-dimensional values; we're only # interested in storing row-by-row differences continue last_seen_idx = idx self.diff_values.append(((col.name, idx), (arra[idx], arrb[idx]))) total_values = len(self.a) * len(self.a.dtype.fields) self.diff_ratio = float(self.diff_total) / float(total_values)
def synthetic_data_generator(size,dgp,parameters,noise,plots = True): """ synthetic_data_generator(size,dgp,plots = True) # ------- PARAMETERS -------- # size: int size of the sample dgp: str data generating process ; "sigmoid", "quadratic", "noise" parameters: dict true parameters, dict keys are "alpha","beta1","beta2" plots : Bool True if you wants the plots to be savedù noise : float Noise coefficient # ------- OUTPUT ------------ # data: np.array data generated """ # get parameters alpha = parameters["alpha"] ; beta1 = parameters["beta1"] ; beta2 = parameters["beta2"] # --------------------------- generate data on independent variables --------------------------- # # generate the independent variables x1 = np.random.normal(loc = 200, scale = 100, size = size) x2 = np.linspace(-300,300,size) # generate the y from the data based on the case if dgp == "sigmoid": # case 1 use a simple logistic true_values = sigmoid(alpha + beta1 * x1 + beta2 * x2) values = sigmoid(alpha + beta1 * x1 + beta2 * x2 + np.random.normal(loc = 0, scale = noise * np.abs((x1 - x1.mean())))) # set threshold threshold = values.mean() y = (values >= threshold).astype(int) elif dgp == "quadratic": # case 2 circle true_values = alpha + beta1 * x1**2 + beta2 * x2**2 values = true_values + np.random.normal(loc = 0, scale = noise * (x1 - x1.mean())**2) # set threshold threshold = values.mean() y = (values >= threshold).astype(int) elif dgp == "noise": # purely random y y = ss.bernoulli.rvs(p = 0.5, size = size) else: raise Exception # --------------------------- plot the data ---------------------------------------- # if plots == True: _,ax = plt.subplots() ax.scatter(x1[np.where(y == 0)],x2[np.where(y == 0)],color="blue") ax.scatter(x1[np.where(y == 1)],x2[np.where(y == 1)],color="yellow") if (dgp == "sigmoid"): ax.plot(x1[np.argsort(x1)],(inv_sigmoid(threshold) - alpha - beta1 * x1[np.argsort(x1)])/beta2) boundary = [] boundary.append(np.array([x1[np.argsort(x1)],(inv_sigmoid(threshold) - alpha - beta1 * x1[np.argsort(x1)])/beta2]).T) elif (dgp == "quadratic"): ax.plot(x1[np.argsort(x1)],np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)) ax.plot(x1[np.argsort(x1)],-np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)) boundary = [] boundary.append(np.array([x1[np.argsort(x1)],-np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)]).T) boundary.append(np.array([x1[np.argsort(x1)],np.sqrt((threshold - alpha - beta1 * x1[np.argsort(x1)]**2)/beta2)]).T) else: boundary = [] ax.set_xlabel("x1") ax.set_ylabel("x2") graph_name = "SyntheticData_scatter_" + dgp plt.savefig("/Users/davideferri/Documents/Repos/sklearn_applications/BivariateClassification_MLexample/graphs/" + graph_name) # -------------------------- return the data ------------------------------------ # # get the data in a single array data = np.array([y,x1,x2]).T return data,boundary
sim = 'SIMBA' snapnum = 33 realizations = 66 # images parameters grid = 250 splits = 5 # parameters of the density field routine periodic = True verbose = False x_min, y_min = 0.0, 0.0 ##################################################################################### # find the numbers that each cpu will work with numbers = np.where(np.arange(realizations)%nprocs==myrank)[0] # define the matrix hosting all the maps maps_T_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_T_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Z_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Z_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_P_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_P_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mg_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mg_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mc_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Mc_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Ms_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Ms_tot = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32) maps_Vg_loc = np.zeros((splits*3*realizations, grid, grid), dtype=np.float32)
def load_hdf5_flex(self, restartFolder, timestep, tXY): """ This function reads and allocates **badlands** parameters & variables from a previous simulation with flexural isostasy turned on. These parameters are obtained from a specific time HDF5 file. Args: restartFolder: restart folder name as defined in tge XML input file. timestep: time step to load defined based on output interval number. tXY: 2D numpy array TIN grid local coordinates. Returns ------- elev numpy 1D array containing the updated elevation from the restart model. cum numpy 1D array containing the updated erosion/deposition values from the restart model. fcum numpy 1D array containing the cumulative flexural isostasy values from the restart model. Note: Flexural isostasy is obtained from the gFlex_ package available on Github! Wickert, A. D. (2016), Open-source modular solutions for flexural isostasy: gFlex v1.0, Geosci. Model Dev., 9(3), 997–1017, `doi:10.5194/gmd-9-997-2016`_. .. _`doi:10.5194/gmd-9-997-2016`: https://doi.org/10.5194/gmd-9-997-2016 .. _gflex: https://github.com/awickert/gFlex """ if os.path.exists(restartFolder): folder = restartFolder + "/h5/" fileCPU = "tin.time%s.hdf5" % timestep restartncpus = len(glob.glob1(folder, fileCPU)) if restartncpus == 0: raise ValueError( "The requested time step for the restart simulation cannot be found in the restart folder." ) else: raise ValueError( "The restart folder is missing or the given path is incorrect." ) df = h5py.File("%s/h5/tin.time%s.hdf5" % (restartFolder, timestep), "r") coords = numpy.array((df["/coords"])) cumdiff = numpy.array((df["/cumdiff"])) cumhill = numpy.array((df["/cumhill"])) cumfail = numpy.array((df["/cumfail"])) cumflex = numpy.array((df["/cumflex"])) x, y, z = numpy.hsplit(coords, 3) c = cumdiff h = cumhill s = cumfail f = cumflex XY = numpy.column_stack((x, y)) tree = cKDTree(XY) distances, indices = tree.query(tXY, k=3) if len(z[indices].shape) == 3: z_vals = z[indices][:, :, 0] c_vals = c[indices][:, :, 0] h_vals = h[indices][:, :, 0] s_vals = s[indices][:, :, 0] f_vals = f[indices][:, :, 0] else: z_vals = z[indices] c_vals = c[indices] s_vals = s[indices] h_vals = h[indices] f_vals = f[indices] distances[distances < 0.0001] = 0.0001 with numpy.errstate(divide="ignore"): elev = numpy.average(z_vals, weights=(1.0 / distances), axis=1) cum = numpy.average(c_vals, weights=(1.0 / distances), axis=1) hcum = numpy.average(h_vals, weights=(1.0 / distances), axis=1) scum = numpy.average(s_vals, weights=(1.0 / distances), axis=1) fcum = numpy.average(f_vals, weights=(1.0 / distances), axis=1) onIDs = numpy.where(distances[:, 0] <= 0.0001)[0] if len(onIDs) > 0: if len(z[indices].shape) == 3: elev[onIDs] = z[indices[onIDs, 0], 0] cum[onIDs] = c[indices[onIDs, 0], 0] hcum[onIDs] = h[indices[onIDs, 0], 0] scum[onIDs] = s[indices[onIDs, 0], 0] fcum[onIDs] = f[indices[onIDs, 0], 0] else: elev[onIDs] = z[indices[onIDs, 0]] cum[onIDs] = c[indices[onIDs, 0]] hcum[onIDs] = h[indices[onIDs, 0]] scum[onIDs] = s[indices[onIDs, 0]] fcum[onIDs] = f[indices[onIDs, 0]] return elev, cum, hcum, scum, fcum
def beam_search(self, data_loader, k=12, max_len=100, avoid_rep=False, avoid_unk=False): """Performs beam search over split and returns the hyps.""" results = [] inf = 1e3 bos = self.trg_vocab['<bos>'] eos = self.trg_vocab['<eos>'] for batch in pbar(data_loader, unit='batch'): n = batch.size # Mask to apply to pdxs.view(-1) to fix indices nk_mask = torch.arange(n * k).long().cuda() pdxs_mask = (nk_mask / k) * k # Tile indices to use in the loop to expand first dim tile = nk_mask / k # We can fill this to represent the beams in tensor format beam = torch.zeros((max_len, n, k)).long().cuda() # Encode source sentences into ctxs (S*N*C) ctxs, ctx_mask = self.enc(to_var(batch[self.sl], volatile=True)) # Get initial decoder state (N*H) h_t = self.dec.f_init(ctxs, ctx_mask) # Initial y_t for <bos> embs: N x emb_dim y_t = self.dec.emb(to_var( torch.ones(n).long() * bos, volatile=True)) log_p, h_t = self.dec.f_next(ctxs, ctx_mask, y_t, h_t) nll, beam[0] = log_p.data.topk(k, sorted=False, largest=False) for t in range(1, max_len): cur_tokens = beam[t - 1].view(-1) fini_idxs = (cur_tokens == eos).nonzero() n_fini = fini_idxs.numel() if n_fini == n * k: break # Fetch embs for the next iteration (N*K, E) y_t = self.dec.emb(to_var(cur_tokens, volatile=True)) # Get log_probs and new LSTM states (log_p, N*K, V) ctxs, ctx_mask = ctxs[:, tile], ctx_mask[:, tile] log_p, h_t = self.dec.f_next(ctxs, ctx_mask, y_t, h_t[tile]) log_p = log_p.data # Suppress probabilities of previous tokens if avoid_rep: log_p.view(-1).index_fill_( 0, cur_tokens + (nk_mask * self.n_trg_vocab), inf) # Avoid <unk> tokens if avoid_unk: log_p[:, self.trg_vocab['<unk>']] = inf # Favor finished hyps to generate <eos> again # Their nll scores will not increase further and they will # always be kept in the beam. if n_fini > 0: fidxs = fini_idxs[:, 0] log_p.index_fill_(0, fidxs, inf) log_p.view(-1).index_fill_( 0, fidxs * self.n_trg_vocab + eos, 0) # Expand to 3D, cross-sum scores and reduce back to 2D nll = (nll.unsqueeze(2) + log_p.view(n, k, -1)).view(n, -1) # Reduce (N, K*V) to k-best nll, idxs = nll.topk(k, sorted=False, largest=False) # previous indices into the beam and current token indices pdxs = idxs / self.n_trg_vocab # Insert current tokens beam[t] = idxs % self.n_trg_vocab # Permute all hypothesis history according to new order beam[:t] = beam[:t].gather(2, pdxs.repeat(t, 1, 1)) # Compute correct previous indices # Mask is needed since we're in flattened regime tile = pdxs.view(-1) + pdxs_mask # Put an explicit <eos> to make idxs_to_sent happy beam[max_len - 1] = eos # Get beam and scores to CPU beam = beam.cpu().numpy() nll = nll.cpu().numpy() # Normalize scores by length lens, idxs = np.where(beam.reshape(max_len, -1) == eos) idxs = idxs.tolist() # Get sequence lengths lens = [lens[idxs.index(i)] for i in range(n * k)] # Avoid divide-by-zero in extreme case where hyp is only <eos> lens = [len_ if len_ != 0 else 1 for len_ in lens] nll /= np.array(lens).reshape(n, k) # Get best hyp for each sample in the batch hyps = beam[:, range(n), nll.argmin(axis=1)].T results.extend([self.trg_vocab.idxs_to_sent(hy) for hy in hyps]) return results
def objective(limit, target): w = np.where(like>limit) count = histogram[w] return count.sum() - target
img_disp_target[pos_mask] = targets return img_disp_target if __name__ == '__main__': # 缺点: 中心采样策略无法反映hw变化,而且既然叫做半径,为啥mask区域不是圆形,而是正方形 center_sampling = True # 是否使用中心采样策略 feature_shape = (100, 100, 3) strides = 4 radius = 3.5 # 默认1.5 center_sample_radius = radius * strides # 扩展半径radius,值越大,扩展面积越大 gt_boox = [20, 30, 80, 71] # 特征图size xyxy gt_bbox = torch.as_tensor(gt_boox, dtype=torch.float32).view(-1, 4) pos_mask, bbox_targets = get_target_mask(gt_bbox, feature_shape, center_sample_radius, center_sampling) # 可视化 pos_mask1 = pos_mask[..., 0].numpy() gray_img = np.where(pos_mask1 > 0, 255, 0).astype(np.uint8) # 绘制原始bbox img = mmcv.gray2bgr(gray_img) cv2.rectangle(img, (gt_boox[0], gt_boox[1]), (gt_boox[2], gt_boox[3]), color=(255, 0, 0)) cv2.namedWindow('img', 0) mmcv.imshow(img, 'img') # 显示centerness centerness_targets = centerness_target(pos_mask, bbox_targets) centerness_targets = centerness_targets.view(feature_shape[0], feature_shape[1]) plt.imshow(centerness_targets) plt.show()
def get_hist_data(self, contract, durationStr='10 D', barSizeSetting='1 day', show='TRADES'): """ :param symbol: e.g. 'SPY' :param type: e.g. 'STK', 'FUT', 'OPT' :param durationStr: [S Seconds, D Day, W Week, M Month, Y Year ] :param barSizeSetting: [1 secs, 5 secs, 10 secs, 15 secs, 30 secs, 1 min, 2 mins, 3 mins, 5 mins, 10 mins ,15 mins, 20 mins, 30 mins, 1 hour, 2 hours, 3 hours, 4 hours, 8 hours, 1 day, 1 week, 1 month] :param show: [OPTION_IMPLIED_VOLATILITY, TRADES] :return: :return: """ # k = 0 # while k < 10: # contract_details_list = self.tws.resolve_ib_contract(symbol, type) # try: # contract_details = contract_details_list[0] # break # except TypeError: # sleep(2) # k = k + 1 # if k == 10: # return -1 # # contract = contract_details.contract id = str(contract.symbol) + str(contract.secType) + str(contract.exchange) + str(contract.primaryExchange) + show + barSizeSetting if id in self.data.keys(): date_updated = self.data[id]['UpdateDate'] hist_data_old = self.data[id]['Data'] if date_updated.date() == datetime.datetime.today().date(): hist_data = hist_data_old else: if hist_data_old.size > 0: # date_last_str = hist_data_old[-1][BAR_DICT['Date']] date_last_date = hist_data_old[-1][BAR_DICT['Date']] date_now = datetime.datetime.now() date_delta = date_now - date_last_date date_delta_days = date_delta.days if date_delta_days > 0: hist_data_new = self.tws.get_IB_historical_data(contract, durationStr='{} D'.format(date_delta_days), barSizeSetting=barSizeSetting, show=show) hist_data_new = Database.convert_hist_data(hist_data_new) z = np.where(hist_data_old[:, 0] == date_last_date)[0][0] - 1 if z is not None and hist_data_new.size > 0: hist_data_old = hist_data_old[:z] hist_data = np.concatenate((hist_data_old, hist_data_new), axis=0) else: hist_data = hist_data_old else: hist_data = hist_data_old else: hist_data = self.tws.get_IB_historical_data(contract, durationStr=durationStr, barSizeSetting=barSizeSetting, show=show) hist_data = Database.convert_hist_data(hist_data) sleep(2) self.data[id]['Data'] = hist_data self.data[id]['UpdateDate'] = datetime.datetime.now() self.save(self.data, 'HistData') else: hist_data = self.tws.get_IB_historical_data(contract, durationStr=durationStr, barSizeSetting=barSizeSetting, show=show) hist_data = Database.convert_hist_data(hist_data) sleep(5) if hist_data.size != 0: self.data[id] = {'Data': None, 'UpdateDate': None} self.data[id]['Data'] = hist_data self.data[id]['UpdateDate'] = datetime.datetime.now() self.save(self.data, 'HistData') else: return -1 # self.data[id]['Data'] = hist_data # self.data[id]['UpdateDate'] = datetime.datetime.now() return hist_data
def rg_update(s , r , ac , af , s_max , rg): rg = np.where(s>s_max , rg_cal(r , ac , af) , rg) s_max = np.maximum(s , s_max) return rg , s_max
def _trim_cell(relative_axes, cell, symprec): """Trim overlapping atoms Parameters ---------- relative_axes: ndarray Transformation matrix to transform supercell to a smaller cell such as: trimmed_lattice = np.dot(relative_axes.T, cell.get_cell()) shape=(3,3) cell: PhonopyAtoms A supercell symprec: float Tolerance to find overlapping atoms in the trimmed cell Returns ------- tuple trimmed_cell, extracted_atoms, mapping_table """ positions = cell.get_scaled_positions() numbers = cell.get_atomic_numbers() masses = cell.get_masses() magmoms = cell.get_magnetic_moments() lattice = cell.get_cell() trimmed_lattice = np.dot(relative_axes.T, lattice) trimmed_positions = [] trimmed_numbers = [] if masses is None: trimmed_masses = None else: trimmed_masses = [] if magmoms is None: trimmed_magmoms = None else: trimmed_magmoms = [] extracted_atoms = [] positions_in_new_lattice = np.dot(positions, np.linalg.inv(relative_axes).T) positions_in_new_lattice -= np.floor(positions_in_new_lattice) trimmed_positions = np.zeros_like(positions_in_new_lattice) num_atom = 0 mapping_table = np.arange(len(positions), dtype='intc') for i, pos in enumerate(positions_in_new_lattice): is_overlap = False if num_atom > 0: diff = trimmed_positions[:num_atom] - pos diff -= np.rint(diff) # Older numpy doesn't support axis argument. # distances = np.linalg.norm(np.dot(diff, trimmed_lattice), axis=1) # overlap_indices = np.where(distances < symprec)[0] distances = np.sqrt( np.sum(np.dot(diff, trimmed_lattice)**2, axis=1)) overlap_indices = np.where(distances < symprec)[0] if len(overlap_indices) > 0: assert len(overlap_indices) == 1 is_overlap = True mapping_table[i] = extracted_atoms[overlap_indices[0]] if not is_overlap: trimmed_positions[num_atom] = pos num_atom += 1 trimmed_numbers.append(numbers[i]) if masses is not None: trimmed_masses.append(masses[i]) if magmoms is not None: trimmed_magmoms.append(magmoms[i]) extracted_atoms.append(i) # scale is not always to become integer. scale = 1.0 / np.linalg.det(relative_axes) if len(numbers) == np.rint(scale * len(trimmed_numbers)): trimmed_cell = PhonopyAtoms( numbers=trimmed_numbers, masses=trimmed_masses, magmoms=trimmed_magmoms, scaled_positions=trimmed_positions[:num_atom], cell=trimmed_lattice, pbc=True) return trimmed_cell, extracted_atoms, mapping_table else: return False
def full_date_pkr(x): return int(x.game_id[3:11]) def full_date_pkr2(x): return int(x.game_id[3:12]) read_df['full_date'] = read_df.apply(full_date_pkr, axis=1) read_df['full_date2'] = read_df.apply(full_date_pkr2, axis=1) df = read_df.copy() df['h'] = np.where(df.h_fl > 0, 1, 0) df = df.groupby(by=['full_date', 'full_date2', 'bat_id']).agg({'h': 'sum'}) df = df.reset_index() df = df.merge(players, left_on='bat_id', right_on='retroID') def debut_pkr(x): return int(x.debut.replace("-", "")) df['debut2'] = df.apply(debut_pkr, axis=1) df = df.query("debut2 >= 19900101") player_list = list(df.bat_id.unique())
def add_reg_noise(data_name, X, y, lat, lon, pr_noise, nlabel, y_perc = np.nan, cutoff = 1., region_name='ENSO'): lat = lat.flatten() lon = lon.flatten() NSAMPLES = np.shape(X)[0] #----------------------------------------------- # define the tranquil and noisy regions if data_name.find('mixedLabels')==0: inoise = np.arange(0,len(y)) tranquil = is_tranquil(NSAMPLES,inoise) elif data_name.find('tranquilFOO') == 0: reg_lats, reg_lons = get_region(region_name = region_name) ilat = np.where(np.logical_and(lat>=reg_lats[0],lat<=reg_lats[1]))[0] ilon = np.where(np.logical_and(lon>=reg_lons[0],lon<=reg_lons[1]))[0] reg_avg = np.nansum(np.nansum(X[:,:,ilon],axis=-1)[:,ilat],axis=-1)/(len(ilat)*len(ilon)) print('region shape = ' + str(len(ilat)) + ' x ' + str(len(ilon))) inoise = np.where(reg_avg<=cutoff)[0] tranquil = is_tranquil(NSAMPLES,inoise) elif data_name.find('noNoise') == 0: inoise = [] tranquil = np.ones((NSAMPLES,)) else: raise ValueError('no such data noise type.') #----------------------------------------------- #change the labels on the noisy data y_new = np.copy(y) corrupt = np.zeros(np.shape(tranquil)) for i in inoise: if np.random.random() < pr_noise: i_rand_sample = np.random.randint(0,NSAMPLES) value = y[i_rand_sample] corrupt_value = 1 else: value = y[i] corrupt_value = 0 y_new[i] = value corrupt[i] = corrupt_value itranquil = np.where(tranquil==1)[0] i = np.where(corrupt==1)[0] k = np.where(np.logical_and(tranquil==1, corrupt==1))[0] print('\n----Mislabeled----') print('# tranquil = ' + str(len(itranquil)) + ' out of ' + str(NSAMPLES) + ' samples') print('percent tranquil = ' + str(np.round(100.*len(itranquil)/len(tranquil))) + '%') try: print('tranquil mislabeled = ' + str(np.round(100.*len(k)/len(itranquil))) + '%') print('non-tranquil mislabeled = ' + str(np.round(100.*len(i)/len(inoise))) + '%') except: pass print('total mislabeled = ' + str(np.round(100.*len(i)/len(y_new))) + '%') #----------------------------------------------- return X, y_new, tranquil, corrupt
def add_bbox_regression_targets(roidb): """Add information needed to train bounding-box regressors.""" assert len(roidb) > 0 assert 'max_classes' in roidb[0], 'Did you call prepare_roidb first?' num_images = len(roidb) # Infer number of classes from the number of columns in gt_overlaps num_classes = roidb[0]['gt_overlaps'].shape[1] for im_i in xrange(num_images): rois = roidb[im_i]['boxes'] max_overlaps = roidb[im_i]['max_overlaps'] max_classes = roidb[im_i]['max_classes'] roidb[im_i]['bbox_targets'] = \ _compute_targets(rois, max_overlaps, max_classes) if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED: # Use fixed / precomputed "means" and "stds" instead of empirical values means = np.tile( np.array(cfg.TRAIN.BBOX_NORMALIZE_MEANS), (num_classes, 1)) stds = np.tile( np.array(cfg.TRAIN.BBOX_NORMALIZE_STDS), (num_classes, 1)) else: # Compute values needed for means and stds # var(x) = E(x^2) - E(x)^2 class_counts = np.zeros((num_classes, 1)) + cfg.EPS sums = np.zeros((num_classes, 4)) squared_sums = np.zeros((num_classes, 4)) for im_i in xrange(num_images): targets = roidb[im_i]['bbox_targets'] for cls in xrange(1, num_classes): cls_inds = np.where(targets[:, 0] == cls)[0] if cls_inds.size > 0: class_counts[cls] += cls_inds.size sums[cls, :] += targets[cls_inds, 1:].sum(axis=0) squared_sums[cls, :] += \ (targets[cls_inds, 1:] ** 2).sum(axis=0) means = sums / class_counts stds = np.sqrt(squared_sums / class_counts - means ** 2) print 'bbox target means:' print means print means[1:, :].mean(axis=0) # ignore bg class print 'bbox target stdevs:' print stds print stds[1:, :].mean(axis=0) # ignore bg class # Normalize targets if cfg.TRAIN.BBOX_NORMALIZE_TARGETS: print "Normalizing targets" for im_i in xrange(num_images): targets = roidb[im_i]['bbox_targets'] for cls in xrange(1, num_classes): cls_inds = np.where(targets[:, 0] == cls)[0] roidb[im_i]['bbox_targets'][cls_inds, 1:] -= means[cls, :] roidb[im_i]['bbox_targets'][cls_inds, 1:] /= stds[cls, :] else: print "NOT normalizing targets" # These values will be needed for making predictions # (the predicts will need to be unnormalized and uncentered) return means.ravel(), stds.ravel()
def iterar(self): #Verifica os novos infectados por infectantes do tipo 1 e 2 #print(self.lista_infectados_tipo_1+self.lista_infectados_tipo_2) lista_novos_infectados_tipo1, lista_novos_infectados_tipo2 = self.verificar_infeccao(self.lista_infectados_tipo_1+self.lista_infectados_tipo_2) for indice in lista_novos_infectados_tipo1: self.criar_individuo(self.INFECTADO_TIPO_1, indice) for indice in lista_novos_infectados_tipo2: self.criar_individuo(self.INFECTADO_TIPO_2, indice) #Verifica morte dos infectados tipo 2 lista_mortos = self.checagem_morte_lista(self.lista_infectados_tipo_2) #retira os indices dos individuos mortos da lista de infectados self.lista_infectados_tipo_2 = [indice for indice in self.lista_infectados_tipo_2 if indice not in lista_mortos] #Instancia individuos mortos na matriz for indice in lista_mortos: self.criar_individuo(self.MORTO, indice) #atualiza o número de mortos na matriz self.num_mortos = self.num_mortos + len(lista_mortos) #Verifica cura dos infectados tipo 1 lista_curados_t1 = self.checagem_cura_lista(self.lista_infectados_tipo_1) #Verifica cura dos infectados tipo 2 lista_curados_t2 = self.checagem_cura_lista(self.lista_infectados_tipo_2 ) #Instancia individuos mortos na matriz for indice in lista_curados_t1+lista_curados_t2: self.criar_individuo(self.CURADO, indice) #atualiza o número de curados na matriz self.num_curados = self.num_curados + len(lista_curados_t1 + lista_curados_t2) #Atualiza a lista de infectados após a cura dos individuos self.lista_infectados_tipo_1 = [indice for indice in self.lista_infectados_tipo_1 if indice not in lista_curados_t1] self.lista_infectados_tipo_2 = [indice for indice in self.lista_infectados_tipo_2 if indice not in lista_curados_t2] #movimentação nova_lista_t1 = [] for indice in self.lista_infectados_tipo_1: nova_lista_t1.append(self.mover_infectante(indice)) self.lista_infectados_tipo_1 = nova_lista_t1 #print(self.lista_infectados_tipo_1) nova_lista_t2 = [] for indice in self.lista_infectados_tipo_2: nova_lista_t2.append(self.mover_infectante(indice)) self.lista_infectados_tipo_2 = nova_lista_t2 #print(self.lista_infectados_tipo_2) # matriz_infectantes = matriz_infectantes[matriz_infectantes < 3] indices_infectados = list(zip(*np.where((self.matriz_status == 1) + (self.matriz_status == 2)))) # indices_infectados = list(zip(*self.matriz_status.nonzero())) #indices_infectados = [indice for indice in indices_infectados if indice not in self.lista_infectados_tipo_1 + self.lista_infectados_tipo_2] # self.num_curados = 0 #self.num_mortos = 0 self.lista_infectados_tipo_1 = [] self.lista_infectados_tipo_2 = [] #novos_t1 = [] #novos_t2 = [] for indice in indices_infectados: #if indice not in self.lista_infectados_tipo_1 and indice not in self.lista_infectados_tipo_2: # print(indice) # print(self.matriz_status.shape) status = self.matriz_status[indice[0], indice[1]] if status == self.INFECTADO_TIPO_1: self.lista_infectados_tipo_1.append(indice) #novos_t1.append(indice) if status == self.INFECTADO_TIPO_2: self.lista_infectados_tipo_2.append(indice) #novos_t2.append(indice) #self.lista_infectados_tipo_1 = self.lista_infectados_tipo_1 + novos_t1 #self.lista_infectados_tipo_2 = self.lista_infectados_tipo_2 + novos_t2 dict = {'num_sadios': self.populacao_inicial - len(self.lista_infectados_tipo_1) -len(self.lista_infectados_tipo_2) -self.num_curados-self.num_mortos, 'num_infect_t1': len(self.lista_infectados_tipo_1), 'num_infect_t2': len(self.lista_infectados_tipo_2), 'num_curados': self.num_curados, 'num_mortos': self.num_mortos} self.dataframe = self.dataframe.append(dict, ignore_index=True) self.salvar_posicionamento() #adiciona 1 ao número de atualizações realizadas na matriz self.num_atualizacoes +=1
import cv2 import numpy as np mask_path = '/mnt/data7/slice_test_seg/ct_lesion-type_20200311_177' lung_path = '/home/xzw/lung_seg' img_path = '/mnt/data6/lung_data' output_path_slices = '/mnt/data7/slice_test_seg/mask_jpgs_new' output_path_raw = '/mnt/data7/slice_test_seg/raw_jpgs_new' os.makedirs(output_path_raw, exist_ok=True) os.makedirs(output_path_slices, exist_ok=True) all_files = glob.glob(mask_path + '/*mask-label.nii') for item in all_files: mask = sitk.ReadImage(item) mask = sitk.GetArrayFromImage(mask) zz, yy, xx = np.where(mask > 0) name = item.split('/')[-1].split('-mask')[0] if int(name.split('_')[0]) < 100: full_name = os.path.join( img_path, 'lung_1st', name.split('_')[0] + '_' + name.split('_')[1] + '.nii') full_name_lung = os.path.join(lung_path, 'illPatient1_' + name + '_label.nii') else: full_name = os.path.join( img_path, 'lung_2rd', str(int(name.split('_')[0]) - 100) + '_' + name.split('_')[1] + '.nii') full_name_lung = os.path.join( lung_path, 'illPatient2_' + str(int(name.split('_')[0]) - 100) +
def calc_statistic_outliers(self, factor: float = 3.0) -> Tuple[pd.DataFrame, pd.DataFrame]: """ Optional: factor (float) - prefactor for stddev to detect statistic outliers Returns: data (DataFrame) - pandas DataFrame with columns 'object_id', 'time', 'cluster_id', 'outlier' outlier_result (DataFrame) - pandas DataFrame with columns 'object_id', 'start_time', 'end_time', 'cluster_end_time', 'rating', 'distance' and 'outlier' """ timestamps = self._data[self._time_column_name].unique() timestamps.sort() self._outlier_rating = pd.DataFrame( columns=[self._object_column_name, 'start_time', 'end_time', 'rating', 'distance', 'outlier']) for t1 in range(len(timestamps) - 1): for t2 in range(t1 + 1, len(timestamps)): subsequence_ratings = self.calc_subsequence_ratings(timestamps[t1], timestamps[t2]) subsequence_ratings = subsequence_ratings.assign(distance=0) subsequence_ratings = subsequence_ratings.assign(outlier=0) std_dev = np.std(subsequence_ratings[subsequence_ratings['rating'] >= 0]['rating'].values) mean = np.mean(subsequence_ratings[subsequence_ratings['rating'] >= 0]['rating'].values) subsequence_ratings = subsequence_ratings.assign(distance=lambda x: abs(x['rating'] - mean)) subsequence_ratings[subsequence_ratings['rating'] >= 0] = subsequence_ratings[subsequence_ratings['rating'] >= 0].assign(outlier=lambda x: (x['distance'] <= factor * std_dev) - 1) self._outlier_rating = self._outlier_rating.append(subsequence_ratings) noise_object_ids = self._data[self._data[self._cluster_column_name] < 0][self._object_column_name].unique() for object_id in noise_object_ids: noisy_timestamps = self._data[(self._data[self._object_column_name] == object_id) & (self._data[self._cluster_column_name] < 0)][ self._time_column_name].unique().tolist() noisy_timestamps.sort() start_index = np.where(timestamps == noisy_timestamps[0])[0][0] counter = 1 for i in range(1, len(noisy_timestamps)): if noisy_timestamps[i] == timestamps[start_index + counter]: counter += 1 else: if counter > 1: self._outlier_rating = self._outlier_rating.append({self._object_column_name: object_id, 'start_time': timestamps[start_index], 'end_time': timestamps[ start_index + counter - 1], 'rating': -1, 'distance': -1, 'outlier': -2}, ignore_index=True) start_index = np.where(timestamps == noisy_timestamps[i])[0][0] counter = 1 if counter > 1: self._outlier_rating = self._outlier_rating.append({self._object_column_name: object_id, 'start_time': timestamps[start_index], 'end_time': timestamps[start_index + counter - 1], 'rating': -1, 'distance': -1, 'outlier': -2}, ignore_index=True) self._outlier_result = self._outlier_rating[self._outlier_rating['outlier'] < 0] timestamps = timestamps.tolist() self._data = self._data.assign(outlier=0) for index, row in self._outlier_result.iterrows(): for time_point in timestamps[timestamps.index(int(row['start_time'])): timestamps.index( int(row['end_time'])) + 1]: self._data.loc[(self._data[self._time_column_name] == time_point) & ( self._data[self._object_column_name] == row[self._object_column_name]), 'outlier'] = row['outlier'] return self._data, self._outlier_result
def detect_pnet20(self, im): """Get face candidates through pnet Parameters: ---------- im: numpy array input image array Returns: ------- boxes: numpy array detected boxes before calibration boxes_c: numpy array boxes after calibration """ h, w, c = im.shape net_size = 20 current_scale = float( net_size) / self.min_face_size # find initial scale im_resized = self.resize_image(im, current_scale) _, _, current_height, current_width = im_resized.shape if self.slide_window: # sliding window temp_rectangles = list() rectangles = list( ) # list of rectangles [x11, y11, x12, y12, confidence] (corresponding to original image) all_cropped_ims = list() while min(current_height, current_width) > net_size: current_y_list = range(0, current_height - net_size + 1, self.stride) if (current_height - net_size) % self.stride == 0 \ else range(0, current_height - net_size + 1, self.stride) + [current_height - net_size] current_x_list = range(0, current_width - net_size + 1, self.stride) if (current_width - net_size) % self.stride == 0 \ else range(0, current_width - net_size + 1, self.stride) + [current_width - net_size] for current_y in current_y_list: for current_x in current_x_list: cropped_im = im_resized[:, :, current_y:current_y + net_size, current_x:current_x + net_size] current_rectangle = [ int(w * float(current_x) / current_width), int(h * float(current_y) / current_height), int(w * float(current_x) / current_width) + int(w * float(net_size) / current_width), int(h * float(current_y) / current_height) + int(w * float(net_size) / current_width), 0.0 ] temp_rectangles.append(current_rectangle) all_cropped_ims.append(cropped_im) current_scale *= self.scale_factor im_resized = self.resize_image(im, current_scale) _, _, current_height, current_width = im_resized.shape ''' # helper for setting PNet batch size num_boxes = len(all_cropped_ims) batch_size = self.pnet_detector.batch_size ratio = float(num_boxes) / batch_size if ratio > 3 or ratio < 0.3: print "You may need to reset PNet batch size if this info appears frequently, \ face candidates:%d, current batch_size:%d"%(num_boxes, batch_size) ''' all_cropped_ims = np.vstack(all_cropped_ims) cls_scores, reg = self.pnet_detector.predict(all_cropped_ims) cls_scores = cls_scores[:, 1].flatten() keep_inds = np.where(cls_scores > self.thresh[0])[0] if len(keep_inds) > 0: boxes = np.vstack(temp_rectangles[ind] for ind in keep_inds) boxes[:, 4] = cls_scores[keep_inds] reg = reg[keep_inds].reshape(-1, 4) else: return None, None keep = py_nms(boxes, 0.7, 'Union') boxes = boxes[keep] boxes_c = self.calibrate_box(boxes, reg[keep]) else: # fcn all_boxes = list() while min(current_height, current_width) > net_size: cls_map, reg = self.pnet_detector.predict(im_resized) cls_map = cls_map.asnumpy() reg = reg.asnumpy() boxes = self.generate_bbox(cls_map[0, 1, :, :], reg, current_scale, self.thresh[0]) current_scale *= self.scale_factor im_resized = self.resize_image(im, current_scale) _, _, current_height, current_width = im_resized.shape if boxes.size == 0: continue keep = py_nms(boxes[:, :5], 0.5, 'Union') boxes = boxes[keep] all_boxes.append(boxes) if len(all_boxes) == 0: return None, None all_boxes = np.vstack(all_boxes) # merge the detection from first stage keep = py_nms(all_boxes[:, 0:5], 0.7, 'Union') all_boxes = all_boxes[keep] boxes = all_boxes[:, :5] bbw = all_boxes[:, 2] - all_boxes[:, 0] + 1 bbh = all_boxes[:, 3] - all_boxes[:, 1] + 1 # refine the boxes boxes_c = np.vstack([ all_boxes[:, 0] + all_boxes[:, 5] * bbw, all_boxes[:, 1] + all_boxes[:, 6] * bbh, all_boxes[:, 2] + all_boxes[:, 7] * bbw, all_boxes[:, 3] + all_boxes[:, 8] * bbh, all_boxes[:, 4] ]) boxes_c = boxes_c.T return boxes, boxes_c
def predict(self, A): return np.where(A > 0.5, 1., 0.)
def operations(img1,img2,threshold): image_1 = cv2.imread(img1) image_2 = cv2.imread(img2) ###transforming input images to grayscale images and then binarization ##image 1 blue_component1 = image_1[:,:,0] green_component1 = image_1[:,:,1] red_component1 = image_1[:,:,2] gray_image_1 = (0.11*blue_component1 + 0.59*green_component1 + 0.3*red_component1) binary_image_1 = np.where(gray_image_1 > threshold, 255,0) print(binary_image_1) ##image 2 blue_component2 = image_2[:,:,0] green_component2 = image_2[:,:,1] red_component2 = image_2[:,:,2] gray_image_2 = (0.11*blue_component2 + 0.59*green_component2 + 0.3*red_component2) binary_image_2 = np.where(gray_image_2 > threshold,255,0) ##addition #image_3 = binary_image_1 + binary_image_2 image_3_add = cv2.add(binary_image_1,binary_image_2) ##substraction #image_3 = binary_image_1 - binary_image_2 image_3_sub = cv2.subtract(binary_image_1,binary_image_2) ### logical xor xor = np.logical_xor(binary_image_1, binary_image_2) image_3_xor = np.where(xor > 0,255,0) ### logical andd #andd = np.logical_and(binary_image_1,binary_image_2) image_3_andd = np.where((binary_image_1*binary_image_2>0), 255,0) ### logical or orr = np.logical_or(binary_image_1,binary_image_2) image_3_orr = np.where(orr > 0,255,0) cv2.imwrite("image_add.jpg",image_3_add) image_add = cv2.imread("image_add.jpg") cv2.imshow('Addition of Images',image_add) cv2.imwrite("image_sub.jpg",image_3_sub) image_sub = cv2.imread("image_sub.jpg") cv2.imshow('Subtraction of Images',image_sub) cv2.imwrite("image_xor.jpg",image_3_xor) image_xor = cv2.imread("image_xor.jpg") cv2.imshow('Logical XOR of Images',image_xor) cv2.imwrite("image_and.jpg",image_3_andd) image_andd = cv2.imread("image_and.jpg") cv2.imshow('Logical AND of Images',image_andd) cv2.imwrite("image_or.jpg",image_3_orr) image_orr = cv2.imread("image_or.jpg") cv2.imshow('Logical OR of Images',image_orr) cv2.waitKey(0) cv2.destroyAllWindows() sys.exit()
plt.contourf(xx, yy, Z, cmap=cmap, alpha=0.25) plt.contour(xx, yy, Z, colors='k', linewidths=0.7) plt.scatter(X[0], X[1], c=Y[0], cmap=cmap, edgecolors='k') plt.savefig('decision_boundary_%s_%s.png' % (layer, node)) if __name__ == '__main__': X1 = np.random.randn(2, 40) X1 /= 1.5 * np.linalg.norm(X1, axis=0) Y1 = np.array([1.] * 40).reshape((1, 40)) X2 = np.random.randn(2, 40) X2 /= np.linalg.norm(X2, axis=0) Y2 = np.array([0.] * 40).reshape((1, 40)) X = np.concatenate([X1, X2], axis=1) Y = np.concatenate([Y1, Y2], axis=1) layers = initiate_layers([(3, tanh()), (1, sigmoid())], X) num_iterations = 10000 losses = [] for i in range(num_iterations): activations = forward_propogation(X, layers) layers = backward_propogation(Y, 1.0, activations, layers) P = np.where(activations[-1] > 0.5, 1., 0) losses.append(np.squeeze(loss(Y, activations[-1]))) print(metrics.accuracy_score(Y[0], P[0])) for i, (W, B, g) in enumerate(layers): for j in range(W.shape[0]): plot_decision_boundary(layers, X, Y, layer=i, node=j) plt.figure(figsize=(5, 5)) plt.plot(losses) plt.savefig('loss.png')
#year forest g = GeoTiff(lossyear) ex = grid.extent_in_crs(crs=wgs84) # l, r, b, t g.set_subset(corners=((ex[0], ex[2]), (ex[1], ex[3])), crs=wgs84, margin=10) ls = g.get_vardata() ls = np.array(ls, dtype=float) # ls[ls >100] = np.nan # ls = grid.map_gridded_data(ls, g.grid) ls = np.round(ls).astype(int) year_on_ds = ds18_present.salem.lookup_transform(ls, grid=g.grid, lut=lut, method=lambda x:np.bincount(x[np.where(x>0)]).argmax()) year_on_ds = year_on_ds.assign_coords(lon=grid.ll_coordinates[0][0,:], lat=grid.ll_coordinates[1][:,0]) # deforestation before 2009: set to 0 #pdb.set_trace() #lst_on_ds[np.where(year_on_ds<=9)]=0 lst_on_ds.values[np.where(year_on_ds.values<=9)]=0 # mask_river = grid.region_of_interest(shape=river) # mask_lakes = grid.region_of_interest(shape=lakes, roi=mask_river, all_touched=True) # mask_river[mask_lakes]= 1 # larr = ds18_present.copy() # larr.values = mask_river ds2f = ds18_present.values.flatten() dsf = ds18_hist.values.flatten()
def SE_BDD(H_mat, Z_mat, Threshold, Verbose = "True"): import numpy as np from numpy.random import seed from numpy.random import randn from numpy import mean from numpy import std #print("\n ************** State Estimation ************* \n") # takes H matrix Z measurements, Threshold and Verbose as input std_cutOff_th = 1.5 Max_Threshold = 10 Threshold_step = 2 doneFlag = 0 fullRank = True numberOfMeasurements = H_mat.shape[0] numberOfStates = H_mat.shape[1] #print("Considering only the taken measurements") ############# starting State Estimation ############### Z_mat = Z_mat[Z_mat[:,0].argsort(kind='mergesort')] Z_mat_Copy = Z_mat.copy() # Space holder for Z_est Z_est = np.zeros(numberOfMeasurements) list_of_removed_sensors = [] while(doneFlag == 0): #list_of_removed_sensors = [] #considering only the taken measurements Z_msr = Z_mat[Z_mat[:,1]==1][:,2] # considering only the corresponding columns in H H_msr = H_mat[Z_mat[:,1]==1] #Chekcing rank of H Rank = np.linalg.matrix_rank(H_msr) if Verbose == "True": print("Current Rank", Rank) print("H_msr Shape: ", H_msr.shape) ###### H is full rank --> Start Estimating if Rank == numberOfStates: #Estimating the states using WMSE estimator States = (np.linalg.inv(H_msr.T@H_msr)@H_msr.T)@Z_msr # Estimating the measurement from the estimated states Z_est = H_mat@States if Verbose == "True": pass #print("\n Initital Z_m: "+str(Z_mat)) #print("\n Sates: \n"+str(States)) #print("\n Z_est: \n"+str(Z_est)) ##################### Checking for Bad Data ################################## # Calculating the Noise M_Noise, P_Noise, Noisy_Index = Calc_Percent_of_Noise (Z_est, Z_mat, Threshold, fullRank, Verbose =="False") ##### If there are some noisy data ############# if( len(Noisy_Index) > 0): #print("***************** Bad Data Detected *********************") #changing the noisy measurement parameter (taken) from 1 to 0 # Analyzig the percentage of noise Noise_mean, Noise_std = mean(P_Noise), std(P_Noise) # identify outliers Noise_cut_off = Noise_mean + Noise_std * std_cutOff_th indxtbremoved = np.where(P_Noise > Noise_cut_off)[0] target_noisy_indx = indxtbremoved if indxtbremoved.size > 0 else Noisy_Index #np.argsort(P_Noise)[::-1] if Verbose == "True": print(f"Noise_mean: {Noise_mean}, Noise_std: {Noise_std}") print("Removing sensor: ", target_noisy_indx) ############################################################# # removing noisy sensors from the state estimation process Z_mat[target_noisy_indx,1] = -1 list_of_removed_sensors = list_of_removed_sensors + target_noisy_indx.tolist() else: if Verbose == "True": print("!!!! Done !!!\nNo more noisy data") doneFlag = 1 else: if Threshold < Max_Threshold: Threshold += Threshold_step if Verbose == "True": print(f"Relaxing the nosie threshold to {Threshold}") list_of_removed_sensors = [] Z_mat = Z_mat_Copy.copy() else: doneFlag = -1 if Verbose == "True": print(f"\n\n\nSystem Unobservable !, Rank = {Rank}") ##### Returning ############## fullRank = False Z_est = np.zeros(numberOfMeasurements) States = np.zeros(numberOfStates) ############### Returning ######################## #print("list_of_removed_sensors:", list_of_removed_sensors) return States, Z_est, Z_mat, fullRank
def predict(self, A): return np.where(A > 0, 1., -1.)
def pattern_gen(n, capacity): import numpy as np r = len(capacity) if r == 0: print(" number of sections cant be zero") return list_ = [] avg_frac = n/r avg_int = n//r diff = (avg_frac - avg_int) if diff >= 0.5: diff = 1 - diff step = avg_int + 1 ceil_sign = -1 else: step = avg_int ceil_sign = +1 liability = 0 repeat = 0 while repeat < r: repeat = repeat + 1 liability = liability + diff * ceil_sign if abs(liability) >= 0.99: added = round(step + liability) list_.append(added) n = n - added liability = liability - ceil_sign else: list_.append(round(step)) n = n - round(step) ################################## ######### checking capacity ##3## #print(list_) cap = np.array(capacity) dis = np.array(list_) diff = cap - dis #print("diff:", diff) lagg = np.sum(diff[np.where(diff<0)]) #print("lagg:", lagg) index = 0 while(lagg < 0): positiveIndex = np.where(diff > 0)[0] if index >= positiveIndex.size: index = 0 if positiveIndex.size == 0: print("Distribution not possible!!!!!") break # print(positiveIndex) # print(index) # print(diff[positiveIndex[index]]) diff[positiveIndex[index]] -= 1 lagg += 1 index += 1 #print(diff, lagg) diff[np.where(diff < 0)] = 0 dis = cap - diff #print(dis) return dis.astype(int).tolist()
def SE_BDD_(H_mat, Z_mat, W_list, Threshold_min, Threshold_max, Verbose = "True"): import numpy as np from numpy.random import seed from numpy.random import randn from numpy import mean from numpy import std if Verbose == "True": print("\n ************** State Estimation ************* \n") doneFlag = 0 fullRank = True Threshold = Threshold_min Threshold_step = 2 numberOfMeasurements = H_mat.shape[0] numberOfStates = H_mat.shape[1] ############# starting State Estimation ############### #Z_mat = Z_mat[Z_mat[:, 0].argsort(kind='mergesort')] Z_mat_Copy = Z_mat.copy() W_list = np.array(W_list) # Space holder for Z_est #Z_est = np.zeros(numberOfMeasurements) #list_of_removed_sensors = [] ################# Starting the loop ################################### while(doneFlag == 0): #list_of_removed_sensors = [] #considering only the taken measurements consideredIndx = np.where(Z_mat[:,1] == 1)[0] #print(Z_mat) Z_msr = Z_mat[consideredIndx][:,2] # considering only the corresponding columns in H H_msr = H_mat[consideredIndx] # Measurement Covarriance Residual Matrix R_msr = np.diag(W_list[consideredIndx]) #print(R_msr) R_inv = np.linalg.inv(R_msr) #print(R_inv) #Chekcing rank of H Rank = np.linalg.matrix_rank(H_msr) if H_msr.shape[0] > 0 else 0 if Verbose == "True": print("Current Rank", Rank) print("H_msr Shape: ", H_msr.shape) ###### H is full rank --> Start Estimating if Rank == numberOfStates: #Estimating the states using WMSE estimator inv__Ht_Rinv_H__Ht = np.linalg.inv(H_msr.T@R_inv@H_msr)@H_msr.T States = inv__Ht_Rinv_H__Ht@R_inv@Z_msr # Ω = R − H. (H_T.R−1.H)−1.HT # Omega is a residual covarience matrix Omega_mat = R_msr - (H_msr@inv__Ht_Rinv_H__Ht) #print("Check :\n", R_msr - ([email protected](H_msr.T@R_inv@H_msr)@H_msr.T)) #print(f"R_msr: {R_msr} \n Shape {R_msr.shape}") #print(f"Omega_mat: {Omega_mat} \n Shape {Omega_mat.shape}") # Estimating the measurement from the estimated states Z_est = H_mat@States if Verbose == "True": print("\n Initital Z_m: "+str(Z_mat)) print("\n Sates: \n"+str(States)) print("\n Z_est: \n"+str(Z_est)) print("Calling Noise.. CheckNoise...") ##################### Checking for Bad Data ################################## # Calculating the Noise M_Noise, P_Noise, doneFlag = CheckNoise( Z_est, Z_mat, Omega_mat, R_msr, Threshold, fullRank, Verbose =="False") # H is not a full rank matrix ............. abort estimation else: if Threshold < Threshold_max: Threshold += Threshold_step Z_mat = Z_mat_Copy.copy() if Verbose == "True": print(f"Relaxing the threshold to {Threshold}") else: doneFlag = -1 #system unobservable if Verbose == "True": print(f"\n\n\nSystem Unobservable !, Rank = {Rank}") ##### Returning ############## fullRank = False Z_est = np.zeros(numberOfMeasurements) States = np.zeros(numberOfStates) M_Noise = np.zeros(numberOfMeasurements) ############################################################################## Noisy_Indx = np.where(Z_mat[:,1] == -1)[0] return States, Z_est, Z_mat, M_Noise, Noisy_Indx, fullRank, Threshold
# Iterative read file ; each time 100000 records are read and processed for i in range(1, 10): data = np.genfromtxt( "C:\\Users\\Tejal\\Documents\\Tejal\\College\\INF552\\Project\\rating.csv", delimiter=",", max_rows=i * 100000, usecols=(0, 1, 2), skip_header=((i - 1) * 100000)) if i == 1: hdr = list(data[0]) data = np.delete(data, [0], axis=0) # Get unique users s = list(set(data[:, 0])) #Find number of occurences of the user for user in s: c = np.where(data[:, 0] == user) # Select only those users who have rated atleast 100 movies if total_users == 3000 or len(fin_data) > 200000: flag = 1 break if len(c[0]) > 100: total_users += 1 for j in c: fin_data = np.append(fin_data, data[j], axis=0) if flag == 1: break # Keep ratings of only those movies that have atleast 10 ratings # Get unique movies movie = list(set(fin_data[:, 1]))
def CheckNoiseCor (Z_est, Z_mat, Omega_mat, R_msr, Threshold, fullRank, Corr, Verbose): import math import numpy as np if fullRank != True: # if Verbose == "True": # print("System Unobservable!" return None, None, Z_mat[:,0] Z_msr = Z_mat[Z_mat[:, 1] == 1][:,2].copy() #################################################### Here --------------------> '''boolean index did not match indexed array along dimension 0; dimension is 53 but corresponding boolean dimension is 55''' if Verbose == "True": print("Starting BDD") print("Z_est: ", Z_est.shape) #print("Z_mat[:, 1] == 1", Z_mat[:, 1] == 1) #print(Z_mat) Z_est_msr = Z_est[Z_mat[:, 1] == 1] # Calculating the measurement error M_Noise = (Z_msr - Z_est_msr) M_Noise_norm = M_Noise.copy() # Calculating the normalized residuals for index, _ in enumerate(M_Noise): if index == 0: continue try: M_Noise_norm [index] = np.absolute(M_Noise [index])/math.sqrt(Omega_mat[index, index]) except: M_Noise_norm [index] = 0 if Verbose == "True": print("index: ", index, np.absolute(M_Noise [index])) print(f" Value Error, Expected postive, Got {Omega_mat[index, index]}") # Noise_mat_actual = np.zeros(Z_mat.shape[0]) # Noise_mat_actual[Z_mat[:,1] == 1] = M_Noise # Noise_mat_norm = np.zeros(Z_mat.shape[0]) # Noise_mat_norm[Z_mat[:,1] == 1] = M_Noise_norm # print(M_Noise.shape) Noise_mat_actual = M_Noise.copy() Noise_mat_norm = M_Noise_norm.copy() active_idx = np.where(Z_mat[:,1] == 1)[0] # Checking for Noisy data if np.max(Noise_mat_norm) > Threshold: tIndx = np.argmax(Noise_mat_norm) if Verbose == "True": print(f"targetedIndx in cut: {tIndx}--> Value : {Noise_mat_norm[tIndx]}") print("Updating Z_mat...") print("Before: ", Z_mat[tIndx]) #print("R_msr: ", R_msr.shape, Omega_mat.shape) if Corr == True: Z_mat[active_idx[tIndx], 2] = Z_mat[active_idx[tIndx], 2] - R_msr[tIndx,tIndx]/Omega_mat[tIndx,tIndx]*M_Noise[tIndx] else: Z_mat[active_idx[tIndx], 1] = -1 doneFlag = 0 if Verbose == "True": print("After: ", Z_mat[tIndx]) else: if Verbose == "True": print("No Bad Data Detected....") doneFlag = 1 Noise_mat_actual = np.zeros(Z_mat.shape[0]) Noise_mat_actual[Z_mat[:,1] == 1] = M_Noise.copy() Noise_mat_norm = np.zeros(Z_mat.shape[0]) Noise_mat_norm[Z_mat[:,1] == 1] = M_Noise_norm.copy() ############################################## return Noise_mat_actual, Noise_mat_norm, doneFlag