def get_cut_coords(map3d, n_axials=12, delta_z_axis=3): """ Heuristically computes optimal cut_coords for plot_map(...) call. Parameters ---------- map3d: 3D array the data under consideration n_axials: int, optional (default 12) number of axials in the plot delta_z_axis: int, optional (default 3) z-axis spacing Returns ------- cut_coords: 1D array of length n_axials the computed cut_coords """ z_axis_max = np.unravel_index(np.abs(map3d).argmax(), map3d.shape)[2] z_axis_min = np.unravel_index((-np.abs(map3d)).argmin(), map3d.shape)[2] z_axis_min, z_axis_max = (min(z_axis_min, z_axis_max), max(z_axis_max, z_axis_min)) z_axis_min = min(z_axis_min, z_axis_max - delta_z_axis * n_axials) cut_coords = np.linspace(z_axis_min, z_axis_max, n_axials) return cut_coords
def test_pad_input(): """Test `match_template` when `pad_input=True`. This test places two full templates (one with values lower than the image mean, the other higher) and two half templates, which are on the edges of the image. The two full templates should score the top (positive and negative) matches and the centers of the half templates should score 2nd. """ # Float prefactors ensure that image range is between 0 and 1 template = 0.5 * diamond(2) image = 0.5 * np.ones((9, 19)) mid = slice(2, 7) image[mid, :3] -= template[:, -3:] # half min template centered at 0 image[mid, 4:9] += template # full max template centered at 6 image[mid, -9:-4] -= template # full min template centered at 12 image[mid, -3:] += template[:, :3] # half max template centered at 18 result = match_template(image, template, pad_input=True, constant_values=image.mean()) # get the max and min results. sorted_result = np.argsort(result.flat) i, j = np.unravel_index(sorted_result[:2], result.shape) assert_equal(j, (12, 0)) i, j = np.unravel_index(sorted_result[-2:], result.shape) assert_equal(j, (18, 6))
def ensemble_tuning(X_train, y_train): kf = sklearn.cross_validation.KFold(X_train.shape[0], n_folds=5, shuffle=True, random_state=1234) model1 = sklearn.ensemble.RandomForestClassifier(criterion="entropy", max_depth=15, n_estimators=500, min_samples_leaf=4, min_samples_split=16, random_state=1234) #model2 = sklearn.neighbors.KNeighborsClassifier(n_neighbors=170, algorithm="kd_tree", weights="distance") model2 = sklearn.linear_model.LogisticRegression(C=2, penalty="l2") scores = np.zeros((5, 101), dtype=np.float32) for fold, (train_index, test_index) in enumerate(kf): model1.fit(X_train[train_index], y_train[train_index]) pred1 = model1.predict_proba(X_train[test_index])[:, 1] model2.fit(X_train[train_index], y_train[train_index]) pred2 = model2.predict_proba(X_train[test_index])[:, 1] print("Calculating scores...") for alpha in np.ndindex(101): scores[fold][alpha] = sklearn.metrics.roc_auc_score(y_train[test_index], 0.01*alpha[0]*pred1 + np.max(1 - 0.01*alpha[0], 0)*pred2) sc1 = np.mean(scores, axis = 0) * 1.0 / (fold+1) * 5 print(np.max(sc1), np.unravel_index(sc1.argmax(), sc1.shape), sc1[0], sc1[100]) scores1 = np.mean(scores, axis = 0) print(np.max(scores1), np.unravel_index(scores1.argmax(), scores1.shape), scores1[0], scores1[100])
def agg_lut(lathi,lonhi,latlo,lonlo): # outputlut=np.zeros((lathi.shape[0],lathi.shape[1],2))-9999 print("Computing Geographic Look Up Table") outputlut=np.empty((latlo.shape[0],latlo.shape[1],2),dtype=list) nx=lathi.shape[1] ny=lathi.shape[0] maxdist=((latlo[0,0]-latlo[1,1])**2 + (lonlo[0,0]-lonlo[1,1])**2)/1.5 for i in range(ny): dists=(latlo-lathi[i,0])**2+(lonlo-lonhi[i,0])**2 y,x=np.unravel_index(dists.argmin(),dists.shape) # print(i,ny,y,x,lathi[i,0],lonhi[i,0],latlo[y,x],lonlo[y,x]) for j in range(nx): xmax=min(nx,x+3) xmin=max(0,x-3) ymax=min(ny,y+3) ymin=max(0,y-3) windists=((latlo[ymin:ymax,xmin:xmax]-lathi[i,j])**2+ (lonlo[ymin:ymax,xmin:xmax]-lonhi[i,j])**2) yoff,xoff=np.unravel_index(windists.argmin(),windists.shape) x=xoff+xmin y=yoff+ymin if not outputlut[y,x,0]: outputlut[y,x,0]=list() outputlut[y,x,1]=list() if windists[yoff,xoff]<maxdist: outputlut[y,x,0].append(i) outputlut[y,x,1].append(j) return outputlut
def test_normalization(): """Test that `match_template` gives the correct normalization. Normalization gives 1 for a perfect match and -1 for an inverted-match. This test adds positive and negative squares to a zero-array and matches the array with a positive template. """ n = 5 N = 20 ipos, jpos = (2, 3) ineg, jneg = (12, 11) image = np.full((N, N), 0.5) image[ipos:ipos + n, jpos:jpos + n] = 1 image[ineg:ineg + n, jneg:jneg + n] = 0 # white square with a black border template = np.zeros((n + 2, n + 2)) template[1:1 + n, 1:1 + n] = 1 result = match_template(image, template) # get the max and min results. sorted_result = np.argsort(result.flat) iflat_min = sorted_result[0] iflat_max = sorted_result[-1] min_result = np.unravel_index(iflat_min, result.shape) max_result = np.unravel_index(iflat_max, result.shape) # shift result by 1 because of template border assert np.all((np.array(min_result) + 1) == (ineg, jneg)) assert np.all((np.array(max_result) + 1) == (ipos, jpos)) assert np.allclose(result.flat[iflat_min], -1) assert np.allclose(result.flat[iflat_max], 1)
def test_dtypes(self): # Test with different data types for dtype in [np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]: coords = np.array( [[1, 0, 1, 2, 3, 4], [1, 6, 1, 3, 2, 0]], dtype=dtype) shape = (5, 8) uncoords = 8*coords[0]+coords[1] assert_equal(np.ravel_multi_index(coords, shape), uncoords) assert_equal(coords, np.unravel_index(uncoords, shape)) uncoords = coords[0]+5*coords[1] assert_equal( np.ravel_multi_index(coords, shape, order='F'), uncoords) assert_equal(coords, np.unravel_index(uncoords, shape, order='F')) coords = np.array( [[1, 0, 1, 2, 3, 4], [1, 6, 1, 3, 2, 0], [1, 3, 1, 0, 9, 5]], dtype=dtype) shape = (5, 8, 10) uncoords = 10*(8*coords[0]+coords[1])+coords[2] assert_equal(np.ravel_multi_index(coords, shape), uncoords) assert_equal(coords, np.unravel_index(uncoords, shape)) uncoords = coords[0]+5*(coords[1]+8*coords[2]) assert_equal( np.ravel_multi_index(coords, shape, order='F'), uncoords) assert_equal(coords, np.unravel_index(uncoords, shape, order='F'))
def test_grad(self): eps = 1e-7 f, args, vals = self.get_args() output0 = f(*vals) # Go through and backpropagate all of the gradients from the outputs grad0 = [] for i in range(len(output0) - 2): grad0.append([]) for j in range(output0[i].size): ind = np.unravel_index(j, output0[i].shape) g = theano.function( args, theano.grad(self.op(*args)[i][ind], args)) grad0[-1].append(g(*vals)) # Loop over each input and numerically compute the gradient for k in range(len(vals)): for l in range(vals[k].size): inner = np.unravel_index(l, vals[k].shape) vals[k][inner] += eps plus = f(*vals) vals[k][inner] -= 2*eps minus = f(*vals) vals[k][inner] += eps # Compare to the backpropagated gradients for i in range(len(output0) - 2): for j in range(output0[i].size): ind = np.unravel_index(j, output0[i].shape) delta = 0.5 * (plus[i][ind] - minus[i][ind]) / eps ref = grad0[i][j][k][inner] assert np.abs(delta - ref) < 2*eps, \ "{0}".format((k, l, i, j, delta, ref, delta-ref))
def find_subset(target_grid,lon_src,lat_src): ny,nx = lon_src.shape lon_bl_tgt = target_grid.coords[0][0][0,0] ; lat_bl_tgt = target_grid.coords[0][1][0,0] # bottom left lon_br_tgt = target_grid.coords[0][0][-1,0] ; lat_br_tgt = target_grid.coords[0][1][-1,0] # bottom right lon_ul_tgt = target_grid.coords[0][0][0,-1] ; lat_ul_tgt = target_grid.coords[0][1][0,-1] # upper left lon_ur_tgt = target_grid.coords[0][0][-1,-1] ; lat_ur_tgt = target_grid.coords[0][1][-1,-1] # upper right dist_2_bottom_left_corner = distance_on_unit_sphere(lon_bl_tgt,lat_bl_tgt,lon_src,lat_src) j_bl_src, i_bl_src = _np.unravel_index(dist_2_bottom_left_corner.argmin(),dist_2_bottom_left_corner.shape) dist_2_bottom_right_corner = distance_on_unit_sphere(lon_br_tgt,lat_br_tgt,lon_src,lat_src) j_br_src, i_br_src = _np.unravel_index(dist_2_bottom_right_corner.argmin(),dist_2_bottom_right_corner.shape) dist_2_upper_left_corner = distance_on_unit_sphere(lon_ul_tgt,lat_ul_tgt,lon_src,lat_src) j_ul_src, i_ul_src = _np.unravel_index(dist_2_upper_left_corner.argmin(),dist_2_upper_left_corner.shape) dist_2_upper_right_corner = distance_on_unit_sphere(lon_ur_tgt,lat_ur_tgt,lon_src,lat_src) j_ur_src, i_ur_src = _np.unravel_index(dist_2_upper_right_corner.argmin(),dist_2_upper_right_corner.shape) imin = min(i_bl_src,i_ul_src) ; imax = max(i_br_src,i_ur_src) jmin = min(j_bl_src,j_br_src) ; jmax = max(j_ul_src,j_ur_src) # for safety imin = max(imin-2,0) jmin = max(jmin-2,0) imax = min(imax+2,nx) jmax = min(jmax+2,ny) print('Subset source grid : full dimension is ', nx , ny, ' subset is ', imin, imax, jmin, jmax) return imin, imax, jmin, jmax
def get_x_y_wind(lat, lon, dataset, model): indices = [] # m = Basemap(rsphere=(6371229.00, 6356752.3142), projection='merc', # llcrnrlat=40.5833284543, urcrnrlat=47.4999927992, # llcrnrlon=-129, urcrnrlon=-123.7265625) # xpt, ypt = m(lon,lat) # print "x ", xpt/100 # print "y ", ypt/100 print "lat ", lat print "lon ", lon lat_name = 'lat' lon_name = 'lon' file_lats = dataset.variables[lat_name][:] file_lons = dataset.variables[lon_name][:] file_lat = np.abs(file_lats - lat).argmin() file_lon = np.abs(file_lons - lon).argmin() print "Argmin lat", file_lat print "Argmin lon", file_lon file_lat = np.unravel_index(file_lat, file_lats.shape) file_lon = np.unravel_index(file_lon, file_lons.shape) print "Unravel lat", file_lat print "Unravel lon", file_lon file_lat_y = file_lat[0] file_lat_x = file_lat[1] print"lat y x", file_lat_y, file_lat_x file_lon_y = file_lon[0] file_lon_x = file_lon[1] print"lon y x", file_lon_y, file_lon_x print "lat ", file_lats[file_lat_y][file_lat_x] print "lon ", file_lons[file_lon_y][file_lon_x] indices.append(file_lat_y) indices.append(file_lat_x) return indices
def move(_particles, _flighttime, clean = False): ''' Двигаем частицы до первого столкновения. Возвращаем : передвинутые частицы новый массив времен время полета до столкновения кортеж пар столкнувшихся частиц ''' if clean : particles = copy.copy(_particles) flighttime = copy.copy(_flighttime) else : particles = _particles flighttime = _flighttime # Движение t = flighttime.min() for p in particles: p.r += p.v*t flighttime -= t # Ищем столкнувшиеся пары z = flighttime.flatten() < almostzero indx = np.where(np.logical_and(z.data,np.logical_not(z.mask)))[0] n = len(particles) # Оптимизация на наиболее частого случая if len(indx) == 1: return particles, flighttime, t, [np.unravel_index(indx[0], (n,n)),] pairs = [np.unravel_index(i,(n,n)) for i in indx] collidingparts = np.array(pairs).flatten() unique_collidingparts = np.unique(collidingparts) if len(collidingparts) <> len(unique_collidingparts): raise ArithmeticError, "Multibody collision detected!!!" return particles, flighttime,t,pairs
def classify_obj(self, zchi2arr1, zfit1, flags1, zchi2arr2, zfit2, flags2): flag_val = int('0b100',2) # From BOSS zwarning flag definitions for ifiber in xrange(zchi2arr1.shape[0]): self.minrchi2[ifiber,0] = n.min(zchi2arr1[ifiber]) / (self.npixflux) # Calculate reduced chi**2 values to compare templates of differing lengths if zchi2arr2 != None: self.minrchi2[ifiber,1] = n.min(zchi2arr2[ifiber]) / (self.npixflux) minpos = self.minrchi2[ifiber].argmin() # Location of best chi2 of array of best (individual template) chi2s if minpos == 0: # Means overall chi2 minimum came from template 1 self.type.append('GALAXY') self.minvector.append(zfit1.minvector[ifiber]) self.z[ifiber] = zfit1.z[ifiber] self.z_err[ifiber] = zfit1.z_err[ifiber] minloc = n.unravel_index(zchi2arr1[ifiber].argmin(), zchi2arr1[ifiber].shape)[:-1] self.zwarning = n.append(self.zwarning, flags1[ifiber]) argsort = self.minrchi2[ifiber].argsort() if len(argsort) > 1: if argsort[1] == 1: if ( n.min(zchi2arr2[ifiber]) - n.min(zchi2arr1[ifiber]) ) < zfit1.threshold: self.zwarning[ifiber] = int(self.zwarning[ifiber]) | flag_val #THIS THRESHOLD PROBABLY ISN'T RIGHT AND NEEDS TO BE CHANGED elif minpos == 1: # Means overall chi2 minimum came from template 2 self.type.append('STAR') self.minvector.append(zfit2.minvector[ifiber]) self.z[ifiber] = zfit2.z[ifiber] self.z_err[ifiber] = zfit2.z_err[ifiber] minloc = n.unravel_index(zchi2arr2[ifiber].argmin(), zchi2arr2[ifiber].shape)[:-1] self.zwarning = n.append(self.zwarning, flags2[ifiber]) argsort = self.minrchi2[ifiber].argsort() if argsort[1] == 0: if ( n.min(zchi2arr1[ifiber]) - n.min(zchi2arr2[ifiber]) ) < zfit2.threshold: self.zwarning[ifiber] = int(self.zwarning[ifiber]) | flag_val
def maxmindam(self,probsize,meanvasdam): m = probsize[0] n = probsize[1] damage = vasdam.damcalc(self,probsize,meanvasdam) maxdam = np.argmax(damage) mindam = np.argmin(damage) maxdamval = np.max(damage) mindamval = np.min(damage) dimsdam = damage.shape maxidx = np.unravel_index(maxdam,dimsdam) minidx = np.unravel_index(mindam,dimsdam) maxvasdam = np.array([maxidx[0],m+maxidx[0],maxidx[1],n+maxidx[1]]).reshape([2,2]) minvasdam = np.array([minidx[0],m+minidx[0],minidx[1],n+minidx[1]]).reshape([2,2]) location = np.array([maxvasdam,int(maxdamval),minvasdam,int(mindamval)]) return location
def nearest(array, value): """ Find nearest position in array to value. Parameters ---------- array : 'array_like' value: 'float', 'list' Returns ---------- Searches n x 1 and n x m x 1 arrays for floats. Searches n x m and n x m x p arrays for lists of size m or p. """ if isinstance(array, (list, tuple)): array = np.asarray(array) if isinstance(value, (float, int)): pos = (np.abs(array - value)).argmin() if len(array.shape) == 2: return np.unravel_index(pos, array.shape) else: return pos else: pos = (np.sum((array - value)**2, axis=1)**(1 / 2)).argmin() if len(array.shape) == 3: return np.unravel_index(pos, array.shape) else: return pos
def split_quater(img1,img2,n): iq1 = img1 iq2 = img2 if n == 1: iq1 = np.copy(iq1[0:239,0:239]) iq2 = np.copy(iq2[0:239,0:239]) elif n == 2: iq1 = np.copy(iq1[0:239,240:479]) iq2 = np.copy(iq2[0:239,240:479]) elif n == 3: iq1 = np.copy(iq1[240:479,0:239]) iq2 = np.copy(iq2[240:479,0:239]) elif n == 4: iq1 = np.copy(iq1[240:479,240:479]) iq2 = np.copy(iq2[240:479,240:479]) elif n == 5: iq1 = img1 iq2 = img2 iq1 = cv2.GaussianBlur(iq1,(5,5),0) iq2 = cv2.GaussianBlur(iq2,(5,5),0) corr_img11 = cross_image(iq1,iq1) corr_img12 = cross_image(iq1,iq2) #cv2.imwrite('corr_img11.jpg',corr_img11) #cv2.imwrite('corr_img12.jpg',corr_img22) y_self,x_self = np.unravel_index(np.argmax(corr_img11), corr_img11.shape) y,x = np.unravel_index(np.argmax(corr_img12), corr_img12.shape) y_diff = y - y_self x_diff = x - x_self print(y_diff,x_diff) return (y_diff,x_diff)
def _fixEvenKernel(kernel): """Take a kernel with even dimensions and make them odd, centered correctly. Parameters ---------- kernel : `numpy.array` a numpy.array Returns ------- out : `numpy.array` a fixed kernel numpy.array """ # Make sure the peak (close to a delta-function) is in the center! maxloc = np.unravel_index(np.argmax(kernel), kernel.shape) out = np.roll(kernel, kernel.shape[0]//2 - maxloc[0], axis=0) out = np.roll(out, out.shape[1]//2 - maxloc[1], axis=1) # Make sure it is odd-dimensioned by trimming it. if (out.shape[0] % 2) == 0: maxloc = np.unravel_index(np.argmax(out), out.shape) if out.shape[0] - maxloc[0] > maxloc[0]: out = out[:-1, :] else: out = out[1:, :] if out.shape[1] - maxloc[1] > maxloc[1]: out = out[:, :-1] else: out = out[:, 1:] return out
def reshape(self, *args, **kwargs): shape = check_shape(args, self.shape) order, copy = check_reshape_kwargs(kwargs) # Return early if reshape is not required if shape == self.shape: if copy: return self.copy() else: return self new = lil_matrix(shape, dtype=self.dtype) if order == 'C': ncols = self.shape[1] for i, row in enumerate(self.rows): for col, j in enumerate(row): new_r, new_c = np.unravel_index(i * ncols + j, shape) new[new_r, new_c] = self[i, j] elif order == 'F': nrows = self.shape[0] for i, row in enumerate(self.rows): for col, j in enumerate(row): new_r, new_c = np.unravel_index(i + j * nrows, shape, order) new[new_r, new_c] = self[i, j] else: raise ValueError("'order' must be 'C' or 'F'") return new
def calculateArea(flowDirectionGrid,flowAcc,areas,noDataValue,mask = None): # Get the sorted indices of the array in reverse order (e.g. largest first) idcs = flowAcc.argsort(axis= None) #Mask out the indexes outside of the mask if not mask is None: allRows,allCols = np.unravel_index(idcs,flowAcc.shape) gdIdcs = mask[allRows,allCols] idcs = idcs[gdIdcs] #Loop through all the indices in sorted order for idx in idcs: #Get the currents row column index [i, j] = np.unravel_index(idx, flowAcc.shape) # Get the row/column indices #Get the index of the point downstream of this and the distance to that point [downI,downJ,inBounds] = getFlowToCell(i,j,flowDirectionGrid[i,j],flowAcc.shape[0],flowAcc.shape[1]) #So long as we are not draining to the outskirts, proceed if inBounds and not flowAcc[downI,downJ] == noDataValue: #Accumulate area areas[downI,downJ] += areas[i,j] return areas
def estrampcpx(inData, estData=None): if estData is not None: if np.any(np.iscomplex(estData)): estData=estData.conj() inData=inData*estData else: estData=np.exp(-1.j*estData) inData=inData*estData indx=np.log10(abs(np.fft.fftshift(np.fft.fft2(np.angle(inData))))).argmax() #to display fft you may want to take log10 of the value here subi=np.unravel_index(indx, inData.shape) X,Y=np.meshgrid(np.r_[0:inData.shape[1]], np.r_[0:inData.shape[0]]) fxmax=inData.shape[1]/2. fymax=inData.shape[0]/2. surface=np.pi*( (subi[0]-fymax)*Y/fymax + (subi[1]-fxmax)*X/fxmax ) #print [ subi, fxmax, fymax ] outData=inData*np.exp(-1.j*surface) outVal=[(subi[0]-fymax)/fymax , (subi[1]-fxmax)/fxmax] #test if we get the direction right. indx2=np.log10(abs(np.fft.fftshift(np.fft.fft2(np.angle(outData))))).argmax() subi2=np.unravel_index(indx2, inData.shape) dist1=np.sqrt( (subi[0]-fymax)**2. + (subi[1]-fxmax)**2.) dist2=np.sqrt( (subi2[0]-fymax)**2. + (subi2[1]-fxmax)**2.) #print [dist1, dist2] if dist1<dist2: #we got the direction wrong outVal=[-(subi[0]-fymax)/fymax , -(subi[1]-fxmax)/fxmax] #outData=np.exp(-1.j*surface) return outVal
def diagnose_performance(output_parallel, output_serial, print_extradeets_or_not): error_perc = 100 * np.abs(output_serial - output_parallel) / np.abs(output_serial) error = np.abs(output_serial - output_parallel) # Relative error print('Relative error') worst_ind = np.unravel_index(np.argmax(error_perc), output_parallel.shape) print('Max error of {}% at {}'.format(error_perc[worst_ind], worst_ind)) print('Worst match: naive = {}, parallel = {}' .format(output_serial[worst_ind], output_parallel[worst_ind])) # Absolute error print('Absolute error') worst_ind = np.unravel_index(np.argmax(error), output_parallel.shape) print('Max error of {}% at {}'.format(error[worst_ind], worst_ind)) print('Worst match: naive = {}, parallel = {}' .format(output_serial[worst_ind], output_parallel[worst_ind])) correctness = np.allclose(output_serial, output_parallel, rtol=0, atol=1e-3) print('Outputs match? {}'.format(correctness)) return(correctness) if print_extradeets_or_not: print('Inputs:') print(inputs) print('Weights:') print(weights) print('Outputs (dim: {})'.format(output_parallel.shape)) print(output_parallel) print('Naive output (dim: {})'.format(output_serial.shape)) print(output_serial)
def iterkeys(self,index): #import pdb; pdb.set_trace() if not isinstance(index,tuple) and self.shape[0] == 1: index = (1,index) if isinstance(index, int): key = np.unravel_index(index-1, self.shape, order='F') yield tuple(k+1 for k in key) elif isinstance(index,slice): index = range((index.start or 1)-1, index.stop or np.prod(self.shape), index.step or 1) for key in np.transpose(np.unravel_index(index, self.shape, order='F')): # 0-based yield tuple(k+1 for k in key) elif isinstance(index,(list,np.ndarray)): index = np.asarray(index)-1 for key in np.transpose(np.unravel_index(index, self.shape, order='F')): yield tuple(k+1 for k in key) else: assert isinstance(index,tuple),index.__class__ indices = [] # 1-based for i,ix in enumerate(index): if isinstance(ix,slice): indices.append(np.arange((ix.start or 1), (ix.stop or self.shape[i]) + 1, ix.step or 1, dtype=int)) else: indices.append(np.asarray(ix)) assert len(index) == 2 indices[0].shape = (-1,1) for key in np.broadcast(*indices): yield tuple(map(int,key))
def testPredict(self): #Create a set of indices lmbda = 0.0 iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10) matrixIterator = iter(self.matrixList) ZList = iterativeSoftImpute.learnModel(matrixIterator) XhatList = iterativeSoftImpute.predict(ZList, self.indsList) #Check we get the exact matrices returned for i, Xhat in enumerate(XhatList): nptst.assert_array_almost_equal(numpy.array(Xhat.todense()), self.matrixList[i].todense()) self.assertEquals(Xhat.nnz, self.indsList[i].shape[0]) self.assertAlmostEquals(MCEvaluator.meanSqError(Xhat, self.matrixList[i]), 0) self.assertAlmostEquals(MCEvaluator.rootMeanSqError(Xhat, self.matrixList[i]), 0) #Try moderate lambda lmbda = 0.1 iterativeSoftImpute = IterativeSoftImpute(lmbda, k=10) matrixIterator = iter(self.matrixList) ZList = list(iterativeSoftImpute.learnModel(matrixIterator)) XhatList = iterativeSoftImpute.predict(iter(ZList), self.indsList) for i, Xhat in enumerate(XhatList): for ind in self.indsList[i]: U, s, V = ZList[i] Z = (U*s).dot(V.T) self.assertEquals(Xhat[numpy.unravel_index(ind, Xhat.shape)], Z[numpy.unravel_index(ind, Xhat.shape)]) self.assertEquals(Xhat.nnz, self.indsList[i].shape[0])
def grid_maximum(matrix): """ Find where in grid highest probability lies """ arr = np.copy(matrix) # 1st maximum i, j = np.unravel_index(arr.argmax(), arr.shape) lon_bins = np.linspace(llcrnrlon, urcrnrlon+2, xBins) lat_bins = np.linspace(llcrnrlat, urcrnrlat+1, xBins*xyRatio) lon_corr = lon_bins[1]-lon_bins[0] lat_corr = lat_bins[1]-lat_bins[0] #for lon in lon_bins: # for lat in lat_bins: # coord = lat+lat_corr*0.5, lon+lon_corr*0.5 # print rg.get(coord)['admin1'] maximum = lat_bins[i]+lat_corr*0.5, lon_bins[j]+lon_corr*0.5 # 2nd maximum arr[i, j] = 0 i, j = np.unravel_index(arr.argmax(), arr.shape) second_maximum = lat_bins[i]+lat_corr*0.5, lon_bins[j]+lon_corr*0.5 # 3rd maximum arr[i, j] = 0 i, j = np.unravel_index(arr.argmax(), arr.shape) third_maximum = lat_bins[i]+lat_corr*0.5, lon_bins[j]+lon_corr*0.5 return maximum, second_maximum, third_maximum
def write_cv_results_toFile(self, scores, precision_scores, recall_scores, param_grid, result_path): final_results_dict = {} all_scores_dict = {'error' : scores, 'precision' : precision_scores, 'recall' : recall_scores} for score in all_scores_dict: avg_score = np.mean(all_scores_dict[score], axis=1) std_score = np.std(all_scores_dict[score], axis=1) if score == 'error': opt_ind = np.unravel_index(np.argmin(avg_score), avg_score.shape) else: opt_ind = np.unravel_index(np.argmax(avg_score), avg_score.shape) final_results_dict[score] = avg_score[opt_ind] final_results_dict[score + '_std'] = std_score[opt_ind] params_dict = {} for element in self.grid_dictionary: params_dict[element] = param_grid[opt_ind[0]][self.grid_dictionary[element]] final_results_dict[score + '_params'] = params_dict n_estimators = opt_ind[1] + 1 final_results_dict[score + '_params'].update(dict(n_trees = str(n_estimators))) if not 'fe_params' in params_dict: final_results_dict[score + '_params'].update(dict(fe_params = None)) final_results_dict['channel_type'] = self.config.channel_type json.dump(final_results_dict, open(result_path,'w'))
def reorder_in_x(self,xmat): xfactors=self.xfactors tdim=self.xmatrix.shape ndim=len(tdim) mf=xfactors.shape[0] xmatredim=np.zeros(mf,dtype=int) for i in range(mf): xmatredim[i]=np.prod(xfactors[i,:]) xmatre=np.zeros(xmatredim) pdim=np.prod(tdim) i_in=np.arange(pdim) i_inx=np.array(np.unravel_index(i_in,tdim)) i_outx=[None]*ndim for i in range(ndim): i_outx[i]=np.array(np.unravel_index(i_inx[i],xfactors[:,i])) i_outx=np.array(i_outx) i_out=[None]*mf for i in range(mf): i_out[i]=np.array(np.ravel_multi_index(i_outx[:,i],xfactors[i])) i_out=np.array(i_out) xmatre[tuple(i_out)]=self.xmatrix[tuple(i_inx)] return(xmatre)
def invert_reorder_in_x(self,xmatre,xfactors): (mf,nf)=xfactors.shape ## tdim2=xmatre.shape ## ndim2=len(tdim2) ndim=nf tdim=np.zeros(nf,dtype=int) for i in range(nf): tdim[i]=np.prod(xfactors[:,i]) xmatrix=np.zeros(tdim) pdim=np.prod(tdim) i_in=np.arange(pdim) i_inx=np.array(np.unravel_index(i_in,tdim)) i_outx=[None]*ndim for i in range(ndim): i_outx[i]=np.array(np.unravel_index(i_inx[i],xfactors[:,i])) i_outx=np.array(i_outx) i_out=[None]*mf for i in range(mf): i_out[i]=np.array(np.ravel_multi_index(i_outx[:,i],xfactors[i])) i_out=np.array(i_out) xmatrix[tuple(i_inx)]=xmatre[tuple(i_out)] xmatrix=xmatrix.astype(np.uint8) return(xmatrix)
def klst(X): history=[] add_vec =[] print("データ行列の大きさ ", X.shape) count = X.shape[0] dist = calc(X) print(dist.shape) exit() i,j = np.unravel_index(dist.argmax(), dist.shape) max = dist[i,j] ind=[x for x in range(0,count)] while dist.shape[0] > 1: dist = pd.DataFrame(dist, index = ind ,columns = ind) p =slm(dist.values) print("最短距離ベクトル", p) # print(dist) dist, add_vec = cut_n_comb(dist, p, add_vec) ind= dist.index dist = dist.values i,j = np.unravel_index(dist.argmax(), dist.shape) max = dist[i,j] if max == 0:#の例外処理が必要 break print("合成結果:", add_vec) count -= 1 print(count-1, "残り処理回数") print("-----------------------------")
def seuillage_80(masque,image,handle=False): """ Permet de retirer 20 pourcent des pixels les plus clairs. """ nb_pixel=sum(sum(masque)) seuil=nb_pixel*20/100 image=humuscle(image) if handle: print('nombre de pixel avant seuillage') print(nb_pixel) print('nombre de pixel a supprimer') print(seuil) if handle: imshow('masque initial',masque) k=0 while(k<seuil): masque[np.unravel_index(np.argmax(image),image.shape)]=0 image[np.unravel_index(np.argmax(image),image.shape)]=image.min() k+=1 if handle: imshow('masque apres seuillage a 80%',masque) print('nombre de pixel apres seuillage') print(sum(sum(masque))) return masque
def nonMaximumSuppression(I, dims, pos, c, maxima): # if pos== true, find local maximum, else - find local minimum #copy image I_temp = I w = dims[0] h = dims[1] n = 5 margin = 5 tau = 50 #try np here too for i in range(n + margin, w - n - margin): # print "1 entered 1st cycle i=", i for j in range(n + margin, h - n - margin): window = I_temp[i:i+n, j:j+n] if np.sum(window) < 0: mval = 0 elif pos: mval = np.amax(window) mcoords = np.unravel_index(np.argmax(window), window.shape) + np.array((i,j)) else: mval = np.amin(window) mcoords = np.unravel_index(np.argmax(window), window.shape) + np.array((i,j)) I_temp [i:i+n, j:j+n] = -1 print mcoords, "mval= ", mval if pos: if mval >= tau: maxima.append(maximum(mcoords[0],mcoords[1], mval, c)) else: if mval <= tau and mval > 0: maxima.append(maximum(mcoords[0],mcoords[1], mval, c))
def plot_drain_pit(self, pit, drain, prop, s, elev, area): from matplotlib import pyplot cmap = 'Greens' ipit, jpit = np.unravel_index(pit, elev.shape) Idrain, Jdrain = np.unravel_index(drain, elev.shape) Iarea, Jarea = np.unravel_index(area, elev.shape) imin = max(0, min(ipit, Idrain.min(), Iarea.min())-1) imax = min(elev.shape[0], max(ipit, Idrain.max(), Iarea.max()) + 2) jmin = max(0, min(jpit, Jdrain.min(), Jarea.min())-1) jmax = min(elev.shape[1], max(jpit, Jdrain.max(), Jarea.max()) + 2) roi = (slice(imin, imax), slice(jmin, jmax)) pyplot.figure() ax = pyplot.subplot(221); pyplot.axis('off') im = pyplot.imshow(elev[roi], interpolation='none'); im.set_cmap(cmap) pyplot.plot(jpit-jmin, ipit-imin, lw=0, color='k', marker='$P$', ms=10) pyplot.plot(Jarea-jmin, Iarea-imin, 'k.') pyplot.plot(Jdrain-jmin, Idrain-imin, lw=0, color='k', marker='$D$', ms=10) pyplot.subplot(223, sharex=ax, sharey=ax); pyplot.axis('off') im = pyplot.imshow(elev[roi], interpolation='none'); im.set_cmap(cmap) for i, j, val in zip(Idrain, Jdrain, prop): pyplot.plot(j-jmin, i-imin, lw=0, color='k', marker='$%.2f$' % val, ms=16) pyplot.subplot(224, sharex=ax, sharey=ax); pyplot.axis('off') im = pyplot.imshow(elev[roi], interpolation='none'); im.set_cmap(cmap) for i, j, val in zip(Idrain, Jdrain, s): pyplot.plot(j-jmin, i-imin, lw=0, color='k', marker='$%.2f$' % val, ms=16) pyplot.tight_layout()
def from_hdu(cls, hdu, hdu_bands=None): """Make a WcsNDMap object from a FITS HDU. Parameters ---------- hdu : `~astropy.io.fits.BinTableHDU` or `~astropy.io.fits.ImageHDU` The map FITS HDU. hdu_bands : `~astropy.io.fits.BinTableHDU` The BANDS table HDU. """ geom = WcsGeom.from_header(hdu.header, hdu_bands) shape = tuple([ax.nbin for ax in geom.axes]) shape_wcs = tuple([np.max(geom.npix[0]), np.max(geom.npix[1])]) meta = cls._get_meta_from_header(hdu.header) map_out = cls(geom, meta=meta) # TODO: Should we support extracting slices? if isinstance(hdu, fits.BinTableHDU): pix = hdu.data.field('PIX') pix = np.unravel_index(pix, shape_wcs[::-1]) vals = hdu.data.field('VALUE') if 'CHANNEL' in hdu.data.columns.names and shape: chan = hdu.data.field('CHANNEL') chan = np.unravel_index(chan, shape[::-1]) idx = chan + pix else: idx = pix map_out.set_by_idx(idx[::-1], vals) else: map_out.data = hdu.data return map_out
def forward(self, forest, features, num_obj, labels=None, boxes_for_nms=None, batch_size=0): # generate dropout if self.dropout > 0.0: tree_dropout_mask = get_dropout_mask(self.dropout, self.hidden_size / 2) chain_dropout_mask = get_dropout_mask(self.dropout, self.hidden_size / 2) else: tree_dropout_mask = None chain_dropout_mask = None # generate tree lstm input/output class tree_out_h = None tree_out_dists = None tree_out_commitments = None tree_h_order = Variable(torch.LongTensor(num_obj).zero_().cuda()) tree_order_idx = 0 tree_lstm_io = tree_utils.TreeLSTM_IO(tree_out_h, tree_h_order, tree_order_idx, tree_out_dists, tree_out_commitments, tree_dropout_mask) #chain_out_h = None #chain_out_dists = None #chain_out_commitments = None #chain_h_order = Variable(torch.LongTensor(num_obj).zero_().cuda()) #chain_order_idx = 0 #chain_lstm_io = tree_utils.TreeLSTM_IO(chain_out_h, chain_h_order, chain_order_idx, chain_out_dists, # chain_out_commitments, chain_dropout_mask) for idx in range(len(forest)): self.decoderTreeLSTM(forest[idx], features, tree_lstm_io) #self.decoderChainLSTM(forest[idx], features, chain_lstm_io) out_tree_h = torch.index_select(tree_lstm_io.hidden, 0, tree_lstm_io.order.long()) out_dists = torch.index_select(tree_lstm_io.dists, 0, tree_lstm_io.order.long())[:-batch_size] out_commitments = torch.index_select( tree_lstm_io.commitments, 0, tree_lstm_io.order.long())[:-batch_size] # Do NMS here as a post-processing step if boxes_for_nms is not None and not self.training: is_overlap = nms_overlaps(boxes_for_nms.data).view( boxes_for_nms.size(0), boxes_for_nms.size(0), boxes_for_nms.size(1)).cpu().numpy() >= self.nms_thresh # is_overlap[np.arange(boxes_for_nms.size(0)), np.arange(boxes_for_nms.size(0))] = False out_dists_sampled = F.softmax(out_dists, 1).data.cpu().numpy() out_dists_sampled[:, 0] = 0 out_commitments = out_commitments.data.new( out_commitments.shape[0]).fill_(0) for i in range(out_commitments.size(0)): box_ind, cls_ind = np.unravel_index(out_dists_sampled.argmax(), out_dists_sampled.shape) out_commitments[int(box_ind)] = int(cls_ind) out_dists_sampled[is_overlap[box_ind, :, cls_ind], cls_ind] = 0.0 out_dists_sampled[ box_ind] = -1.0 # This way we won't re-sample out_commitments = Variable(out_commitments.view(-1)) else: out_commitments = out_commitments.view(-1) if self.training and (labels is not None): out_commitments = labels.clone() else: out_commitments = torch.cat( (out_commitments, Variable( torch.randn(batch_size).long().fill_(0).cuda()).view(-1)), 0) return out_dists, out_commitments
def semi_automatic_training(): """ This appplication lets you maneuver through windows selected by the OW process. In short: OW below the OW_start threshold, which passes the R2_cirterion are considered eddies, have a look at https://github.com/JASaa/eddies-R2 for more info about the sample selection process""" # Loop through every netcdf file in directory, usually they are spaced by 5 days for fName in os.listdir(args.fDir): if not fName.endswith(".nc"): continue logger.info("loading netcdf") # load data (ds,t,lon,lat,depth,uvel_full,vvel_full,sst_full,ssl_full) = load_netcdf4(args.fDir + fName) # Confidence level, usually 90% R2_criterion = 0.90 # OW value at which to begin the evaluation of R2, default was -1, want to use -8 to be absolutely sure OW_start = -6.0 # Number of local minima to evaluate using R2 method. # Set low (like 20) to see a few R2 eddies quickly. # Set high (like 1e5) to find all eddies in domain. max_evaluation_points = 100000 # Minimum number of cells required to be identified as an eddie. min_eddie_cells = 3 # set to 3 to be coherent with the use of the R2 method, 3 points seems like a reasonable minimun for a correlation # z-level to plot. Usually set to 0 for the surface. k_plot = 0 dlon = abs(lon[0]-lon[1]) dlat = abs(lat[0]-lat[1]) # Create eddy images for each day in datase #for day, time in enumerate(t): # Shuffle the time so that the expert won't see the same long-lasting eddies for day in random.sample(range(0, len(t)), len(t)): dateStr = "{:%d-%m-%Y}".format(datetime.date(1950, 1, 1) + datetime.timedelta(hours=float(t[day])) ) logger.info(f"Creating images for dataset {dateStr}") # create a text trap text_trap = io.StringIO() sys.stdout = text_trap # Run the OW-R2 algorithm lon,lat,u,v,vorticity,OW,OW_eddies,eddie_census,nEddies,circulation_mask = eddy_detection( lon,lat,depth,uvel_full,vvel_full,day,R2_criterion,OW_start,max_evaluation_points,min_eddie_cells) # restore stdout sys.stdout = sys.__stdout__ sst_train = [] ssl_train = [] uvel_train = [] vvel_train = [] phase_train = [] nDataset = 5 # ========================================================= # ============== Prepare datasets and lists =============== # ========================================================= eddyCtrIdx = [] for i in range(0,nEddies): lonIdx = np.argmax(lon>eddie_census[2,i])-1 latIdx = np.argmax(lat>eddie_census[3,i])-1 eddyCtrIdx.append( (lonIdx, latIdx) ) # Netcdf uses (lat,lon) we want to use (lon,lat) and discard the depth sst = sst_full[day,:,:].T ssl = ssl_full[day,:,:].T uvel = uvel_full[day,0,:,:].T vvel = vvel_full[day,0,:,:].T # Calculate the phase angle (direction) of the current with np.errstate(all='ignore'): # Disable zero div warning phase = xr.ufuncs.rad2deg( xr.ufuncs.arctan2(vvel, uvel) ) + 180 OW = OW[:,:,0] nLon = len(lon) nLat = len(lat) datasets = (sst, ssl, uvel, vvel, phase, OW) # ========================================================= # ======= Create rectangular patches around eddies ======== # ========================================================= logger.info(f"+ Creating rectangles for {nEddies} eddies") savedImgCounter = 0 # saved image counter for file ID for eddyId, ctrIdx in enumerate(eddyCtrIdx): # nEddies ctrCoord = lon[ctrIdx[0]], lat[ctrIdx[1]] diameter_km = eddie_census[5][eddyId] bfs_diameter_km, bfs_center = eddy_metrics(OW_eddies, ctrIdx, lon, lat) # Positive rotation (counter-clockwise) is a cyclone in the northern hemisphere because of the coriolis effect if (eddie_census[1][eddyId] > 0.0): cyclone = 1 # 1 is a cyclone, 0 is nothing and -1 is anti-cyclone (negative rotation) else: cyclone = -1 logger.info(f"+++ Creating rectangles for {check_cyclone(cyclone)} with center {ctrCoord} and diameter {diameter_km}") # Find rectangle metrics height = args.size * abs(diameter_km / 110.54) # 1 deg = 110.54 km, 1.2 to be sure the image covers the eddy width = args.size * abs(diameter_km / (111.320 * cos(lat[ctrIdx[1]]))) # 1 deg = 111.320*cos(latitude) km, using center latitude as ref lon_bnds = ctrCoord[0]-width/2.0, ctrCoord[0]+width/2.0 lat_bnds = ctrCoord[1]-height/2.0, ctrCoord[1]+height/2.0 # Indeces of current eddy image lonIdxs = np.where((lon >= lon_bnds[0]) & (lon <= lon_bnds[1]))[0] latIdxs = np.where((lat >= lat_bnds[0]) & (lat <= lat_bnds[1]))[0] eddy_data = np.array([np.zeros((lonIdxs.size,latIdxs.size)) for _ in range(6)]) # Plot and flag to save eddy #add = plot_grids(eddy_data, lo, la, title) #-------- Move closer to center of eddy ------------ title = dateStr + "_" + check_cyclone(cyclone) choices = ('Center', 'incLon', 'incLat', 'decLon', 'decLat') response = 'Center' #response = 'Yes' # Skip this section for debugging non-eddy section while response in choices: lo = lon[lonIdxs] la = lat[latIdxs] for i, loIdx in enumerate(lonIdxs): for j, laIdx in enumerate(latIdxs): for k, measurement in enumerate(datasets): # for every measurement type in datasets eddy_data[k,i,j] = measurement[loIdx,laIdx] # Store a larger grid to make it easier to see if we have an eddy and if we should center image if (lonIdxs[0]-5 < 0 or lonIdxs[-1]+5 >= nLon) or (latIdxs[0]-3 < 0 or latIdxs[-1]+3 >= nLat): larger_grid = None else: larger_grid = [ np.zeros(lonIdxs.size+10), np.zeros(latIdxs.size+6), np.zeros((lonIdxs.size+10,latIdxs.size+6)), ] for i, loIdx in enumerate(range(lonIdxs[0]-5, lonIdxs[-1]+6)): for j, laIdx in enumerate(range(latIdxs[0]-3, latIdxs[-1]+4)): larger_grid[0][i] = lon[loIdx] larger_grid[1][j] = lat[laIdx] larger_grid[2][i,j] = ssl[loIdx,laIdx] response = plot_grids(eddy_data, lo, la, larger_grid, title) if response not in choices: # TODO: feel like this is a silly way of doing this break if response == 'Center': # Find the center from water level logger.info(f"+++ Centering eddy towards a minima/maxima depending on eddy type") if cyclone==1: idx = np.unravel_index(eddy_data[1].argmax(), eddy_data[1].shape) ctrCoord = lon[lonIdxs[idx[0]]], lat[latIdxs[idx[1]]] logger.info(f"+++ Argmax center -> lon: {ctrCoord[0]}, Center lat: {ctrCoord[1]}") else: idx = np.unravel_index(eddy_data[1].argmin(), eddy_data[1].shape) ctrCoord = lon[lonIdxs[idx[0]]], lat[latIdxs[idx[1]]] logger.info(f"+++ Argmin center -> lon: {ctrCoord[0]}, Center lat: {ctrCoord[1]}") # New width and height in case we've moved in lon/lat direction width, height = abs(lo[0]-lo[-1])+dlon, abs(la[0]-la[-1])+dlat lon_bnds = ctrCoord[0]-width/2.0, ctrCoord[0]+width/2.0 lat_bnds = ctrCoord[1]-height/2.0, ctrCoord[1]+height/2.0 # Indeces of current eddy image lonIdxs = np.where((lon >= lon_bnds[0]) & (lon <= lon_bnds[1]))[0] latIdxs = np.where((lat >= lat_bnds[0]) & (lat <= lat_bnds[1]))[0] elif response == 'incLon': if (lonIdxs[0] <= 0 or lonIdxs[-1] >= nLon-1): logger.info(f"+++ Longitude can't be increased further") else: lonIdxs = np.arange(lonIdxs[0]-1, lonIdxs[-1]+2) logger.info(f"+++ Increasing lontitude by 1 cell in both directions to ({lonIdxs[0]}:{lonIdxs[-1]})") elif response == 'incLat': if (latIdxs[0] <= 0 or latIdxs[-1] >= nLat-1): logger.info(f"+++ Latitude can't be increased further") else: latIdxs = np.arange(latIdxs[0]-1, latIdxs[-1]+2) logger.info(f"+++ Increasing latitude by 1 cell in both directions to ({latIdxs[0]}:{latIdxs[-1]})") elif response == 'decLon': lonIdxs = np.arange(lonIdxs[0]+1, lonIdxs[-1]) logger.info(f"+++ Decreasing lontitude by 1 cell in both directions to ({lonIdxs[0]}:{lonIdxs[-1]})") elif response == 'decLat': latIdxs = np.arange(latIdxs[0]+1, latIdxs[-1]) logger.info(f"+++ Decreasing latitude by 1 cell in both directions to ({latIdxs[0]}:{latIdxs[-1]})") eddy_data = np.array([np.zeros((lonIdxs.size,latIdxs.size)) for _ in range(6)]) #---------------------------------------------------------- lo = lon[lonIdxs] la = lat[latIdxs] #guiEvent, guiValues = show_figure(fig) #add = 'Yes' # Bypass GUI selection if response=='Yes': savedImgCounter = savedImgCounter + 1 # Create images? ''' dirPath = 'C:/Master/TTK-4900-Master/images/'+dateStr+'/' if not os.path.exists(dirPath): os.makedirs(dirPath) imPath = dirPath + title + f"_{savedImgCounter}.png" plt.savefig(imPath, bbox_inches='tight') ''' sst_train.append([eddy_data[0], cyclone]) # [data, label] ssl_train.append([eddy_data[1], cyclone]) uvel_train.append([eddy_data[2], cyclone]) vvel_train.append([eddy_data[3], cyclone]) phase_train.append([eddy_data[4], cyclone]) logger.info(f"+++++ Saving image {eddyId} as an eddy") else: logger.info(f"+++++ Discarding image {eddyId}") # ========================================================= # ================ Select non-eddy images ================= # ========================================================= if savedImgCounter <= 0: logger.info(f"+++++ No eddies found") continue # Subgrid (sg) longitude and latitude length sgLon, sgLat = dim.find_avg_dim(sst_train, start_axis=0) logger.info(f"+++++ Using average dimensions ({sgLon}, {sgLat}) for non-eddy") loRange, laRange = range(0, nLon, sgLon), range(0, nLat, sgLat) # Create OW array of compatible dimensions for comparing masks OW_noeddy = OW[:loRange[-1],:laRange[-1]] OW_noeddy = create_subgrids( np.ma.masked_where(OW_noeddy < -0.8, OW_noeddy), sgLon, sgLat, 1 ) # Get a 2d grid of indeces -> make it moldable to the average grid -> convert to subgrids idx_subgrids = create_subgrids( np.array( index_list(nLon, nLat) )[:loRange[-1],:laRange[-1]], sgLon, sgLat, 2 ) noneddy_idx_subgrids = [] for i, grid in enumerate(OW_noeddy): if not np.ma.is_masked(grid): noneddy_idx_subgrids.append(idx_subgrids[i]) nNoneddies = len(noneddy_idx_subgrids) data_noeddy = np.array([[np.zeros((sgLon,sgLat)) for _ in range(nNoneddies)] for _ in range(6)]) # Shuffle the noneddies and loop thorugh untill we have chosen the same amount of non-eddies as eddies random.shuffle(noneddy_idx_subgrids) added = 0 for grid_id, idx_grid in enumerate(noneddy_idx_subgrids): OW_ = np.zeros((idx_grid.shape[:2])) for i in range(len(idx_grid)): for j in range(len(idx_grid[0])): idx = idx_grid[i,j][0], idx_grid[i,j][1] for k in range(len(data_noeddy)): data_noeddy[k,grid_id,i,j] = datasets[k][idx] #print(idx_grid) lo, la = lon[idx_grid[:,0,0]], lat[idx_grid[0,:,1]] title = dateStr + "_noneddy" add = plot_grids(data_noeddy[:,grid_id,:,:], lo, la, None, title) if add=='Yes': added = added + 1 sst_train.append([data_noeddy[0,grid_id,:,:], 0]) # [data, label] ssl_train.append([data_noeddy[0,grid_id,:,:], 0]) uvel_train.append([data_noeddy[0,grid_id,:,:], 0]) vvel_train.append([data_noeddy[0,grid_id,:,:], 0]) phase_train.append([data_noeddy[0,grid_id,:,:], 0]) logger.info(f"+++++ Saving noneddy") if added >= savedImgCounter: break # ========================================================= # ============== Interpolate ============== # ========================================================= #sst_out = np.array(sst_train) #ssl_out = np.array(ssl_train) #uvel_out = np.array(uvel_train) #vvel_out = np.array(vvel_train) #phase_out = np.array(phase_train) #nTeddies = sst_out.shape[0] logger.info(f"Compressing and storing training data so far") # ========================================================= # ========== Save data as compressed numpy array ========== # ========================================================= save_npz_array( (sst_train, ssl_train, uvel_train, vvel_train, phase_train) )
def search(tile): if os.path.exists('rogue-%s-02.png' % tile) and not os.path.exists('rogue-%s-03.png' % tile): print 'Skipping', tile return fn = os.path.join(tile[:3], tile, 'unwise-%s-w2-%%s-m.fits' % tile) try: II = [fitsio.read(os.path.join('e%i' % e, fn % 'img')) for e in [1, 2]] PP = [fitsio.read(os.path.join('e%i' % e, fn % 'std')) for e in [1, 2]] wcs = Tan(os.path.join('e%i' % 1, fn % 'img')) except: import traceback print print 'Failed to read data for tile', tile traceback.print_exc() print return H, W = II[0].shape ps = PlotSequence('rogue-%s' % tile) aa = dict(interpolation='nearest', origin='lower') ima = dict(interpolation='nearest', origin='lower', vmin=-100, vmax=500) plt.clf() plt.imshow(II[0], **ima) plt.title('Epoch 1') ps.savefig() plt.clf() plt.imshow(II[1], **ima) plt.title('Epoch 2') ps.savefig() # X = gaussian_filter(np.abs((II[0] - II[1]) / np.hypot(PP[0], PP[1])), 1.0) # plt.clf() # plt.imshow(X, interpolation='nearest', origin='lower') # plt.title('Blurred abs difference / per-pixel-std') # ps.savefig() # Y = (II[0] - II[1]) / reduce(np.hypot, [PP[0], PP[1], np.hypot(100,II[0]), np.hypot(100,II[1]) ]) Y = (II[0] - II[1]) / reduce(np.hypot, [PP[0], PP[1]]) X = gaussian_filter(np.abs(Y), 1.0) xthresh = 3. print 'Value at rogue:', X[1452, 1596] print 'pp at rogue:', [pp[1452, 1596] for pp in PP] plt.clf() plt.imshow(X, interpolation='nearest', origin='lower') plt.title('X') ps.savefig() # plt.clf() # plt.hist(np.minimum(100, PP[0].ravel()), 100, range=(0,100), # histtype='step', color='r') # plt.hist(np.minimum(100, PP[1].ravel()), 100, range=(0,100), # histtype='step', color='b') # plt.title('Per-pixel std') # ps.savefig() #Y = ((II[0] - II[1]) / np.hypot(PP[0], PP[1])) #Y = gaussian_filter( # (II[0] - II[1]) / np.hypot(100, np.hypot(II[0], II[1])) # , 1.0) #I = np.argsort(-X.ravel()) #yy,xx = np.unravel_index(I[:25], X.shape) #print 'xx', xx #print 'yy', yy hot = (X > xthresh) peak = find_peaks(hot, X) dilate = 2 hot = binary_dilation(hot, structure=np.ones((3, 3)), iterations=dilate) blobs, nblobs = label(hot, np.ones((3, 3), int)) blobslices = find_objects(blobs) # Find maximum pixel within each blob. BX, BY = [], [] BV = [] for b, slc in enumerate(blobslices): sy, sx = slc y0, y1 = sy.start, sy.stop x0, x1 = sx.start, sx.stop bl = blobs[slc] i = np.argmax((bl == (b + 1)) * X[slc]) iy, ix = np.unravel_index(i, dims=bl.shape) by = iy + y0 bx = ix + x0 BX.append(bx) BY.append(by) BV.append(X[by, bx]) BX = np.array(BX) BY = np.array(BY) BV = np.array(BV) I = np.argsort(-BV) xx, yy = BX[I], BY[I] keep = [] S = 15 for i, (x, y) in enumerate(zip(xx, yy)): #print x,y if x < S or y < S or x + S >= W or y + S >= H: continue slc = slice(y - S, y + S + 1), slice(x - S, x + S + 1) slc2 = slice(y - 3, y + 3 + 1), slice(x - 3, x + 3 + 1) mx = np.max((II[0][slc] + II[1][slc]) / 2.) #print 'Max within slice:', mx #if mx > 5e3: if mx > 2e3: continue mx2 = np.max((II[0][slc2] + II[1][slc2]) / 2.) print 'Flux near object:', mx2 if mx2 < 250: continue #miny = np.min(Y[slc2]) #maxy = np.max(Y[slc2]) keep.append(i) keep = np.array(keep) if len(keep) == 0: print 'No objects passed cuts' return xx = xx[keep] yy = yy[keep] plt.clf() plt.imshow(X, interpolation='nearest', origin='lower', cmap='gray') plt.title('X') ax = plt.axis() plt.plot(xx, yy, 'r+') plt.plot(1596, 1452, 'o', mec=(0, 1, 0), mfc='none') plt.axis(ax) ps.savefig() ylo, yhi = [], [] for i in range(min(len(xx), 100)): x, y = xx[i], yy[i] slc2 = slice(y - 3, y + 3 + 1), slice(x - 3, x + 3 + 1) ylo.append(np.min(Y[slc2])) yhi.append(np.max(Y[slc2])) plt.clf() plt.plot(ylo, yhi, 'r.') plt.axis('scaled') ps.savefig() for i, (x, y) in enumerate(zip(xx, yy)[:50]): print x, y rows, cols = 2, 3 ra, dec = wcs.pixelxy2radec(x + 1, y + 1) slc = slice(y - S, y + S + 1), slice(x - S, x + S + 1) slc2 = slice(y - 3, y + 3 + 1), slice(x - 3, x + 3 + 1) mx = max(np.max(II[0][slc]), np.max(II[1][slc])) print 'Max within slice:', mx miny = np.min(Y[slc2]) maxy = np.max(Y[slc2]) plt.clf() plt.subplot(rows, cols, 1) plt.imshow(II[0][slc], **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('epoch 1') plt.subplot(rows, cols, 2) plt.imshow(II[1][slc], **ima) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('epoch 2') plt.subplot(rows, cols, 3) plt.imshow(PP[0][slc], **aa) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('std 1') plt.subplot(rows, cols, 6) plt.imshow(PP[1][slc], **aa) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('std 2') plt.subplot(rows, cols, 4) plt.imshow(X[slc], **aa) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('X') plt.subplot(rows, cols, 5) plt.imshow(Y[slc], **aa) plt.xticks([]) plt.yticks([]) plt.colorbar() plt.title('Y') #plt.suptitle('Tile %s, Flux: %4.0f, Range: %.2g %.2g' % (tile,mx,miny,maxy)) plt.suptitle('Tile %s, RA,Dec (%.4f, %.4f)' % (tile, ra, dec)) ps.savefig()
def giveShipOrders(ship, currentOrders, collectingStop): # build ship status global GLOBAL_DEPO global GLOBAL_DEPO_BUILD_OK global SAVE_UP_FOR_DEPO global DEPO_ONE_SHIP_AT_A_TIME global FIRST_DEPO_BUILT global WAIT_TO_BUILD_DEPOT global ATTACK_CURRENT_HALITE global ATTACK_TARGET_HALITE global BUILD_DEPO_TIMER global DEPO_MIN_SHIPS moveFlag = False turns_left = (constants.MAX_TURNS - game.turn_number) #logging.info("Ship {} was {}".format(ship, currentOrders)) # enemy locations, look if they are next to you runFlag = False attackFlag = False surroundings = game_map.get_normalized_cardinals(ship.position) #logging.info("Enemy ship halite \n {}".format(game_map.enemyShipHalite)) # check to run shipX = ship.position.x shipY = ship.position.y distanceRule = 1 dist = game_map.distanceMatrixNonZero[shipX][shipY].copy() dist[dist > distanceRule] = 0 enemyInSight = dist * game_map.shipFlag #logging.info("ship {} dist {} enemy in sight {}".format(ship.id, dist, enemyInSight)) # is an enemy in zone if np.sum(enemyInSight) > 0: enemyHalite = game_map.enemyShipHalite * dist enemyMA = np.ma.masked_equal(enemyHalite, 0, copy=False) if len(game.players) == 2: fightHalite = dist * (game_map.enemyShipHalite + game_map.shipFlag * game_map.npMap * (0.25 + 0.5 * game_map.negInspirationBonus)) else: fightHalite = dist * 1 # check if we should run if enemyMA.min() < 300 and \ ship.halite_amount > 700 and \ len(game.players) == 2: logging.info("ship {} runs!!!".format(ship.id)) runFlag = True elif enemyMA.min() < 500 and \ ship.halite_amount>700 and \ len(game.players) == 4 and \ turns_left < 50: runFlag = True elif len(game.players) == 2 and \ (ship.halite_amount + game_map.npMap[shipY,shipX] *(0.25 + 0.5 * game_map.inspirationBonus[shipY,shipX]))>500 and \ game_map.friendlyShipCount[shipY,shipX] <= game_map.enemyShipCount[shipY,shipX] and \ enemyMA.min() < 500: logging.info("ship {} needs to move!".format(ship.id)) moveFlag = np.unravel_index(enemyMA.argmin(), enemyMA.shape) # check if we should fight elif np.max(fightHalite) > ship.halite_amount and \ len(game.players)==2 and \ game_map.friendlyShipCount[shipY,shipX] > game_map.enemyShipCount[shipY,shipX]: logging.info("ship {} attacks!!!".format(ship.id)) attackFlag = True elif np.max(fightHalite) > 750 and \ ship.halite_amount < 200 and \ game_map.friendlyShipCount[shipY,shipX] == game_map.enemyShipCount[shipY,shipX]: attackFlag = True okToBuildDepo = False # we wait if we just built a depo if FIRST_DEPO_BUILT == False: okToBuildDepo = True elif WAIT_TO_BUILD_DEPOT < 1: okToBuildDepo = True #logging.info("ship {} in max zone {}".format(ship.id, game_map.dropCalc.inMaxZone(ship.position))) #logging.info("ship {} friendly count {}".format(ship.id, game_map.returnFriendlyCount(ship.position, 7))) status = None if currentOrders is None: #new ship status = "exploring" elif currentOrders == 'build depo' and BUILD_DEPO_TIMER < 45: status = 'build depo' BUILD_DEPO_TIMER += 1 elif GLOBAL_DEPO < MAX_DEPO and \ min(GLOBAL_DEPO+1,2) * 11 < game.me.get_ship_count() and \ game.turn_number > shipBuildingTurns and \ game_map.dropCalc.inMaxZone(ship.position) and \ min([game_map.calculate_distance(ship.position, i) for i in me.get_all_drop_locations()]) >= DEPO_DISTANCE-6 and \ GLOBAL_DEPO_BUILD_OK == True and \ ship.position not in game.return_all_drop_locations() and \ DEPO_ONE_SHIP_AT_A_TIME == False and\ okToBuildDepo == True and \ turns_left > 75 and \ game_map.returnFriendlyCount(ship.position, 7) > DEPO_MIN_SHIPS: status = 'build depo' SAVE_UP_FOR_DEPO = True DEPO_ONE_SHIP_AT_A_TIME = True elif ship.halite_amount < game_map[ship.position].halite_amount * 0.1: status = 'mining' elif min([ game_map.calculate_distance(ship.position, i) for i in me.get_all_drop_locations() ]) >= turns_left - SUICIDE_TURN_FLAG: status = "returnSuicide" elif currentOrders == "returning": status = "returning" if ship.position == me.shipyard.position or ship.position in me.get_dropoff_locations( ): status = "exploring" elif ship.halite_amount >= returnHaliteFlag or runFlag == True: status = "returning" #elif ship.halite_amount < game_map[ship.position].halite_amount * 0.1 or game_map[ship.position].halite_amount > collectingStop: # status = 'mining' #create attack squad near end #elif (ship.halite_amount < 50 and game_map.averageHalite < 50 and game_map.width < 48 and len(game.players)==2) or attackFlag == True: elif attackFlag == True: status = 'attack' elif currentOrders == "exploring": status = "exploring" else: status = 'exploring' #logging.info("ship {} status is {}".format(ship.id, status)) return status, moveFlag
def get_best_parameters(w0, w1, losses): """Get the best w from the result of grid search.""" min_row, min_col = np.unravel_index(np.argmin(losses), losses.shape) return losses[min_row, min_col], w0[min_row], w1[min_col]
def stack_registration(s, align_to_this_slice=0, refinement='spike_interpolation', register_in_place=True, fourier_cutoff_radius=None, debug=False): """Calculate shifts which would register the slices of a three-dimensional stack `s`, and optionally register the stack in-place. Axis 0 is the "z-axis", axis 1 is the "up-down" (Y) axis, and axis 2 is the "left-right" (X) axis. For each XY slice, we calculate the shift in the XY plane which would line that slice up with the slice specified by `align_to_this_slice`. If `align_to_this_slice` is a number, it indicates which slice of `s` to use as the reference slice. If `align_to_this_slice` is a numpy array, it is used as the reference slice, and must be the same shape as a 2D slice of `s`. `refinement` is one of `integer`, `spike_interpolation`, or `phase_fitting`, in order of increasing precision/slowness. I don't yet have any evidence that my implementation of phase fitting gives any improvement over (faster, simpler) spike interpolation, so caveat emptor. `register_in_place`: If `True`, modify the input stack `s` by shifting its slices to line up with the reference slice. `fourier_cutoff_radius`: Ignore the Fourier phases of spatial frequencies higher than this cutoff, since they're probably lousy due to aliasing and noise anyway. If `None`, attempt to estimate a resonable cutoff. """ assert len(s.shape) == 3 try: assert align_to_this_slice in range(s.shape[0]) align_to_this_slice = s[align_to_this_slice, :, :] except ValueError: align_to_this_slice = np.squeeze(align_to_this_slice) assert align_to_this_slice.shape == s.shape[-2:] assert refinement in ('integer', 'spike_interpolation', 'phase_fitting') if refinement == 'phase_fitting' and minimize is None: raise UserWarning("Failed to import scipy minimize; no phase fitting.") assert register_in_place in (True, False) assert debug in (True, False) if fourier_cutoff_radius is None: fourier_cutoff_radius = estimate_fourier_cutoff_radius(s, debug) assert (0 < fourier_cutoff_radius <= 0.5) if debug and np_tif is None: raise UserWarning("Failed to import np_tif; no debug mode.") ## Multiply each slice of the stack by an XY mask that goes to zero ## at the edges, to prevent periodic boundary artifacts when we ## Fourier transform. mask_ud = np.sin(np.linspace(0, np.pi, s.shape[1])).reshape(s.shape[1], 1) mask_lr = np.sin(np.linspace(0, np.pi, s.shape[2])).reshape(1, s.shape[2]) masked_reference_slice = align_to_this_slice * mask_ud * mask_lr ## We'll base our registration on the phase of the low spatial ## frequencies of the cross-power spectrum. We'll need the complex ## conjugate of the Fourier transform of the masked reference slice, ## and a mask in the Fourier domain to pick out the low spatial ## frequencies: ref_slice_ft_conj = np.conj(np.fft.rfftn(masked_reference_slice)) k_ud = np.fft.fftfreq(s.shape[1]).reshape(ref_slice_ft_conj.shape[0], 1) k_lr = np.fft.rfftfreq(s.shape[2]).reshape(1, ref_slice_ft_conj.shape[1]) fourier_mask = (k_ud**2 + k_lr**2) < (fourier_cutoff_radius)**2 ## Now we'll loop over each slice of the stack, calculate our ## registration shifts, and optionally apply the shifts to the ## original stack. registration_shifts = [] if debug: ## Save some intermediate data to help with debugging masked_stack = np.zeros_like(s) masked_stack_ft = np.zeros((s.shape[0], ) + ref_slice_ft_conj.shape, dtype=np.complex128) masked_stack_ft_vs_ref = np.zeros_like(masked_stack_ft) cross_power_spectra = np.zeros_like(masked_stack_ft) spikes = np.zeros(s.shape, dtype=np.float64) for which_slice in range(s.shape[0]): if debug: print("Calculating registration for slice", which_slice) ## Compute the cross-power spectrum of our slice, and mask out ## the high spatial frequencies. current_slice = s[which_slice, :, :] * mask_ud * mask_lr current_slice_ft = np.fft.rfftn(current_slice) cross_power_spectrum = current_slice_ft * ref_slice_ft_conj cross_power_spectrum = (fourier_mask * cross_power_spectrum / np.abs(cross_power_spectrum)) ## Inverse transform to get a 'spike' in real space. The ## location of this spike gives the desired registration shift. ## Start by locating the spike to the nearest integer: spike = np.fft.irfftn(cross_power_spectrum, s=current_slice.shape) loc = np.array(np.unravel_index(np.argmax(spike), spike.shape)) if refinement in ('spike_interpolation', 'phase_fitting'): ## Use (very simple) three-point polynomial interpolation to ## refine the location of the peak of the spike: neighbors = np.array([-1, 0, 1]) ud_vals = spike[(loc[0] + neighbors) % spike.shape[0], loc[1]] lr_vals = spike[loc[0], (loc[1] + neighbors) % spike.shape[1]] lr_fit = np.poly1d(np.polyfit(neighbors, lr_vals, deg=2)) ud_fit = np.poly1d(np.polyfit(neighbors, ud_vals, deg=2)) lr_max_shift = -lr_fit[1] / (2 * lr_fit[2]) ud_max_shift = -ud_fit[1] / (2 * ud_fit[2]) loc = loc + (ud_max_shift, lr_max_shift) ## Convert our shift into a signed number near zero: loc = ((np.array(spike.shape) // 2 + loc) % np.array(spike.shape) - np.array(spike.shape) // 2) if refinement == 'phase_fitting': if debug: print("Phase fitting slice", which_slice, "...") ## (Attempt to) further refine our registration shift by ## fitting Fourier phases. I'm not sure this does any good, ## perhaps my implementation is lousy? def minimize_me(loc, cross_power_spectrum): disagreement = np.abs( expected_cross_power_spectrum(loc, k_ud, k_lr) - cross_power_spectrum)[fourier_mask].sum() if debug: print(" Shift:", loc, "Disagreement:", disagreement) return disagreement loc = minimize(minimize_me, x0=loc, args=(cross_power_spectrum, ), method='Nelder-Mead').x registration_shifts.append(loc) if debug: ## Save some intermediate data to help with debugging masked_stack[which_slice, :, :] = current_slice masked_stack_ft[which_slice, :, :] = (np.fft.fftshift( current_slice_ft, axes=0)) masked_stack_ft_vs_ref[which_slice, :, :] = (np.fft.fftshift( current_slice_ft * ref_slice_ft_conj, axes=0)) cross_power_spectra[which_slice, :, :] = (np.fft.fftshift( cross_power_spectrum, axes=0)) spikes[which_slice, :, :] = np.fft.fftshift(spike) if register_in_place: ## Modify the input stack in-place so it's registered. if refinement == 'integer': registration_type = 'nearest_integer' else: registration_type = 'fourier_interpolation' apply_registration_shifts(s, registration_shifts, registration_type=registration_type) if debug: np_tif.array_to_tif(masked_stack, 'DEBUG_masked_stack.tif') np_tif.array_to_tif(np.log(np.abs(masked_stack_ft)), 'DEBUG_masked_stack_FT_log_magnitudes.tif') np_tif.array_to_tif(np.angle(masked_stack_ft), 'DEBUG_masked_stack_FT_phases.tif') np_tif.array_to_tif(np.angle(masked_stack_ft_vs_ref), 'DEBUG_masked_stack_FT_phase_vs_ref.tif') np_tif.array_to_tif(np.angle(cross_power_spectra), 'DEBUG_cross_power_spectral_phases.tif') np_tif.array_to_tif(spikes, 'DEBUG_spikes.tif') if register_in_place: np_tif.array_to_tif(s, 'DEBUG_registered_stack.tif') return np.array(registration_shifts)
def binned_statistic_dd(sample, values, statistic='mean', bins=10, range=None, expand_binnumbers=False): """ Compute a multidimensional binned statistic for a set of data. This is a generalization of a histogramdd function. A histogram divides the space into bins, and returns the count of the number of points in each bin. This function allows the computation of the sum, mean, median, or other statistic of the values within each bin. Parameters ---------- sample : array_like Data to histogram passed as a sequence of D arrays of length N, or as an (N,D) array. values : (N,) array_like or list of (N,) array_like The data on which the statistic will be computed. This must be the same shape as `x`, or a list of sequences - each with the same shape as `x`. If `values` is such a list, the statistic will be computed on each independently. statistic : string or callable, optional The statistic to compute (default is 'mean'). The following statistics are available: * 'mean' : compute the mean of values for points within each bin. Empty bins will be represented by NaN. * 'median' : compute the median of values for points within each bin. Empty bins will be represented by NaN. * 'count' : compute the count of points within each bin. This is identical to an unweighted histogram. `values` array is not referenced. * 'sum' : compute the sum of values for points within each bin. This is identical to a weighted histogram. * 'min' : compute the minimum of values for points within each bin. Empty bins will be represented by NaN. * 'max' : compute the maximum of values for point within each bin. Empty bins will be represented by NaN. * function : a user-defined function which takes a 1D array of values, and outputs a single numerical statistic. This function will be called on the values in each bin. Empty bins will be represented by function([]), or NaN if this returns an error. bins : sequence or int, optional The bin specification must be in one of the following forms: * A sequence of arrays describing the bin edges along each dimension. * The number of bins for each dimension (nx, ny, ... = bins). * The number of bins for all dimensions (nx = ny = ... = bins). range : sequence, optional A sequence of lower and upper bin edges to be used if the edges are not given explicitly in `bins`. Defaults to the minimum and maximum values along each dimension. expand_binnumbers : bool, optional 'False' (default): the returned `binnumber` is a shape (N,) array of linearized bin indices. 'True': the returned `binnumber` is 'unraveled' into a shape (D,N) ndarray, where each row gives the bin numbers in the corresponding dimension. See the `binnumber` returned value, and the `Examples` section of `binned_statistic_2d`. .. versionadded:: 0.17.0 Returns ------- statistic : ndarray, shape(nx1, nx2, nx3,...) The values of the selected statistic in each two-dimensional bin. bin_edges : list of ndarrays A list of D arrays describing the (nxi + 1) bin edges for each dimension. binnumber : (N,) array of ints or (D,N) ndarray of ints This assigns to each element of `sample` an integer that represents the bin in which this observation falls. The representation depends on the `expand_binnumbers` argument. See `Notes` for details. See Also -------- numpy.digitize, numpy.histogramdd, binned_statistic, binned_statistic_2d Notes ----- Binedges: All but the last (righthand-most) bin is half-open in each dimension. In other words, if `bins` is ``[1, 2, 3, 4]``, then the first bin is ``[1, 2)`` (including 1, but excluding 2) and the second ``[2, 3)``. The last bin, however, is ``[3, 4]``, which *includes* 4. `binnumber`: This returned argument assigns to each element of `sample` an integer that represents the bin in which it belongs. The representation depends on the `expand_binnumbers` argument. If 'False' (default): The returned `binnumber` is a shape (N,) array of linearized indices mapping each element of `sample` to its corresponding bin (using row-major ordering). If 'True': The returned `binnumber` is a shape (D,N) ndarray where each row indicates bin placements for each dimension respectively. In each dimension, a binnumber of `i` means the corresponding value is between (bin_edges[D][i-1], bin_edges[D][i]), for each dimension 'D'. .. versionadded:: 0.11.0 """ known_stats = ['mean', 'median', 'count', 'sum', 'std','min','max'] if not callable(statistic) and statistic not in known_stats: raise ValueError('invalid statistic %r' % (statistic,)) # `Ndim` is the number of dimensions (e.g. `2` for `binned_statistic_2d`) # `Dlen` is the length of elements along each dimension. # This code is based on np.histogramdd try: # `sample` is an ND-array. Dlen, Ndim = sample.shape except (AttributeError, ValueError): # `sample` is a sequence of 1D arrays. sample = np.atleast_2d(sample).T Dlen, Ndim = sample.shape # Store initial shape of `values` to preserve it in the output values = np.asarray(values) input_shape = list(values.shape) # Make sure that `values` is 2D to iterate over rows values = np.atleast_2d(values) Vdim, Vlen = values.shape # Make sure `values` match `sample` if(statistic != 'count' and Vlen != Dlen): raise AttributeError('The number of `values` elements must match the ' 'length of each `sample` dimension.') nbin = np.empty(Ndim, int) # Number of bins in each dimension edges = Ndim * [None] # Bin edges for each dim (will be 2D array) dedges = Ndim * [None] # Spacing between edges (will be 2D array) try: M = len(bins) if M != Ndim: raise AttributeError('The dimension of bins must be equal ' 'to the dimension of the sample x.') except TypeError: bins = Ndim * [bins] # Select range for each dimension # Used only if number of bins is given. if range is None: smin = np.atleast_1d(np.array(sample.min(axis=0), float)) smax = np.atleast_1d(np.array(sample.max(axis=0), float)) else: smin = np.zeros(Ndim) smax = np.zeros(Ndim) for i in xrange(Ndim): smin[i], smax[i] = range[i] # Make sure the bins have a finite width. for i in xrange(len(smin)): if smin[i] == smax[i]: smin[i] = smin[i] - .5 smax[i] = smax[i] + .5 # Create edge arrays for i in xrange(Ndim): if np.isscalar(bins[i]): nbin[i] = bins[i] + 2 # +2 for outlier bins edges[i] = np.linspace(smin[i], smax[i], nbin[i] - 1) else: edges[i] = np.asarray(bins[i], float) nbin[i] = len(edges[i]) + 1 # +1 for outlier bins dedges[i] = np.diff(edges[i]) nbin = np.asarray(nbin) # Compute the bin number each sample falls into, in each dimension sampBin = [ np.digitize(sample[:, i], edges[i]) for i in xrange(Ndim) ] # Using `digitize`, values that fall on an edge are put in the right bin. # For the rightmost bin, we want values equal to the right # edge to be counted in the last bin, and not as an outlier. for i in xrange(Ndim): # Find the rounding precision decimal = int(-np.log10(dedges[i].min())) + 6 # Find which points are on the rightmost edge. on_edge = np.where(np.around(sample[:, i], decimal) == np.around(edges[i][-1], decimal))[0] # Shift these points one bin to the left. sampBin[i][on_edge] -= 1 # Compute the sample indices in the flattened statistic matrix. binnumbers = np.ravel_multi_index(sampBin, nbin) result = np.empty([Vdim, nbin.prod()], float) if statistic == 'mean': result.fill(np.nan) flatcount = np.bincount(binnumbers, None) a = flatcount.nonzero() for vv in xrange(Vdim): flatsum = np.bincount(binnumbers, values[vv]) result[vv, a] = flatsum[a] / flatcount[a] elif statistic == 'std': result.fill(0) flatcount = np.bincount(binnumbers, None) a = flatcount.nonzero() for vv in xrange(Vdim): flatsum = np.bincount(binnumbers, values[vv]) flatsum2 = np.bincount(binnumbers, values[vv] ** 2) result[vv, a] = np.sqrt(flatsum2[a] / flatcount[a] - (flatsum[a] / flatcount[a]) ** 2) elif statistic == 'count': result.fill(0) flatcount = np.bincount(binnumbers, None) a = np.arange(len(flatcount)) result[:, a] = flatcount[np.newaxis, :] elif statistic == 'sum': result.fill(0) for vv in xrange(Vdim): flatsum = np.bincount(binnumbers, values[vv]) a = np.arange(len(flatsum)) result[vv, a] = flatsum elif statistic == 'median': result.fill(np.nan) for i in np.unique(binnumbers): for vv in xrange(Vdim): result[vv, i] = np.median(values[vv, binnumbers == i]) elif statistic == 'min': result.fill(np.nan) for i in np.unique(binnumbers): for vv in xrange(Vdim): result[vv, i] = np.min(values[vv, binnumbers == i]) elif statistic == 'max': result.fill(np.nan) for i in np.unique(binnumbers): for vv in xrange(Vdim): result[vv, i] = np.max(values[vv, binnumbers == i]) elif callable(statistic): with np.errstate(invalid='ignore'), suppress_warnings() as sup: sup.filter(RuntimeWarning) try: null = statistic([]) except: null = np.nan result.fill(null) for i in np.unique(binnumbers): for vv in xrange(Vdim): result[vv, i] = statistic(values[vv, binnumbers == i]) # Shape into a proper matrix result = result.reshape(np.append(Vdim, nbin)) # Remove outliers (indices 0 and -1 for each bin-dimension). core = [slice(None)] + Ndim * [slice(1, -1)] result = result[core] # Unravel binnumbers into an ndarray, each row the bins for each dimension if(expand_binnumbers and Ndim > 1): binnumbers = np.asarray(np.unravel_index(binnumbers, nbin)) if np.any(result.shape[1:] != nbin - 2): raise RuntimeError('Internal Shape Error') # Reshape to have output (`reulst`) match input (`values`) shape result = result.reshape(input_shape[:-1] + list(nbin-2)) return BinnedStatisticddResult(result, edges, binnumbers)
def nd_argmax(array): return np.unravel_index(np.argmax(array.flatten()), array.shape)
def _laser_algo(self, params): time = params[0] dist_meas = params[1] if not dist_meas or dist_meas < self._laser_min_range or dist_meas > self._laser_max_range: raise PositioningException( 'invalid laser distance measurement: %s' % dist_meas) d1_v, d1_q, d2_v, d2_q, sc_v, sc_q = self._parse_poses(params, offset=2) q = SystemModel.sc2gl_q.conj() * sc_q.conj() d1, d2 = self.asteroids ast = d2 if self._target_d2 else d1 ast_v = d2_v if self._target_d2 else d1_v ast_q = d2_q if self._target_d2 else d1_q rel_rot_q = q * ast_q * ast.ast2sc_q.conj() rel_pos_v = tools.q_times_v(q, ast_v - sc_v) * 1000 max_r = ast.max_radius max_diam = 2 * max_r / 1000 # set orthographic projection self._onboard_renderer.set_orth_frustum(max_diam, max_diam, -max_diam / 2, max_diam / 2) # render orthographic depth image _, zz = self._onboard_renderer.render(self._target_obj_idx, [0, 0, 0], rel_rot_q, [1, 0, 0], get_depth=True, shadows=False, textures=False) # restore regular perspective projection self._onboard_renderer.set_frustum(self._sm.cam.x_fov, self._sm.cam.y_fov, self._sm.min_altitude * .1, self._sm.max_distance) zz[zz > max_diam / 2 * 0.999] = float('nan') zz = zz * 1000 - rel_pos_v[2] xx, yy = np.meshgrid( np.linspace(-max_r, max_r, self._sm.view_width) - rel_pos_v[0], np.linspace(-max_r, max_r, self._sm.view_height) - rel_pos_v[1]) x_expected = np.clip( (rel_pos_v[0] + max_r) / max_r / 2 * self._sm.view_width + 0.5, 0, self._sm.view_width - 1.001) y_expected = np.clip( (rel_pos_v[1] + max_r) / max_r / 2 * self._sm.view_height + 0.5, 0, self._sm.view_height - 1.001) dist_expected = tools.interp2(zz, x_expected, y_expected, discard_bg=True) # mse cost function balances between adjusted location and measurement error adj_dist_sqr = (zz - dist_meas)**2 + xx**2 + yy**2 cost = self._laser_adj_loc_weight * adj_dist_sqr \ + (self._laser_meas_err_weight - self._laser_adj_loc_weight) * (zz - dist_meas)**2 j, i = np.unravel_index(np.nanargmin(cost), cost.shape) if np.isnan(zz[j, i]): raise PositioningException( 'laser algo results in off asteroid pointing') if math.sqrt(adj_dist_sqr[j, i]) >= self._laser_max_adj_dist: raise PositioningException( 'laser algo solution too far (%.0fm, limit=%.0fm), spurious measurement assumed' % (math.sqrt(adj_dist_sqr[j, i]), self._laser_max_adj_dist)) dx, dy, dz = xx[0, i], yy[j, 0], zz[j, i] - dist_meas if self._result_frame == ApiServer.FRAME_GLOBAL: # return global ast-sc vector est_sc_ast_v = ast_v * 1000 - tools.q_times_v( q.conj(), rel_pos_v + np.array([dx, dy, dz])) else: # return local sc-ast vector est_sc_ast_v = tools.q_times_v(SystemModel.sc2gl_q, rel_pos_v + np.array([dx, dy, dz])) dist_expected = float( dist_expected) if not np.isnan(dist_expected) else -1.0 return json.dumps([list(est_sc_ast_v), dist_expected])
def calc_beta_oris_from_boundary_misori( grain: ebsd.Grain, neighbour_network: nx.Graph, quat_array: np.ndarray, alpha_phase_id: int, burg_tol: float = 5 ) -> Tuple[List[List[Quat]], List[float], List[Quat]]: """Calculate the possible beta orientations for pairs of alpha and neighbour orientations using the misorientation relation to neighbour orientations. Parameters ---------- grain The grain currently being reconstructed neighbour_network A neighbour network mapping grain boundary connectivity quat_array Array of quaternions, representing the orientations of the pixels of the EBSD map burg_tol : The threshold misorientation angle to determine neighbour relations Returns ------- list of lists of defdap.Quat.quat Possible beta orientations, grouped by each neighbour. Any neighbour with deviation greater than the tolerance is excluded. list of float Deviations from perfect Burgers transformation list of Quat Alpha orientations """ # This needed to move further up calculation process unq_cub_sym_comps = Quat.extract_quat_comps(unq_cub_syms) beta_oris = [] beta_devs = [] alpha_oris = [] neighbour_grains = neighbour_network.neighbors(grain) neighbour_grains = [ grain for grain in neighbour_grains if grain.phaseID == alpha_phase_id ] for neighbour_grain in neighbour_grains: bseg = neighbour_network[grain][neighbour_grain]['boundary'] # check sense of bseg if grain is bseg.grain1: ipoint = 0 else: ipoint = 1 for boundary_point_pair in bseg.boundaryPointPairsX: point = boundary_point_pair[ipoint] alpha_ori = quat_array[point[1], point[0]] point = boundary_point_pair[ipoint - 1] neighbour_ori = quat_array[point[1], point[0]] min_misoris, min_cub_sym_idxs = calc_misori_of_variants( alpha_ori.conjugate, neighbour_ori, unq_cub_sym_comps) # find the hex symmetries (i, j) from give the minimum # deviation from the burgers relation for the minimum store: # the deviation, the hex symmetries (i, j) and the cubic # symmetry if the deviation is over a threshold then set # cubic symmetry to -1 min_misori_idx = np.unravel_index(np.argmin(min_misoris), min_misoris.shape) burg_dev = min_misoris[min_misori_idx] if burg_dev < burg_tol / 180 * np.pi: beta_oris.append( beta_oris_from_cub_sym(alpha_ori, min_cub_sym_idxs[min_misori_idx], int(min_misori_idx[0]))) beta_devs.append(burg_dev) alpha_oris.append(alpha_ori) return beta_oris, beta_devs, alpha_oris
def wind_diesel_hybrid( energy_per_hh, # kWh/household/year as defined wind_speed, # annual average wind speed wind_curve, tier, start_year, end_year, wind_no=15, # number of wind panel sizes simulated diesel_no=15, # number of diesel generators simulated discount_rate=0.08, diesel_range=[0.7] ): n_chg = 0.92 # charge efficiency of battery n_dis = 0.92 # discharge efficiency of battery lpsp_max = 0.10 # maximum loss of load allowed over the year, in share of kWh battery_cost = 139 # battery capital capital cost, USD/kWh of storage capacity wind_cost = 2800 # Wind turbine capital cost, USD/kW peak power diesel_cost = 261 # diesel generator capital cost, USD/kW rated power wind_life = 20 # wind panel expected lifetime, years diesel_life = 10 # diesel generator expected lifetime, years wind_om = 0.015 # annual OM cost of wind panels diesel_om = 0.1 # annual OM cost of diesel generator k_t = 0.005 # temperature factor of wind panels inverter_cost = 80 inverter_life = 10 inverter_efficiency = 0.92 charge_controller = 142 wind_curve = wind_curve * wind_speed / np.average(wind_curve) hour_numbers = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23) * 365 dod_max = 0.8 # maximum depth of discharge of battery def load_curve(tier, energy_per_hh): # the values below define the load curve for the five tiers. The values reflect the share of the daily demand # expected in each hour of the day (sum of all values for one tier = 1) tier5_load_curve = [0.021008403, 0.021008403, 0.021008403, 0.021008403, 0.027310924, 0.037815126, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.042016807, 0.046218487, 0.050420168, 0.067226891, 0.084033613, 0.073529412, 0.052521008, 0.033613445, 0.023109244] tier4_load_curve = [0.017167382, 0.017167382, 0.017167382, 0.017167382, 0.025751073, 0.038626609, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.042918455, 0.0472103, 0.051502146, 0.068669528, 0.08583691, 0.075107296, 0.053648069, 0.034334764, 0.021459227] tier3_load_curve = [0.013297872, 0.013297872, 0.013297872, 0.013297872, 0.019060284, 0.034574468, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.044326241, 0.048758865, 0.053191489, 0.070921986, 0.088652482, 0.077570922, 0.055407801, 0.035460993, 0.019946809] tier2_load_curve = [0.010224949, 0.010224949, 0.010224949, 0.010224949, 0.019427403, 0.034764826, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.040899796, 0.04601227, 0.056237219, 0.081799591, 0.102249489, 0.089468303, 0.06390593, 0.038343558, 0.017893661] tier1_load_curve = [0, 0, 0, 0, 0.012578616, 0.031446541, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.037735849, 0.044025157, 0.062893082, 0.100628931, 0.125786164, 0.110062893, 0.078616352, 0.044025157, 0.012578616] if tier == 1: load_curve = tier1_load_curve * 365 elif tier == 2: load_curve = tier2_load_curve * 365 elif tier == 3: load_curve = tier3_load_curve * 365 elif tier == 4: load_curve = tier4_load_curve * 365 else: load_curve = tier5_load_curve * 365 return np.array(load_curve) * energy_per_hh / 365 load_curve = load_curve(tier, energy_per_hh) def wind_diesel_capacities(wind_capacity, battery_size, diesel_capacity, wind_no, diesel_no, battery_no, wind_curve): dod = np.zeros(shape=(24, battery_no, wind_no, diesel_no)) battery_use = np.zeros(shape=(24, battery_no, wind_no, diesel_no)) # Stores the amount of battery discharge during the day fuel_result = np.zeros(shape=(battery_no, wind_no, diesel_no)) battery_life = np.zeros(shape=(battery_no, wind_no, diesel_no)) soc = np.ones(shape=(battery_no, wind_no, diesel_no)) * 0.5 unmet_demand = np.zeros(shape=(battery_no, wind_no, diesel_no)) excess_gen = np.zeros(shape=(battery_no, wind_no, diesel_no)) # TODO annual_diesel_gen = np.zeros(shape=(battery_no, wind_no, diesel_no)) dod_max = np.ones(shape=(battery_no, wind_no, diesel_no)) * 0.6 p_rated = 600 es = 0.85 # losses in wind electricity u_arr = range(1, 26) p_curve = [0, 0, 0, 0, 30, 77, 135, 208, 287, 371, 450, 514, 558, 582, 594, 598, 600, 600, 600, 600, 600, 600, 600, 600, 600] wind_power = np.zeros(8760) wind_curve = np.round(wind_curve) for i in range(len(p_curve)): # wind_power = np.where(wind_curve == i, p_curve[i], wind_power) wind_curve = np.where(wind_curve == i, p_curve[i], wind_curve) # wind_power = wind_power[:, 0] wind_power = wind_curve for i in range(8760): # Battery self-discharge (0.02% per hour) battery_use[hour_numbers[i], :, :] = 0.0002 * soc soc *= 0.9998 # Calculation of wind gen and net load wind_gen = wind_power[i] * wind_capacity / p_rated net_load = load_curve[hour_numbers[i]] - wind_gen # remaining load not met by wind panels # Dispatchable energy from battery available to meet load battery_dispatchable = soc * battery_size * n_dis # Energy required to fully charge battery battery_chargeable = (1 - soc) * battery_size / n_chg # Below is the dispatch strategy for the diesel generator as described in word document if 4 < hour_numbers[i] <= 17: # During the morning and day, the batteries are dispatched primarily. # The diesel generator, if needed, is run at the lowest possible capacity # Minimum diesel capacity to cover the net load after batteries. # Diesel genrator limited by lowest possible capacity (40%) and rated capacity min_diesel = np.minimum( np.maximum(net_load - battery_dispatchable, 0.4 * diesel_capacity), diesel_capacity) diesel_gen = np.where(net_load > battery_dispatchable, min_diesel, 0) elif 17 > hour_numbers[i] > 23: # During the evening, the diesel generator is dispatched primarily, at max_diesel. # Batteries are dispatched if diesel generation is insufficient. # Maximum amount of diesel needed to supply load and charge battery # Diesel genrator limited by lowest possible capacity (40%) and rated capacity max_diesel = np.maximum( np.minimum(net_load + battery_chargeable, diesel_capacity), 0.4 * diesel_capacity) diesel_gen = np.where(net_load > 0, max_diesel, 0) else: # During night, batteries are dispatched primarily. # The diesel generator is used at max_diesel if load is larger than battery capacity # Maximum amount of diesel needed to supply load and charge battery # Diesel genrator limited by lowest possible capacity (40%) and rated capacity max_diesel = np.maximum( np.minimum(net_load + battery_chargeable, diesel_capacity), 0.4 * diesel_capacity) diesel_gen = np.where(net_load > battery_dispatchable, max_diesel, 0) fuel_result += np.where(diesel_gen > 0, diesel_capacity * 0.08145 + diesel_gen * 0.246, 0) annual_diesel_gen += diesel_gen # Reamining load after diesel generator net_load = net_load - diesel_gen # If diesel generation is larger than load, battery is charged # If diesel generation is smaller than load, battery is discharged soc -= np.where(net_load > 0, net_load / n_dis / battery_size, net_load * n_chg / battery_size) # The amount of battery discharge in the hour is stored (measured in State Of Charge) battery_use[hour_numbers[i], :, :] = \ np.minimum(np.where(net_load > 0, net_load / n_dis / battery_size, 0), soc) # If State of charge is negative, that means there's demand that could not be met. unmet_demand += np.where(soc < 0, -soc / n_dis * battery_size, 0) soc = np.maximum(soc, 0) # If State of Charge is larger than 1, that means there was excess wind/diesel generation excess_gen += np.where(soc > 1, (soc - 1) / n_chg * battery_size, 0) # TODO soc = np.minimum(soc, 1) dod[hour_numbers[i], :, :] = 1 - soc # The depth of discharge in every hour of the day is stored if hour_numbers[i] == 23: # The battery wear during the last day is calculated battery_used = np.where(dod.max(axis=0) > 0, 1, 0) battery_life += battery_use.sum(axis=0) / ( 531.52764 * np.maximum(0.1, dod.max(axis=0) * dod_max) ** -1.12297) * battery_used condition = unmet_demand / energy_per_hh # LPSP is calculated excess_gen = excess_gen / energy_per_hh battery_life = np.round(1 / battery_life) diesel_share = annual_diesel_gen / energy_per_hh return diesel_share, battery_life, condition, fuel_result, excess_gen # This section creates the range of wind capacities, diesel capacities and battery sizes to be simulated ref = 5 * load_curve[19] battery_sizes = [0.5 * energy_per_hh / 365, energy_per_hh / 365, 2 * energy_per_hh / 365] wind_caps = [] diesel_caps = [] diesel_extend = np.ones(wind_no) wind_extend = np.ones(diesel_no) for i in range(wind_no): wind_caps.append(ref * (wind_no - i) / wind_no) for j in range(diesel_no): diesel_caps.append(j * max(load_curve) / diesel_no) wind_caps = np.outer(np.array(wind_caps), wind_extend) diesel_caps = np.outer(diesel_extend, np.array(diesel_caps)) # This section creates 2d-arrays to store information on wind capacities, diesel capacities, battery sizes, # fuel usage, battery life and LPSP battery_size = np.ones((len(battery_sizes), wind_no, diesel_no)) wind_panel_size = np.zeros((len(battery_sizes), wind_no, diesel_no)) diesel_capacity = np.zeros((len(battery_sizes), wind_no, diesel_no)) for j in range(len(battery_sizes)): battery_size[j, :, :] *= battery_sizes[j] wind_panel_size[j, :, :] = wind_caps diesel_capacity[j, :, :] = diesel_caps # For the number of diesel, wind and battery capacities the lpsp, battery lifetime, fuel usage and LPSP is calculated diesel_share, battery_life, lpsp, fuel_usage, excess_gen = \ wind_diesel_capacities(wind_panel_size, battery_size, diesel_capacity, wind_no, diesel_no, len(battery_sizes), wind_curve) battery_life = np.minimum(20, battery_life) def calculate_hybrid_lcoe(diesel_price): # Necessary information for calculation of LCOE is defined project_life = end_year - start_year generation = np.ones(project_life) * energy_per_hh generation[0] = 0 # Calculate LCOE sum_costs = np.zeros((len(battery_sizes), wind_no, diesel_no)) sum_el_gen = np.zeros((len(battery_sizes), wind_no, diesel_no)) investment = np.zeros((len(battery_sizes), wind_no, diesel_no)) for year in range(project_life + 1): salvage = np.zeros((len(battery_sizes), wind_no, diesel_no)) fuel_costs = fuel_usage * diesel_price om_costs = (wind_panel_size * wind_cost * wind_om + diesel_capacity * diesel_cost * diesel_om) inverter_investment = np.where(year % inverter_life == 0, max(load_curve) * inverter_cost, 0) diesel_investment = np.where(year % diesel_life == 0, diesel_capacity * diesel_cost, 0) wind_investment = np.where(year % wind_life == 0, wind_panel_size * wind_cost, 0) battery_investment = np.where(year % battery_life == 0, battery_size * battery_cost / dod_max, 0) # TODO Include dod_max here? if year == project_life: salvage = (1 - (project_life % battery_life) / battery_life) * battery_cost * battery_size / dod_max + \ (1 - (project_life % diesel_life) / diesel_life) * diesel_capacity * diesel_cost + \ (1 - (project_life % wind_life) / wind_life) * wind_panel_size * wind_cost + \ (1 - (project_life % inverter_life) / inverter_life) * max(load_curve) * inverter_cost investment += diesel_investment + wind_investment + battery_investment + inverter_investment - salvage sum_costs += (fuel_costs + om_costs + battery_investment + diesel_investment + wind_investment - salvage) / ((1 + discount_rate) ** year) if year > 0: sum_el_gen += energy_per_hh / ((1 + discount_rate) ** year) return sum_costs / sum_el_gen, investment diesel_limit = 0.5 min_lcoe_range = [] investment_range = [] capacity_range = [] ren_share_range = [] for d in diesel_range: lcoe, investment = calculate_hybrid_lcoe(d) lcoe = np.where(lpsp > lpsp_max, 99, lcoe) lcoe = np.where(diesel_share > diesel_limit, 99, lcoe) min_lcoe = np.min(lcoe) min_lcoe_combination = np.unravel_index(np.argmin(lcoe, axis=None), lcoe.shape) ren_share = 1 - diesel_share[min_lcoe_combination] capacity = wind_panel_size[min_lcoe_combination] + diesel_capacity[min_lcoe_combination] ren_capacity = wind_panel_size[min_lcoe_combination] / capacity # excess_gen = excess_gen[min_lcoe_combination] min_lcoe_range.append(min_lcoe) investment_range.append(investment[min_lcoe_combination]) capacity_range.append(capacity) ren_share_range.append(ren_share) return min_lcoe_range, investment_range, capacity_range #, ren_share_range # , ren_capacity, excess_gen #wind_diesel_hybrid(1, 5, wind_curve, 1, 2018, 2030, diesel_price=0.3)
def main() -> None: st.title("Лабораторная работа №2") st.markdown("<h3 style='margin-bottom: 50px'>" "{task}" "</h3>".format(task=task), unsafe_allow_html=True) seller, customer = st.beta_columns(2) seller.text("Вопросы о продавце") customer.text("Вопросы о покупателе") seller_answers, customer_answers = [], [] for question in seller_questions: seller_answers.append(seller.radio(question, default_choices)) for question in customer_questions: customer_answers.append(customer.radio(question, default_choices)) if st.button("Рассчитать результат сделки"): type_seller: Dict[int, int] = {k: 0 for k in range(1, 4)} type_customer: Dict[int, int] = {k: 0 for k in range(1, 4)} # Подсчёт баллов для всех типов продавцов for idx, answer in enumerate(seller_answers): # Если ответ "Нет", то вопрос нас не интересует # Если ответ "Да", то считаем коэффициенты if not dict_default_choices[answer]: continue if idx == 0: type_seller[1] += 5 type_seller[3] += 2.5 if idx == 1: type_seller[2] += 10 if idx == 2: type_seller[1] += 10 if idx == 3: type_seller[2] += 5 if idx == 4: type_seller[2] += 2.5 type_seller[3] += 5 if idx == 5: type_seller[1] += 2.5 type_seller[3] += 10 if idx == 6: type_seller[1] += 5 if idx == 7: type_seller[3] += 5 if idx == 8: type_seller[2] += 5 type_seller[3] += 2.5 if idx == 9: type_seller[1] += 2.5 if idx == 10: type_seller[2] += 2.5 # Подсчёт баллов для всех типов покупателей for idx, answer in enumerate(customer_answers): if not dict_default_choices[answer]: continue if idx == 0: type_customer[1] += 6 if idx == 1: type_customer[1] += 7 if idx == 2: type_customer[1] += 4 if idx == 3: type_customer[1] += 3 if idx == 4: type_customer[2] += 7 if idx == 5: type_customer[2] += 3 if idx == 6: type_customer[2] += 6 if idx == 7: type_customer[2] += 4 if idx == 8: type_customer[3] += 7 if idx == 9: type_customer[3] += 6 if idx == 10: type_customer[3] += 4 if idx == 11: type_customer[3] += 3 seller_prob = list(type_seller.values()) customer_prob = list(type_customer.values()) seller_sum = sum(seller_prob) customer_sum = sum(customer_prob) seller_prob = [elem / seller_sum * 100 for elem in seller_prob] customer_prob = [elem / customer_sum * 100 for elem in customer_prob] probabilities = np.zeros((3, 3)) for idx in range(3): for jdx in range(3): probabilities[idx][ jdx] = seller_prob[idx] * customer_prob[jdx] / 100 df = pd.DataFrame( data=probabilities, index=[f"{idx} тип покупателя" for idx in range(1, 4)], columns=[f"{idx} тип продавца" for idx in range(1, 4)]) st.write(df) row, col = np.unravel_index(probabilities.argmax(), probabilities.shape) st.text(result_messages[row][col]) st.text(f"Вероятность сделки = {probabilities[row][col]:.1f}%")
def largest_indices(ary, n): """Returns the n largest indices from a numpy array.""" flat = ary.flatten() indices = np.argpartition(flat, -n)[-n:] indices = indices[np.argsort(-flat[indices])] return np.unravel_index(indices, ary.shape)
def needle_affine(A, B, matrix=PAM250, gap=-1, opengap=-5, EMBOSS=False, global_align=True, local_align=False): """ Alignment implementation supporting both global and local alignments When running with the EMBOSS flag set to True, two things change: - Gaps in the beginning and at the end of the sequence are not subject to the "opengap" penalty. - The penalty for extending a gap is not added when opening one EMBOSS mode only works reliably with the global alignment mode. """ if local_align: global_align = False maybe_gap = True if EMBOSS: maybe_gap = False if not global_align: print( 'WARNING: EMBOSS mode only works reliably with global alignments.' ) #Initialising scores_matched = [[[-np.inf, (0, 0)]] * (len(B) + 1) for i in range(len(A) + 1)] scores_hgap = [[[-np.inf, (0, 0)]] * (len(B) + 1) for i in range(len(A) + 1)] scores_vgap = [[[-np.inf, (0, 0)]] * (len(B) + 1) for i in range(len(A) + 1)] #Filling first rows and columns scores_matched[0][0] = [0, (0, 0)] for i in range(1, len(A) + 1): scores_hgap[i][0] = [opengap * maybe_gap + (i) * gap, (-1, 0)] if not global_align: scores_matched[i][0] = [0, (0, 0)] for j in range(1, len(B) + 1): scores_vgap[0][j] = [opengap * maybe_gap + (j) * gap, (0, -1)] if not global_align: scores_matched[0][j] = [0, (0, 0)] #Filling the rest of the matrix directions = [(-1, -1), (-1, 0), (0, -1), (0, 0)] zero_option = -np.inf if not global_align: zero_option = 0 for i in range(1, len(A) + 1): for j in range(1, len(B) + 1): options = [ scores_matched[i - 1][j - 1][0], scores_hgap[i - 1][j - 1][0], scores_vgap[i - 1][j - 1][0], zero_option ] scores_matched[i][j] = [ matrix[A[i - 1]][B[j - 1]] + max(options), directions[np.argmax(options)] ] options = [ scores_matched[i - 1][j][0] + opengap + gap * maybe_gap, scores_hgap[i - 1][j][0] + gap, -np.inf ] scores_hgap[i][j] = [max(options), directions[np.argmax(options)]] options = [ scores_matched[i][j - 1][0] + opengap + gap * maybe_gap, -np.inf, scores_vgap[i][j - 1][0] + gap ] scores_vgap[i][j] = [max(options), directions[np.argmax(options)]] #Backtracking out_A = '' out_B = '' pos = [len(A), len(B)] if EMBOSS: scores_hgap[len(A)][len(B)][0] -= opengap scores_vgap[len(A)][len(B)][0] -= opengap options = [ scores_matched[len(A)][len(B)], scores_hgap[len(A)][len(B)], scores_vgap[len(A)][len(B)] ] current_matrix = [scores_matched, scores_hgap, scores_vgap][np.argmax([i[0] for i in options])] if not global_align: current_matrix = scores_matched temp = np.asarray([[i[0] for i in j] for j in current_matrix]) temp = np.unravel_index(np.argmax(temp), temp.shape) pos[0] = temp[0] pos[1] = temp[1] step_pointers = { (-1, -1): scores_matched, (-1, 0): scores_hgap, (0, -1): scores_vgap, (0, 0): 'END' } step = current_matrix[pos[0]][pos[1]] final_score = step[0] step = step[1] while not step == (0, 0): # Processing data of current position if not current_matrix == scores_vgap: out_A += A[pos[0] - 1] pos[0] -= 1 else: out_A += '-' if not current_matrix == scores_hgap: out_B += B[pos[1] - 1] pos[1] -= 1 else: out_B += '-' # Getting new position current_matrix = step_pointers[step] step = current_matrix[pos[0]][pos[1]][1] return out_A[::-1], out_B[::-1], final_score
def find_cut_slices(img, direction='z', n_cuts=7, spacing='auto'): """ Find 'good' cross-section slicing positions along a given axis. Parameters ---------- img : 3D Niimg-like object See http://nilearn.github.io/manipulating_images/input_output.html the brain map. direction : string, optional Sectional direction; possible values are "x", "y", or "z". Default='z'. n_cuts : int, optional Number of cuts in the plot. Default=7. spacing : 'auto' or int, optional Minimum spacing between cuts (in voxels, not milimeters) if 'auto', the spacing is .5 / n_cuts * img_length. Default='auto'. Returns ------- cut_coords : 1D array of length n_cuts The computed cut_coords. Notes ----- This code works by iteratively locating peak activations that are separated by a distance of at least 'spacing'. If n_cuts is very large and all the activated regions are covered, cuts with a spacing less than 'spacing' will be returned. Warnings -------- If a non-diagonal img is given. This function automatically reorders img to get it back to diagonal. This is to avoid finding same cuts in the slices. """ # misc if not direction in 'xyz': raise ValueError( "'direction' must be one of 'x', 'y', or 'z'. Got '%s'" % (direction)) axis = 'xyz'.index(direction) img = check_niimg_3d(img) affine = img.affine if not np.alltrue(np.diag(affine)[:3]): warnings.warn( 'A non-diagonal affine is found in the given ' 'image. Reordering the image to get diagonal affine ' 'for finding cuts in the slices.', stacklevel=2) # resample is set to avoid issues with an image having a non-diagonal # affine and rotation. img = reorder_img(img, resample='nearest') affine = img.affine # note: orig_data is a copy of img._data_cache thanks to np.abs orig_data = np.abs(_safe_get_data(img)) this_shape = orig_data.shape[axis] if not isinstance(n_cuts, numbers.Number): raise ValueError("The number of cuts (n_cuts) must be an integer " "greater than or equal to 1. " "You provided a value of n_cuts=%s. " % n_cuts) # BF issue #575: Return all the slices along and axis if this axis # is the display mode and there are at least as many requested # n_slices as there are slices. if n_cuts > this_shape: warnings.warn('Too many cuts requested for the data: ' 'n_cuts=%i, data size=%i' % (n_cuts, this_shape)) return _transform_cut_coords(np.arange(this_shape), direction, affine) # To smooth data that might be np.int or np.uint, # first convert it to float. data = orig_data.copy() if data.dtype.kind in ('i', 'u'): data = data.astype(np.float) data = _smooth_array(data, affine, fwhm='fast') # to control floating point error problems # during given input value "n_cuts" epsilon = np.finfo(np.float32).eps difference = abs(round(n_cuts) - n_cuts) if round(n_cuts) < 1. or difference > epsilon: message = ("Image has %d slices in direction %s. " "Therefore, the number of cuts must be between 1 and %d. " "You provided n_cuts=%s " % (this_shape, direction, this_shape, n_cuts)) raise ValueError(message) else: n_cuts = int(round(n_cuts)) if spacing == 'auto': spacing = max(int(.5 / n_cuts * data.shape[axis]), 1) slices = [slice(None, None), slice(None, None), slice(None, None)] cut_coords = list() for _ in range(n_cuts): # Find a peak max_along_axis = np.unravel_index(np.abs(data).argmax(), data.shape)[axis] # cancel out the surroundings of the peak start = max(0, max_along_axis - spacing) stop = max_along_axis + spacing slices[axis] = slice(start, stop) # We don't actually fully zero the neighborhood, to avoid ending # up with fully zeros if n_cuts is too big: we can do multiple # passes on the data data[tuple(slices)] *= 1.e-3 cut_coords.append(max_along_axis) # We sometimes get duplicated cuts, so we add cuts at the beginning # and the end cut_coords = np.unique(cut_coords).tolist() while len(cut_coords) < n_cuts: # Candidates for new cuts: slice_below = min(cut_coords) - 2 slice_above = max(cut_coords) + 2 candidates = [slice_above] # One slice where there is the biggest gap in the existing # cut_coords if len(cut_coords) > 1: middle_idx = np.argmax(np.diff(cut_coords)) slice_middle = int( .5 * (cut_coords[middle_idx] + cut_coords[middle_idx + 1])) if not slice_middle in cut_coords: candidates.append(slice_middle) if slice_below >= 0: # We need positive slice to avoid having negative # indices, which would work, but not the way we think of them candidates.append(slice_below) best_weight = -10 for candidate in candidates: if candidate >= this_shape: this_weight = 0 else: this_weight = np.sum(np.rollaxis(orig_data, axis)[candidate]) if this_weight > best_weight: best_candidate = candidate best_weight = this_weight cut_coords.append(best_candidate) cut_coords = np.unique(cut_coords).tolist() cut_coords = np.array(cut_coords) cut_coords.sort() return _transform_cut_coords(cut_coords, direction, affine)
def grid_indices_to_coordinates(self, indices=None): if indices is None: # size = 48 indices = np.arange(self.size) return np.unravel_index(indices, self.shape)
def get_target(bbox, gt_box, score, ref_bbox, ref_gt_box, ref_score): global num_of_is_full_max num_boxes = bbox.shape[0] ref_num_boxes = ref_bbox.shape[0] score_list = get_scores(bbox, gt_box, score) ref_score_list = get_scores(ref_bbox, ref_gt_box, ref_score) output_list = [] ref_output_list = [] for cls_idx in range(0, num_fg_classes): valid_gt_mask = (gt_box[0, :, -1].astype(np.int32)==(cls_idx+1)) valid_gt_box = gt_box[0, valid_gt_mask, :] num_valid_gt = len(valid_gt_box) ref_valid_gt_mask = (ref_gt_box[0, :, -1].astype(np.int32)==(cls_idx+1)) ref_valid_gt_box = ref_gt_box[0, ref_valid_gt_mask, :] ref_num_valid_gt = len(ref_valid_gt_box) score_list_per_class = score_list[cls_idx] ref_score_list_per_class = ref_score_list[cls_idx] bbox_per_class = bbox[:, cls_idx, :] ref_bbox_per_class = ref_bbox[:, cls_idx, :] if num_valid_gt != ref_num_valid_gt: if ref_num_valid_gt > num_valid_gt: num_rm = ref_num_valid_gt - num_valid_gt ref_num_valid_gt = num_valid_gt gt_overlap_mat = bbox_overlaps(ref_valid_gt_box.astype(np.float), valid_gt_box.astype(np.float)) rm_indices = np.argsort(np.sum(gt_overlap_mat, axis=1))[:num_rm] ref_valid_gt_box = np.delete(ref_valid_gt_box, rm_indices, axis=0) # update ref_score_list_per_class ref_score_list_per_class = get_scores_per_class(ref_bbox_per_class, ref_valid_gt_box, ref_score[:, cls_idx:cls_idx+1]) assert ref_valid_gt_box.shape == valid_gt_box.shape, "failed remove ref, {} -> {}".format(ref_valid_gt_box.shape[0], valid_gt_box.shape[0]) print "success remove ref" else: num_rm = num_valid_gt - ref_num_valid_gt num_valid_gt = ref_num_valid_gt gt_overlap_mat = bbox_overlaps(valid_gt_box.astype(np.float), ref_valid_gt_box.astype(np.float)) rm_indices = np.argsort(np.sum(gt_overlap_mat, axis=1))[:num_rm] valid_gt_box = np.delete(valid_gt_box, rm_indices, axis=0) # update score_list_per_class score_list_per_class = get_scores_per_class(bbox_per_class, valid_gt_box, score[:, cls_idx:cls_idx+1]) assert ref_valid_gt_box.shape == valid_gt_box.shape, "failed remove, {} -> {}".format(ref_valid_gt_box.shape[0], valid_gt_box.shape[0]) print "success remove" assert num_valid_gt == ref_num_valid_gt, "gt num are not the same" if len(score_list_per_class) == 0 or len(ref_score_list_per_class) == 0: output_list.append(get_max_socre_bboxes(score_list_per_class, num_boxes)) ref_output_list.append(get_max_socre_bboxes(ref_score_list_per_class, ref_num_boxes)) else: output_list_per_class = [] ref_output_list_per_class = [] for i in range(len(self._target_thresh)): overlap_score = score_list_per_class[i] ref_overlap_score = ref_score_list_per_class[i] output = np.zeros((overlap_score.shape[0],)) ref_output = np.zeros((ref_overlap_score.shape[0],)) if np.count_nonzero(overlap_score) == 0 or np.count_nonzero(ref_overlap_score) == 0: output_list_per_class.append(output) ref_output_list_per_class.append(ref_output) continue for x in range(num_valid_gt): overlap_score_per_gt = overlap_score[:, x] ref_overlap_score_per_gt = ref_overlap_score[:, x] valid_bbox_indices = np.where(overlap_score_per_gt)[0] ref_valid_bbox_indices = np.where(ref_overlap_score_per_gt)[0] target_gt_box = valid_gt_box[x:x+1, :-1] ref_target_gt_box = ref_valid_gt_box[x:x+1, :-1] if len(valid_bbox_indices) == 0 or len(ref_valid_bbox_indices) == 0: continue dist_mat = translation_dist(bbox_per_class[valid_bbox_indices], target_gt_box)[:, 0, :] ref_dist_mat = translation_dist(ref_bbox_per_class[ref_valid_bbox_indices], ref_target_gt_box)[:, 0, :] dist_mat_shape = (bbox_per_class[valid_bbox_indices].shape[0], ref_bbox_per_class[ref_valid_bbox_indices].shape[0], 4) # print((np.tile(np.expand_dims(dist_mat, 1), (1, dist_mat_shape[1], 1)) - # np.tile(np.expand_dims(ref_dist_mat, 0), (dist_mat_shape[0], 1, 1)))**2) bbox_dist_mat = np.sum((np.tile(np.expand_dims(dist_mat, 1), (1, dist_mat_shape[1], 1)) - np.tile(np.expand_dims(ref_dist_mat, 0), (dist_mat_shape[0], 1, 1)))**2, axis=2) assert bbox_dist_mat.shape == (len(bbox_per_class[valid_bbox_indices]), len(ref_bbox_per_class[ref_valid_bbox_indices])) # top_k = 10 # translation_thresh = 1.1*np.min(bbox_dist_mat) # top_k = np.sum(bbox_dist_mat < translation_thresh) top_k = int(0.1 * len(bbox_dist_mat.flatten()) + 0.5) top_k = max(1, top_k) top_k = min(top_k, len(bbox_dist_mat.flatten())) top_k = 1 if DEBUG: print("{} of out {} stable pair".format(top_k, len(bbox_dist_mat.flatten()))) ind_list, ref_ind_list = np.unravel_index(np.argsort(bbox_dist_mat, axis=None)[:top_k], bbox_dist_mat.shape) score_sum_list = [] rank_sum_list = [] for ind, ref_ind in zip(ind_list, ref_ind_list): score_sum = overlap_score_per_gt[valid_bbox_indices[ind]] + ref_overlap_score_per_gt[ref_valid_bbox_indices[ref_ind]] rank_sum = valid_bbox_indices[ind] + ref_valid_bbox_indices[ref_ind] score_sum_list.append(score_sum) rank_sum_list.append(rank_sum) score_max_idx = np.argmax(np.array(score_sum_list)) rank_max_idx = np.argmin(np.array(rank_sum_list)) if DEBUG: if score_max_idx == rank_max_idx: score_rank_max[0] += 1 score_rank_max[1] += 1 # max_idx = rank_max_idx max_idx = score_max_idx ind = ind_list[max_idx] ref_ind = ref_ind_list[max_idx] if DEBUG: if ind == np.argmax(overlap_score_per_gt[valid_bbox_indices]): num_of_is_full_max[0] += 1 print('cur takes the max') if ref_ind == np.argmax(ref_overlap_score_per_gt[ref_valid_bbox_indices]): num_of_is_full_max[0] += 1 print('ref takes the max') output[valid_bbox_indices[ind]] = 1 ref_output[ref_valid_bbox_indices[ref_ind]] = 1 output_list_per_class.append(output) ref_output_list_per_class.append(ref_output) output_per_class = np.stack(output_list_per_class, axis=-1) ref_output_per_class = np.stack(ref_output_list_per_class, axis=-1) output_list.append(output_per_class) ref_output_list.append(ref_output_per_class) # [num_boxes, num_fg_classes, num_thresh] blob = np.stack(output_list, axis=1).astype(np.float32, copy=False) ref_blob = np.stack(ref_output_list, axis=1).astype(np.float32, copy=False) return blob, ref_blob
def cross(function=lambda x: x, domain=None, tensors=None, function_arg='vectors', ranks_tt=None, kickrank=3, rmax=100, eps=1e-6, max_iter=25, val_size=1000, verbose=True, return_info=False, record_samples=False, _minimize=False, device=None, batch=False, suppress_warnings=False, detach_evaluations=False): """ Cross-approximation routine that samples a black-box function and returns an N-dimensional tensor train approximating it. It accepts either: - A domain (tensor product of :math:`N` given arrays) and a function :math:`\\mathbb{R}^N \\to \\mathbb{R}` - A list of :math:`K` tensors of dimension :math:`N` and equal shape and a function :math:`\\mathbb{R}^K \\to \\mathbb{R}` :Examples: >>> tn.cross(function=lambda x: x**2, tensors=[t]) # Compute the element-wise square of `t` using 5 TT-ranks >>> domain = [torch.linspace(-1, 1, 32)]*5 >>> tn.cross(function=lambda x, y, z, t, w: x**2 + y*z + torch.cos(t + w), domain=domain) # Approximate a function over the rectangle :math:`[-1, 1]^5` >>> tn.cross(function=lambda x: torch.sum(x**2, dim=1), domain=domain, function_arg='matrix') # An example where the function accepts a matrix References: - I. Oseledets, E. Tyrtyshnikov: `"TT-cross Approximation for Multidimensional Arrays" (2009) <http://www.mat.uniroma2.it/~tvmsscho/papers/Tyrtyshnikov5.pdf>`_ - D. Savostyanov, I. Oseledets: `"Fast Adaptive Interpolation of Multi-dimensional Arrays in Tensor Train Format" (2011) <https://ieeexplore.ieee.org/document/6076873>`_ - S. Dolgov, R. Scheichl: `"A Hybrid Alternating Least Squares - TT Cross Algorithm for Parametric PDEs" (2018) <https://arxiv.org/pdf/1707.04562.pdf>`_ - A. Mikhalev's `maxvolpy package <https://bitbucket.org/muxas/maxvolpy>`_ - I. Oseledets (and others)'s `ttpy package <https://github.com/oseledets/ttpy>`_ :param function: should produce a vector of :math:`P` elements. Accepts either :math:`N` comma-separated vectors, or a matrix (see `function_arg`) :param domain: a list of :math:`N` vectors (incompatible with `tensors`) :param tensors: a :class:`Tensor` or list thereof (incompatible with `domain`) :param function_arg: if 'vectors', `function` accepts :math:`N` vectors of length :math:`P` each. If 'matrix', a matrix of shape :math:`P \\times N`. :param ranks_tt: int or list of :math:`N-1` ints. If None, will be determined adaptively :param kickrank: when adaptively found, ranks will be increased by this amount after every iteration (full sweep left-to-right and right-to-left) :param rmax: this rank will not be surpassed :param eps: the procedure will stop after this validation error is met (as measured after each iteration) :param max_iter: int :param val_size: size of the validation set :param verbose: default is True :param return_info: if True, will also return a dictionary with informative metrics about the algorithm's outcome :param device: PyTorch device :param batch: Boolean :param suppress_warnings: Boolean, if True, will hide the message about insufficient accuracy :param detach_evaluations: Boolean, if True, will remove gradient buffers for the function :return: an N-dimensional TT :class:`Tensor` (if `return_info`=True, also a dictionary) """ if device is None and tensors is not None: if type(tensors) == list: device = tensors[0].cores[0].device else: device = tensors.cores[0].device if verbose: print('cross device is', device) try: import maxvolpy.maxvol except ModuleNotFoundError: raise ModuleNotFoundError( "Functions that require cross-approximation require the optional maxvolpy package, which can be installed by 'pip install maxvolpy'. More info is available at https://bitbucket.org/muxas/maxvolpy" ) assert domain is not None or tensors is not None assert function_arg in ('vectors', 'matrix') if function_arg == 'matrix': def f(*args): return function(torch.cat([arg[:, None] for arg in args], dim=1)) else: f = function if detach_evaluations: def build_function_wrapper(func): def g(*args): res = func(*args) if hasattr(res, '__len__') and not isinstance(res, torch.Tensor): for i in range(len(res)): if isinstance(res[i], torch.Tensor): res[i] = res[i].detach() else: if isinstance(res, torch.Tensor): res = res.detach() return res return g f = build_function_wrapper(f) if tensors is None: tensors = tn.meshgrid(domain, batch=batch) if not hasattr(tensors, '__len__'): tensors = [tensors] tensors = [t.decompress_tucker_factors(_clone=False) for t in tensors] Is = list(tensors[0].shape) N = len(Is) # Process ranks and cap them, if needed if ranks_tt is None: ranks_tt = 1 else: kickrank = None if not hasattr(ranks_tt, '__len__'): ranks_tt = [ranks_tt] * (N - 1) ranks_tt = [1] + list(ranks_tt) + [1] Rs = np.array(ranks_tt) for n in list(range(1, N)) + list(range(N - 1, -1, -1)): Rs[n] = min(Rs[n - 1] * Is[n - 1], Rs[n], Is[n] * Rs[n + 1]) # Initialize cores at random cores = [torch.randn(Rs[n], Is[n], Rs[n + 1]).to(device) for n in range(N)] # Prepare left and right sets lsets = [np.array([[0]])] + [None] * (N - 1) randint = np.hstack( [np.random.randint(0, Is[n + 1], [max(Rs), 1]) for n in range(N - 1)] + [np.zeros([max(Rs), 1], dtype=np.int)]) rsets = [randint[:Rs[n + 1], n:] for n in range(N - 1)] + [np.array([[0]])] # Initialize left and right interfaces for `tensors` def init_interfaces(): t_linterfaces = [] t_rinterfaces = [] for t in tensors: linterfaces = [torch.ones(1, t.ranks_tt[0]).to(device) ] + [None] * (N - 1) rinterfaces = [None] * (N - 1) + [ torch.ones(t.ranks_tt[t.dim()], 1).to(device) ] for j in range(N - 1): M = torch.ones(t.cores[-1].shape[-1], len(rsets[j])).to(device) for n in range(N - 1, j, -1): if t.cores[n].dim() == 3: # TT core M = torch.einsum('iaj,ja->ia', [ t.cores[n][:, rsets[j][:, n - 1 - j], :].to(device), M ]) else: # CP factor M = torch.einsum('ai,ia->ia', [ t.cores[n][rsets[j][:, n - 1 - j], :].to(device), M ]) rinterfaces[j] = M t_linterfaces.append(linterfaces) t_rinterfaces.append(rinterfaces) return t_linterfaces, t_rinterfaces t_linterfaces, t_rinterfaces = init_interfaces() # Create a validation set Xs_val = [ torch.as_tensor(np.random.choice(I, int(val_size))).to(device) for I in Is ] ys_val = f(*[t[Xs_val].torch() for t in tensors]) if ys_val.dim() > 1: assert ys_val.dim() == 2 assert ys_val.shape[1] == 1 ys_val = ys_val[:, 0] assert len(ys_val) == val_size norm_ys_val = torch.norm(ys_val) if verbose: print( 'Cross-approximation over a {}D domain containing {:g} grid points:' .format(N, tensors[0].numel())) start = time.time() converged = False info = { 'nsamples': 0, 'eval_time': 0, 'val_epss': [], 'min': 0, 'argmin': None } if record_samples: info['sample_positions'] = torch.zeros(0, N).to(device) info['sample_values'] = torch.zeros(0).to(device) def evaluate_function( j ): # Evaluate function over Rs[j] x Rs[j+1] fibers, each of size I[j] Xs = [] for k, t in enumerate(tensors): if tensors[k].cores[j].dim() == 3: # TT core V = torch.einsum('ai,ibj,jc->abc', [ t_linterfaces[k][j], tensors[k].cores[j], t_rinterfaces[k][j] ]) else: # CP factor V = torch.einsum('ai,bi,ic->abc', [ t_linterfaces[k][j], tensors[k].cores[j], t_rinterfaces[k][j] ]) Xs.append(V.flatten()) eval_start = time.time() evaluation = f(*Xs) if record_samples: info['sample_positions'] = torch.cat( (info['sample_positions'], torch.cat([x[:, None] for x in Xs], dim=1)), dim=0) info['sample_values'] = torch.cat( (info['sample_values'], evaluation)) info['eval_time'] += time.time() - eval_start if _minimize: evaluation = np.pi / 2 - torch.atan( (evaluation - info['min']) ) # Function used by I. Oseledets for TT minimization in ttpy evaluation_argmax = torch.argmax(evaluation) eval_min = torch.tan(np.pi / 2 - evaluation[evaluation_argmax]) + info['min'] if info['min'] == 0 or eval_min < info['min']: coords = np.unravel_index(evaluation_argmax, [Rs[j], Is[j], Rs[j + 1]]) info['min'] = eval_min info['argmin'] = tuple(lsets[j][coords[0]][1:]) + tuple( [coords[1]]) + tuple(rsets[j][coords[2]][:-1]) # Check for nan/inf values if evaluation.dim() == 2: evaluation = evaluation[:, 0] invalid = (torch.isnan(evaluation) | torch.isinf(evaluation)).nonzero() if len(invalid) > 0: invalid = invalid[0].item() raise ValueError( 'Invalid return value for function {}: f({}) = {}'.format( function, ', '.join('{:g}'.format(x[invalid].detach().cpu().numpy()) for x in Xs), f(*[x[invalid:invalid + 1][:, None] for x in Xs]).item())) V = torch.reshape(evaluation, [Rs[j], Is[j], Rs[j + 1]]) info['nsamples'] += V.numel() return V # Sweeps for i in range(max_iter): if verbose: print('iter: {: <{}}'.format(i, len('{}'.format(max_iter)) + 1), end='') sys.stdout.flush() left_locals = [] # Left-to-right for j in range(N - 1): # Update tensors for current indices V = evaluate_function(j) # QR + maxvol towards the right V = torch.reshape(V, [-1, V.shape[2]]) # Left unfolding Q, R = torch.qr(V) if _minimize: local, _ = maxvolpy.maxvol.rect_maxvol( Q.detach().cpu().numpy(), maxK=Q.shape[1]) else: local, _ = maxvolpy.maxvol.maxvol(Q.detach().cpu().numpy()) V = torch.lstsq(Q.t(), Q[local, :].t())[0].t() cores[j] = torch.reshape(V, [Rs[j], Is[j], Rs[j + 1]]) left_locals.append(local) # Map local indices to global ones local_r, local_i = np.unravel_index(local, [Rs[j], Is[j]]) lsets[j + 1] = np.c_[lsets[j][local_r, :], local_i] for k, t in enumerate(tensors): if t.cores[j].dim() == 3: # TT core t_linterfaces[k][j + 1] = torch.einsum( 'ai,iaj->aj', [ t_linterfaces[k][j][local_r, :], t.cores[j][:, local_i, :] ]) else: # CP factor t_linterfaces[k][j + 1] = torch.einsum( 'ai,ai->ai', [ t_linterfaces[k][j][local_r, :], t.cores[j][local_i, :] ]) # Right-to-left sweep for j in range(N - 1, 0, -1): # Update tensors for current indices V = evaluate_function(j) # QR + maxvol towards the left V = torch.reshape(V, [Rs[j], -1]) # Right unfolding Q, R = torch.qr(V.t()) if _minimize: local, _ = maxvolpy.maxvol.rect_maxvol( Q.detach().cpu().numpy(), maxK=Q.shape[1]) else: local, _ = maxvolpy.maxvol.maxvol(Q.detach().cpu().numpy()) V = torch.lstsq(Q.t(), Q[local, :].t())[0] cores[j] = torch.reshape(torch.as_tensor(V), [Rs[j], Is[j], Rs[j + 1]]) # Map local indices to global ones local_i, local_r = np.unravel_index(local, [Is[j], Rs[j + 1]]) rsets[j - 1] = np.c_[local_i, rsets[j][local_r, :]] for k, t in enumerate(tensors): if t.cores[j].dim() == 3: # TT core t_rinterfaces[k][j - 1] = torch.einsum( 'iaj,ja->ia', [ t.cores[j][:, local_i, :], t_rinterfaces[k][j][:, local_r] ]) else: # CP factor t_rinterfaces[k][j - 1] = torch.einsum( 'ai,ia->ia', [ t.cores[j][local_i, :], t_rinterfaces[k][j][:, local_r] ]) # Leave the first core ready V = evaluate_function(0) cores[0] = V # Evaluate validation error val_eps = torch.norm(ys_val - tn.Tensor(cores)[Xs_val].torch()) / norm_ys_val info['val_epss'].append(val_eps) if val_eps < eps: converged = True if verbose: # Print status if _minimize: print('| best: {:.8g}'.format(info['min']), end='') else: print('| eps: {:.3e}'.format(val_eps), end='') print(' | total time: {:8.4f} | largest rank: {:3d}'.format( time.time() - start, max(Rs)), end='') if converged: print(' <- converged: eps < {}'.format(eps)) elif i == max_iter - 1: print(' <- max_iter was reached: {}'.format(max_iter)) else: print() if converged: break elif i < max_iter - 1 and kickrank is not None: # Augment ranks newRs = Rs.copy() newRs[1:-1] = np.minimum(rmax, newRs[1:-1] + kickrank) for n in list(range(1, N)) + list(range(N - 1, 0, -1)): newRs[n] = min(newRs[n - 1] * Is[n - 1], newRs[n], Is[n] * newRs[n + 1]) extra = np.hstack([ np.random.randint(0, Is[n + 1], [max(newRs), 1]) for n in range(N - 1) ] + [np.zeros([max(newRs), 1], dtype=np.int)]) for n in range(N - 1): if newRs[n + 1] > Rs[n + 1]: rsets[n] = np.vstack( [rsets[n], extra[:newRs[n + 1] - Rs[n + 1], n:]]) Rs = newRs t_linterfaces, t_rinterfaces = init_interfaces( ) # Recompute interfaces if val_eps > eps and not _minimize and not suppress_warnings: logging.warning( 'eps={:g} (larger than {}) when cross-approximating {}'.format( val_eps, eps, function)) if verbose: print( 'Did {} function evaluations, which took {:.4g}s ({:.4g} evals/s)'. format(info['nsamples'], info['eval_time'], info['nsamples'] / info['eval_time'])) print() if return_info: info['lsets'] = lsets info['rsets'] = rsets info['Rs'] = Rs info['left_locals'] = left_locals info['total_time'] = time.time() - start info['val_eps'] = val_eps return tn.Tensor([ c if isinstance(c, torch.Tensor) else torch.tensor(c) for c in cores ], batch=batch), info else: return tn.Tensor([ c if isinstance(c, torch.Tensor) else torch.tensor(c) for c in cores ], batch=batch)
def image_correlate(image, reference, real_filter=1, k_filter=1, shift_func='shift', verbose=False): """ Align image to reference by cross-correlation. Outputs shifts and shifted images. Uses the real FFT for ~2x speed improvement. The k_filter must have a shape that matches the np.fft.rfft2() of image and reference. Uses scipy.ndimage.shift() or np.roll to move the image. Use 'roll' to avoid losing data off the edge for multiple shifting operations. Use shift to avoid wrap around problems and when there is only one shifting operation. Note ---- image, reference and real_filter must all have the same shape (N, M). k_filter must have a shape that matches the np.fft.rfft2() of the other inputs: (N, M/2+1) Parameters ---------- image : ndarray A image as a 2D ndarray. reference : ndarray The reference image to align to. real_filter : ndarray, optional, default = 1 A real space filter applied to image and reference before calculating the shift. k_filter : ndarray, optional, default = 1 A Fourier space filter applied to the fourier transform of image and reference before calculating the cross-correlation. shift_func : str, default is 'shift' The function to use to shift the images. 'roll' uses np.roll and 'shift' uses ndimage.shift. verbose : bool Plots the cross-correlation using matplotlib for debugging purposes. Returns ------ : tuple, (ndarray, tuple) A tuple containing the shifted image and the shifts applied. """ output = None if shift_func is not 'shift' and shift_func is not 'roll': raise KeyError('Shift function has to be either shift or roll') image_f = np.fft.rfft2((image - np.mean(image)) * real_filter) reference_f = np.fft.rfft2((reference - np.mean(reference)) * real_filter) xcor = abs(np.fft.irfft2(np.conj(image_f) * reference_f * k_filter)) if verbose: import matplotlib.pyplot as plt plt.imshow(np.fft.fftshift(xcor)) plt.title('imageCrossCorRealShift xcor') shifts = np.unravel_index(np.fft.fftshift(xcor).argmax(), xcor.shape) shifts = (shifts[0] - xcor.shape[0] / 2, shifts[1] - xcor.shape[1] / 2) shifts = [int(i) for i in shifts] # convert to integers if shift_func == 'shift': # shift image using ndimage.shift output = ndimage.interpolation.shift(image, shifts, order=0) elif shift_func == 'roll': # shift image using roll to be reversible output = np.roll(image, shifts[0], axis=0) output = np.roll(output, shifts[1], axis=1) return output, shifts
def _simulate_frame_range(id, n_processes, orbit_xyz, frame_range, sim_n_samples, local_direction_g, UTMNorthing_g, UTMEasting_g, UTMZone_g, rxwin_ref, input_scr_data_shifts_alignment_t, dt, dem_obj, geom_obj, alt_sim_matrix, act_sim_matrix, gaussian_filter_sigma, further_pars=[]): _C = 299792458. n_frames_to_be_simulated = len(frame_range) sim_image = np.zeros([sim_n_samples, n_frames_to_be_simulated]) #, dtype="int16") # int16 and uint8 saves memory but considerably slows down the processing as numpy cannot handle ints in a fast way uncert_image = np.zeros(sim_image.shape, dtype="uint8") ground_distance_min_image = np.nan * np.ones(sim_image.shape) ground_distance_max_image = np.nan * np.ones(sim_image.shape) first_return_lats = np.zeros(n_frames_to_be_simulated) first_return_lons = np.zeros(n_frames_to_be_simulated) status_str_pre = '\t'+"|\t\t"*id status_str_post = "\t\t|"*(n_processes-id-1) last_status = -1 for count, i_frame_tbs in enumerate(frame_range): if count == n_frames_to_be_simulated-1: new_status = 100 else: new_status = int(count / n_frames_to_be_simulated * 10) * 10 if new_status != last_status: print(status_str_pre + '{:>2}'.format(new_status) + status_str_post) last_status = new_status local_direction = local_direction_g[i_frame_tbs] UTMNorthing = UTMNorthing_g[i_frame_tbs] UTMEasting = UTMEasting_g[i_frame_tbs] UTMZone = UTMZone_g[i_frame_tbs] sc_x_cart_local = orbit_xyz[0][i_frame_tbs] sc_y_cart_local = orbit_xyz[1][i_frame_tbs] sc_z_cart_local = orbit_xyz[2][i_frame_tbs] UTMNorthing_surface_matrix = UTMNorthing + (alt_sim_matrix * np.sin(local_direction) - act_sim_matrix * np.cos(local_direction)) UTMEasting_surface_matrix = UTMEasting + (alt_sim_matrix * np.cos(local_direction) + act_sim_matrix * np.sin(local_direction)) Lat_surface_matrix, Long_surface_matrix = geom_obj.UTM_to_LL(UTMNorthing_surface_matrix, UTMEasting_surface_matrix, UTMZone) dem_surface_radius_matrix = dem_obj.get_dem_radius_from_lat_lon(Lat_surface_matrix, Long_surface_matrix) ground_distance_from_nadir = np.sqrt((UTMNorthing_surface_matrix-UTMNorthing)**2 + (UTMEasting_surface_matrix-UTMEasting)**2) dummy_check_matrix = (dem_surface_radius_matrix == dem_obj.dummy_value) # smooth the DEM to avoid "stripes" in the simulation dem_surface_radius_matrix = gaussian_filter(dem_surface_radius_matrix, gaussian_filter_sigma) if dummy_check_matrix.any(): uncert_image[:, count] = 1 dem_surface_radius_matrix[dummy_check_matrix] = 0 # radius too far for being recorded in the simulation x_cart_surface_matrix, y_cart_surface_matrix, z_cart_surface_matrix = geom_obj.to_xyz_from_latlon_and_radius(Lat_surface_matrix, Long_surface_matrix, dem_surface_radius_matrix) d_surface = geom_obj.euclidean_distance_cart(sc_x_cart_local, sc_y_cart_local, sc_z_cart_local, x_cart_surface_matrix, y_cart_surface_matrix, z_cart_surface_matrix) t_surface = 2. * d_surface / _C sample_pos_surface = np.round(((t_surface - rxwin_ref + input_scr_data_shifts_alignment_t[i_frame_tbs]) / dt)).astype("int") # use rxwin_ref instead of input_rxwin_data_t_local # set out-of-range sample positions to the farthest position (useful later for detecting the first-return position) sample_pos_surface[(sample_pos_surface < 0) | (sample_pos_surface >= sim_n_samples)] = sim_n_samples # find first return coordinates first_return_ids = np.unravel_index(np.argmin(sample_pos_surface), sample_pos_surface.shape) first_return_lats[count] = Lat_surface_matrix[first_return_ids] first_return_lons[count] = Long_surface_matrix[first_return_ids] ground_distance_from_nadir = ground_distance_from_nadir.flatten() sample_pos_surface = sample_pos_surface.flatten() # find and keep only the sample_pos_surface items within the simulation range sample_pos_surface_ok_ids = np.where(sample_pos_surface < sim_n_samples)[0] sample_pos_surface = sample_pos_surface[sample_pos_surface_ok_ids] # increment sim_image values by 1 each time they correspond to a valid sample_pos_surface np.add.at(sim_image[:, count], sample_pos_surface, 1) # keep only ground_distance_from_nadir items corresponding to valid sample_pos_surface items (selected before) ground_distance_from_nadir = ground_distance_from_nadir[sample_pos_surface_ok_ids] # sort distances and sample positions both by increasing distance sorting_indexes = np.argsort(ground_distance_from_nadir) ground_distance_from_nadir = ground_distance_from_nadir[sorting_indexes] sample_pos_surface = sample_pos_surface[sorting_indexes] # for each simulation sample detect the corresponding ground distance # note: as the arrays were first ordered by ground distance, and considering that # "unique" selects the first instance of the array items, the "shortest" distance # associated to a certain sample is kept unique_sample_pos_surface, unique_sample_pos_surface_ids = np.unique(sample_pos_surface, return_index=True) unique_ground_distance_from_nadir = ground_distance_from_nadir[unique_sample_pos_surface_ids] ground_distance_min_image[unique_sample_pos_surface, count] = unique_ground_distance_from_nadir # reverse the ordered arrays and use the sample principle to save the "longest" distances for every sample sample_pos_surface = np.flip(sample_pos_surface) ground_distance_from_nadir = np.flip(ground_distance_from_nadir) unique_sample_pos_surface, unique_sample_pos_surface_ids = np.unique(sample_pos_surface, return_index=True) unique_ground_distance_from_nadir = ground_distance_from_nadir[unique_sample_pos_surface_ids] ground_distance_max_image[unique_sample_pos_surface, count] = unique_ground_distance_from_nadir return (sim_image, uncert_image, first_return_lats, first_return_lons, ground_distance_min_image, ground_distance_max_image)
# Save model and weights if not os.path.isdir(save_dir): os.makedirs(save_dir) model_path = os.path.join(save_dir, model_name) model.save(model_path) print('Saved trained model at %s ' % model_path) # Score trained model. scores = model.evaluate(x_test, y_test, verbose=1) print('Test loss:', scores[0]) print('Test accuracy:', scores[1]) y_predict = model.predict(x_test, verbose=0) print("PREDICT:", y_predict) for i in range(y_predict.shape[0]): maximun = np.argmax(y_predict[i]) prediction = np.unravel_index(maximun, y_predict[i].shape) maximun2 = np.argmax(y_test[i]) real = np.unravel_index(maximun2, y_test[i].shape) if prediction[0] == 0: if real[0] == 0: print("REAL ANIMAL: CAT PREDICTED ANIMAL: CAT") else: print("REAL ANIMAL: DOG PREDICTED ANIMAL: CAT") else: if real[0] == 0: print("REAL ANIMAL: CAT PREDICTED ANIMAL: DOG") else: print("REAL ANIMAL: DOG PREDICTED ANIMAL: DOG")
def track(self, img): """ args: img(np.ndarray): BGR image return: bbox(list):[x, y, width, height] """ w_z = self.size[0] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size) h_z = self.size[1] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(self.size) s_z = np.sqrt(w_z * h_z) scale_z = cfg.TRACK.EXEMPLAR_SIZE / s_z s_x = s_z * (cfg.TRACK.INSTANCE_SIZE / cfg.TRACK.EXEMPLAR_SIZE) s_x = round(s_x) x_crop = self.get_subwindow(img, self.center_pos, cfg.TRACK.INSTANCE_SIZE, s_x, self.channel_average) crop_box = [self.center_pos[0] - s_x / 2, self.center_pos[1] - s_x / 2, s_x, s_x] outputs = self.model.track(x_crop) score = self._convert_score(outputs['cls']) pred_bbox = self._convert_bbox(outputs['loc'], self.anchors) def change(r): return np.maximum(r, 1. / r) def sz(w, h): pad = (w + h) * 0.5 return np.sqrt((w + pad) * (h + pad)) # scale penalty s_c = change(sz(pred_bbox[2,:], pred_bbox[3,:]) / (sz(self.size[0]*scale_z, self.size[1]*scale_z))) # aspect ratio penalty r_c = change((self.size[0]/self.size[1]) / (pred_bbox[2,:]/pred_bbox[3,:])) penalty = np.exp(-(r_c * s_c - 1) * cfg.TRACK.PENALTY_K) pscore = penalty * score # window penalty pscore = pscore * (1 - cfg.TRACK.WINDOW_INFLUENCE) + \ self.window * cfg.TRACK.WINDOW_INFLUENCE best_idx = np.argmax(pscore) bbox = pred_bbox[:, best_idx] / scale_z lr = penalty[best_idx] * score[best_idx] * cfg.TRACK.LR cx = bbox[0] + self.center_pos[0] cy = bbox[1] + self.center_pos[1] # smooth bbox width = self.size[0] * (1 - lr) + bbox[2] * lr height = self.size[1] * (1 - lr) + bbox[3] * lr # clip boundary cx, cy, width, height = self._bbox_clip(cx, cy, width, height, img.shape[:2]) # udpate state self.center_pos = np.array([cx, cy]) self.size = np.array([width, height]) bbox = [cx - width / 2, cy - height / 2, width, height] best_score = score[best_idx] # processing mask pos = np.unravel_index(best_idx, (5, self.score_size, self.score_size)) delta_x, delta_y = pos[2], pos[1] mask = self.model.mask_refine((delta_y, delta_x)).sigmoid().squeeze() out_size = cfg.TRACK.MASK_OUTPUT_SIZE mask = mask.view(out_size, out_size).cpu().data.numpy() s = crop_box[2] / cfg.TRACK.INSTANCE_SIZE base_size = cfg.TRACK.BASE_SIZE stride = cfg.ANCHOR.STRIDE sub_box = [crop_box[0] + (delta_x - base_size/2) * stride * s, crop_box[1] + (delta_y - base_size/2) * stride * s, s * cfg.TRACK.EXEMPLAR_SIZE, s * cfg.TRACK.EXEMPLAR_SIZE] s = out_size / sub_box[2] im_h, im_w = img.shape[:2] back_box = [-sub_box[0] * s, -sub_box[1] * s, im_w*s, im_h*s] mask_in_img = self._crop_back(mask, back_box, (im_w, im_h)) polygon = self._mask_post_processing(mask_in_img) polygon = polygon.flatten().tolist() # calculate predict z pz_center_pos = np.array([bbox[0]+(bbox[2]-1)/2, bbox[1]+(bbox[3]-1)/2]) pz_size = np.array([bbox[2], bbox[3]]) w_z = pz_size[0] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(pz_size) h_z = pz_size[1] + cfg.TRACK.CONTEXT_AMOUNT * np.sum(pz_size) s_z = round(np.sqrt(w_z * h_z)) pz_channel_average = np.mean(img, axis=(0, 1)) z_crop = self.get_subwindow(img, pz_center_pos, cfg.TRACK.EXEMPLAR_SIZE, s_z, pz_channel_average) zf_cur = self.model.backbone(z_crop) if cfg.MASK.MASK: zf_cur = zf_cur[-1] if cfg.ADJUST.ADJUST: zf_cur = self.model.neck(zf_cur) return { 'bbox': bbox, 'best_score': best_score, 'mask': mask_in_img, 'polygon': polygon, 'zf_cur': zf_cur }
def main(): parser = argparse.ArgumentParser( "Visualize a set of output files from grid search") parser.add_argument("merged_grid_search_outputs") parser.add_argument("--outfile") parser.add_argument("--resolution", type=int, default=7) parser.add_argument("--best-n", type=int, default=10) parser.add_argument("--ignore-known-controllers", action="store_true") parser.add_argument("--plot", action="store_true") parser.add_argument("--viz", action="store_true") parser.add_argument("--save", action="store_true") parser.add_argument("--exclude-one-class", action="store_true") args = parser.parse_args() if args.plot or args.viz: style_dir = os.path.dirname(os.path.realpath(__file__)) style = os.path.join(style_dir, "mpl.style") import matplotlib.pyplot as plt plt.style.use(style) std_costs = np.zeros((args.resolution**6)) f = np.genfromtxt(args.merged_grid_search_outputs, skip_header=True) env_start_idx = 7 all_costs = f[:, env_start_idx:] if args.exclude_one_class: # skip the first 8 environments, they are for the 1-class scenarios all_costs = all_costs[:, 8:] mean_costs = np.mean(all_costs, axis=1) std_costs = np.std(all_costs, axis=1) params = f[:, 1:7] if args.outfile: writer = csv.writer(open(args.outfile, 'w'), delimiter=',') for i, (p, c, s) in enumerate(zip(params, mean_costs, std_costs)): writer.writerow([i, p, c, s]) if args.viz or args.save: axes_titles = [ r'$v_{l_0}$', r'$v_{r_0}$', r'$v_{l_1}$', r'$v_{r_1}$', r'$v_{l_2}$', r'$v_{r_2}$' ] for x_param in range(6): for y_param in range(x_param + 1, 6): plt.figure() plt.xlabel(axes_titles[x_param], fontsize=32) plt.ylabel(axes_titles[y_param], fontsize=32, rotation=0) labels = ['-1.0', '', '', '', '', '', '1.0'] plt.xticks(np.arange(7), labels, fontsize=24) plt.yticks(np.arange(7), labels, fontsize=24) shape = tuple([args.resolution] * 6) cost_image = np.ones((args.resolution, args.resolution)) * 1e24 for parameter_idx, cost in enumerate(mean_costs): indeces = np.unravel_index(parameter_idx, shape) row = indeces[x_param] col = indeces[y_param] if cost < cost_image[row, col]: cost_image[row, col] = cost plt.imshow(cost_image, cmap='Reds') if args.save: plt.savefig("{:d}_{:d}_grid_img.png".format( x_param, y_param)) # Sorting messes up plotting so we have to do this after sorted_cost_indeces = mean_costs.argsort(axis=0) mean_costs.sort() params = params[sorted_cost_indeces] std_costs = std_costs[sorted_cost_indeces] all_costs = all_costs[sorted_cost_indeces] print("Best Params, Index, Cost") print("{} {:0.0f} {:0.0f}".format(params[0], mean_costs[0], std_costs[0])) print("=" * 85) print("Good params") unknown_controllers = 0 for i, p in enumerate(params[:args.best_n]): # check if this matches the "patterns" of known segregating controllers # this never prints anything because turns out they all can be described this way if args.ignore_known_controllers: if p[0] > p[1]: if p[4] > p[5]: if p[3] >= p[2]: # left-hand circles segregating continue else: # slow clustering segregation? continue elif p[0] < p[1]: if p[5] > p[4]: if p[2] >= p[3]: # right-hand circles segregating continue else: #slow clustering segregation? continue unknown_controllers += 1 print(p, "{:d}th {:0.0f} {:0.0f}".format(i, mean_costs[i], std_costs[i])) best_costs = all_costs[0] inconclusive = True found_og_params = False for i, costs in enumerate(all_costs): t_value, p_value = stats.ttest_ind(best_costs, costs, equal_var=False) if np.allclose(params[i], [1, -2 / 3, 1 / 3, 1, 1, 0]): print("P value of old params: ", i, p_value, mean_costs[i], std_costs[i], mean_costs[0], std_costs[0], params[i]) found_og_params = True break if p_value < 0.05 and inconclusive: inconclusive = False print( "top {} params are not statistically significantly different". format(i)) if found_og_params and not inconclusive: break if inconclusive: print( "cannot conclude that the worst params are not identical mean to the best params..." ) if args.plot: plt.figure() c = mean_costs[::1000] N = len(c) plt.bar(np.arange(N), c, yerr=std_costs[::1000]) plt.ylabel("cost") plt.title("every 1000th parameter set, sorted from best to worst") plt.gca().set_xticklabels([]) if args.plot or args.viz: plt.show()
def image_phase_correlate(image, reference, real_filter=1, k_filter=1, shift_func='shift', verbose=False): """ Align image to reference by phase-correlation. Outputs shifted images and shift. Uses np.fft.rfft2 for ~2x speed improvement. Uses scipy.ndimage.shift() to shift the image and remove border pixels. NOT WORKING OR TESTED YET Note ---- image, reference and real_filter must all have the same shape (N, M). k_filter must have a shape that matches the np.fft.rfft2() of the other inputs: (N, M/2+1) Parameters ---------- image : ndarray A image as a 2D ndarray. reference : ndarray The reference image to align to. real_filter : ndarray, optional, default = 1 A real space filter applied to image and reference before calculating the shift. k_filter : ndarray, optional, default = 1 A Fourier space filter applied to the fourier transform of image and reference before calculating the cross-correlation. shift_func : str, default is 'shift' The function to use to shift the images. 'roll' uses np.roll and 'shift' uses ndimage.shift. verbose : bool Plots the cross-correlation using matplotlib for debugging purposes. Returns ------ : tuple, (ndarray, tuple) A tuple containing the shifted image and the shifts applied. """ output = None image_f = np.fft.rfft2((image - np.mean(image)) * real_filter) reference_f = np.fft.rfft2((reference - np.mean(reference)) * real_filter) xcor = abs(np.fft.irfft2(np.conj(image_f) * reference_f * k_filter)) pcor = xcor / (np.abs(xcor) + 0.001) if verbose: import matplotlib.pyplot as plt plt.imshow(np.fft.fftshift(pcor)) plt.title('imageCrossPhaseRealShift pcor') shifts = np.unravel_index(np.fft.fftshift(pcor).argmax(), pcor.shape) shifts = (shifts[0] - pcor.shape[0] / 2, shifts[1] - pcor.shape[1] / 2) shifts = [int(i) for i in shifts] # convert to integers if shift_func == 'shift': # shift image using ndimage.shift output = ndimage.interpolation.shift(image, shifts, order=0) elif shift_func == 'roll': # shift image using roll to be reversible output = np.roll(image, shifts[0], axis=0) output = np.roll(output, shifts[1], axis=1) return output, shifts
# divide by two because each edge is counted twice. i.e. entry i,j and j,i both count the same # edge start_weight = np.sum(network[network != 100000000]) / 2 print("start total weight: ", start_weight) ds = DisjointSet() for i in range(0,40): ds.make_set(i) new_weight = 0 while ds.get_parts() != 1: # find the minimum element in our network remaining. ind = np.unravel_index(np.argmin(network, axis=None), network.shape) # try to union the sets; if it fails, the two vertices are already connected via some # path so no need to add this edge. If it doesn't add this edge and count its weight. # Either way, set new edge weight to 10000000 so it isnt chosen again try: ds.union(ind[0], ind[1]) new_weight += network[ind] except ValueError: x = 1 # do nothing network[ind[0], ind[1]] = 10000000 network[ind[0], ind[1]] = 10000000
def get_next_insert_idx(cur_idx, prev_move_order, total): step_idx = 0 cumsum = np.cumsum(steps) cur_dir = next(prev_move_order) while True: #print "step idx {}".format(step_idx) #print "total: {}".format(total) if total < cumsum[step_idx]: index_offset = dir[cur_dir] # get next insert idxs else: cur_dir = next(prev_move_order) index_offset = dir[cur_dir] # print index_offset step_idx += 1 total += 1 cur_idx = (cur_idx[0] + index_offset[0], cur_idx[1] + index_offset[1]) #print "new_index {}".format(cur_idx) yield cur_idx idx_gen = get_next_insert_idx(mid_idx, cycle(move_order), 0) for i in np.arange(SIDE_LEN): in_id = next(idx_gen) window = field[in_id[0] - 1:in_id[0] + 2, in_id[1] - 1:in_id[1] + 2] field[in_id] = np.sum(window) print field[np.unravel_index(np.argmin(np.abs(field - SEARCH_VAL)), field.shape)]
def sourcery(ops): change_codes = True i0 = tic() U, sdmov, u = getSVDdata(ops) # get SVD components S, StU, StS = getStU(ops, U) Lyc, Lxc, nsvd = U.shape ops['Lyc'] = Lyc ops['Lxc'] = Lxc d0 = ops['diameter'] sig = np.ceil(d0 / 4) # smoothing constant # make array of radii values of size (2*d0+1,2*d0+1) rs, dy, dx = circleMask(d0) nsvd = U.shape[-1] nbasis = S.shape[-1] codes = np.zeros((0, nsvd), np.float32) LtU = np.zeros((0, nsvd), np.float32) LtS = np.zeros((0, nbasis), np.float32) L = np.zeros((Lyc, Lxc, 0), np.float32) # regress maps onto basis functions and subtract neuropil contribution neu = np.linalg.solve(StS, StU).astype('float32') Ucell = U - (S.reshape((-1, nbasis)) @ neu).reshape(U.shape) it = 0 ncells = 0 refine = -1 ypix, xpix, lam = [], [], [] while 1: if refine < 0: V, us = getVmap(Ucell, sig) if it == 0: vrem = morphOpen(V, rs <= 1.) V = V - vrem # make V more uniform if it == 0: V = V.astype('float64') # find indices of all maxima in +/- 1 range maxV = filters.maximum_filter(V, footprint=np.ones((3, 3)), mode='reflect') imax = V > (maxV - 1e-10) peaks = V[imax] # use the median of these peaks to decide if ROI is accepted thres = ops['threshold_scaling'] * np.median( peaks[peaks > 1e-4]) ops['Vcorr'] = V V = np.minimum(V, ops['Vcorr']) # add extra ROIs here n = ncells while n < ncells + 200: ind = np.argmax(V) i, j = np.unravel_index(ind, V.shape) if V[i, j] < thres: break yp, xp, la, ix, code = iter_extend(i, j, Ucell, us[i, j, :], change_codes=change_codes) codes = np.append(codes, np.expand_dims(code, axis=0), axis=0) ypix.append(yp) xpix.append(xp) lam.append(la) Ucell[ypix[n], xpix[n], :] -= np.outer(lam[n], codes[n, :]) yp, xp = extendROI(yp, xp, Lyc, Lxc, int(np.mean(d0))) V[yp, xp] = 0 n += 1 newcells = len(ypix) - ncells if it == 0: Nfirst = newcells L = np.append(L, np.zeros((Lyc, Lxc, newcells), 'float32'), axis=-1) LtU = np.append(LtU, np.zeros((newcells, nsvd), 'float32'), axis=0) LtS = np.append(LtS, np.zeros((newcells, nbasis), 'float32'), axis=0) for n in range(ncells, len(ypix)): L[ypix[n], xpix[n], n] = lam[n] LtU[n, :] = lam[n] @ U[ypix[n], xpix[n], :] LtS[n, :] = lam[n] @ S[ypix[n], xpix[n], :] ncells += newcells # regression with neuropil LtL = L.reshape((-1, ncells)).transpose() @ L.reshape((-1, ncells)) cellcode = np.concatenate((LtL, LtS), axis=1) neucode = np.concatenate((LtS.transpose(), StS), axis=1) codes = np.concatenate((cellcode, neucode), axis=0) Ucode = np.concatenate((LtU, StU), axis=0) codes = np.linalg.solve(codes + 1e-3 * np.eye((codes.shape[0])), Ucode).astype('float32') neu = codes[ncells:, :] codes = codes[:ncells, :] Ucell = U - (S.reshape((-1, nbasis)) @ neu + L.reshape( (-1, ncells)) @ codes).reshape(U.shape) # reestimate masks n, k = 0, 0 while n < len(ypix): Ucell[ypix[n], xpix[n], :] += np.outer(lam[n], codes[k, :]) ypix[n], xpix[n], lam[n], ix, codes[n, :] = iter_extend( ypix[n], xpix[n], Ucell, codes[k, :], refine, change_codes=change_codes) k += 1 if ix.sum() == 0: print('dropped ROI with no pixels') del ypix[n], xpix[n], lam[n] continue Ucell[ypix[n], xpix[n], :] -= np.outer(lam[n], codes[n, :]) n += 1 codes = codes[:n, :] ncells = len(ypix) L = np.zeros((Lyc, Lxc, ncells), 'float32') LtU = np.zeros((ncells, nsvd), 'float32') LtS = np.zeros((ncells, nbasis), 'float32') for n in range(ncells): L[ypix[n], xpix[n], n] = lam[n] if refine < 0: LtU[n, :] = lam[n] @ U[ypix[n], xpix[n], :] LtS[n, :] = lam[n] @ S[ypix[n], xpix[n], :] err = (Ucell**2).mean() print('ROIs: %d, cost: %2.4f, time: %2.4f' % (ncells, err, toc(i0))) it += 1 if refine == 0: break if refine == 2: # good place to get connected regions stat = [{ 'ypix': ypix[n], 'lam': lam[n], 'xpix': xpix[n] } for n in range(ncells)] stat = connected_region(stat, ops) # good place to remove ROIs that overlap, change ncells, codes, ypix, xpix, lam, L stat, ix = remove_overlaps(stat, ops, Lyc, Lxc) print('removed %d overlapping ROIs' % (len(ypix) - len(ix))) ypix = [stat[n]['ypix'] for n in range(len(stat))] xpix = [stat[n]['xpix'] for n in range(len(stat))] lam = [stat[n]['lam'] for n in range(len(stat))] L = L[:, :, ix] codes = codes[ix, :] ncells = len(ypix) if refine > 0: Ucell = Ucell + (S.reshape((-1, nbasis)) @ neu).reshape(U.shape) if refine < 0 and (newcells < Nfirst / 10 or it == ops['max_iterations']): refine = 3 U, sdmov = getSVDproj(ops, u) Ucell = U if refine >= 0: StU = S.reshape((Lyc * Lxc, -1)).transpose() @ Ucell.reshape( (Lyc * Lxc, -1)) #StU = np.reshape(S, (Lyc*Lxc,-1)).transpose() @ np.reshape(Ucell, (Lyc*Lxc, -1)) neu = np.linalg.solve(StS, StU).astype('float32') refine -= 1 Ucell = U - (S.reshape((-1, nbasis)) @ neu).reshape(U.shape) sdmov = np.reshape(sdmov, (Lyc, Lxc)) ops['sdmov'] = sdmov stat = [{ 'ypix': ypix[n], 'lam': lam[n] * sdmov[ypix[n], xpix[n]], 'xpix': xpix[n] } for n in range(ncells)] stat = postprocess(ops, stat, Ucell, codes) return ops, stat
return False def allShipSunk(gameBoard, hits, boardSize): sunk = True for y in range(boardSize): for x in range(boardSize): if shipsOnGameBoard[x][y] == 1 and hits[x][y] != 1: sunk = False if sunk == False: return False else: return True while allShipSunk(gameBoard, hits, boardSize) is False: algorithm = hunt(gameBoard, boardSize) nextTest = unravel_index(algorithm.argmax(), algorithm.shape) print("next shot:", nextTest) if any(nextTest in y for y in shipList): print("hit!!!!") hits[nextTest] = 1 numberOfTurns += 1 print("turn:", numberOfTurns) targetHits = np.array([[0 for i in range(boardSize)] for j in range(boardSize)]) targetMisses = np.array([[0 for i in range(boardSize)] for j in range(boardSize)]) targetHits[nextTest] = 1 hit = target(targetHits, algorithm, targetMisses) gameBoard = hit + gameBoard else: print("miss!!")
def expand_and_prune_with_thresholds(self, expand_thr, prune_thr): for name, module in self.model.named_modules(): if isinstance(module, MutatingModule): ncc = module.normalized_cross_correlation() splitted = False for name, module in self.model.named_modules(): if not isinstance(module, MutatingModule): continue if module.fixed_feature_count: continue # print large weights: (max_val, max_idx) = torch.max(torch.abs(module.weight.view(-1)), dim=0) if torch.max(max_val.data) > 2: idx = np.unravel_index(max_idx.data[0], module.weight.size()) print("maximum weight at index ", idx, " with value ", max_val.data[0]) if len(module.output_tied_modules) > 0: all_nccs = [module.current_ncc] + [m.current_ncc for m in module.output_tied_modules] ncc_tensor = torch.abs(torch.stack(all_nccs)) ncc = torch.mean(ncc_tensor, dim=0) else: ncc = module.current_ncc offset = 0 feature_number = 0 while feature_number < ncc.size(0): # weighted_value = ncc[feature_number] / (math.sqrt(module.in_channels)) # weighted_value = ncc[feature_number] / (math.log(module.in_channels)+1) weighted_value = ncc[feature_number] if abs(weighted_value) > expand_thr: print("in ", name, ", split feature number ", feature_number + offset, ", ncc: ", weighted_value) ncc = module.split_feature(feature_number=feature_number + offset) all_modules = [module] + module.output_tied_modules [m.normalized_cross_correlation() for m in all_modules] splitted = True self.allow_pruning = True feature_number = 0 continue if abs(weighted_value) < prune_thr and weighted_value != 0 and self.allow_pruning: if feature_number >= module.out_channels: continue print("in ", name, ", prune feature number ", feature_number) ncc = module.prune_feature(feature_number=feature_number + offset) all_modules = [module] + module.output_tied_modules [m.normalized_cross_correlation() for m in all_modules] splitted = True feature_number = 0 continue feature_number += 1 if splitted: # self.optimizer = optim.SGD(self.model.parameters(), # lr=self.lr, # momentum=self.momentum, # weight_decay=self.weight_decay) #self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr, weight_decay=self.weight_decay) print("new parameter count: ", self.parameter_count()) # print("test result after expanding: ") # self.test(epoch) else: print("No feature changed enough to split") return splitted
def make_parcellation(subject, dest_dir='parcellation', roi_mask_file=None): """ Perform a functional parcellation from input fmri data Return: parcellation file name (str) """ # Loading names for folders and files # - T maps (input) #func_files = glob(op.join(op.join(op.join('./', subject), \ # 't_maps'), 'BOLD*nii')) #func_files = glob(op.join('./', subject, 'ASLf', 'spm_analysis', \ # 'Tmaps*img')) func_files = glob(op.join('./', subject, 'ASLf', 'spm_analysis', \ 'spmT*img')) print 'Tmap files: ', func_files # - Mask (input) #spm_mask_file = op.join(spm_maps_dir, 'mask.img') mask_dir = op.join('./', subject, 'preprocessed_data') if not op.exists(mask_dir): os.makedirs(mask_dir) mask_file = op.join(mask_dir, 'mask.nii') mask = op.join(mask_dir, 'rcut_tissue_mask.nii') volume = op.join('./', subject, 'ASLf', 'funct', 'coregister', \ 'mean' + subject + '_ASLf_correctionT1_0001.nii') make_mask(mask, volume, mask_file) # - parcellation (output) parcellation_dir = op.join('./', subject, dest_dir) if not op.exists(parcellation_dir): os.makedirs(parcellation_dir) pfile = op.join(parcellation_dir, 'parcellation_func.nii') # Parcellation from pyhrf.parcellation import make_parcellation_from_files #make_parcellation_from_files(func_files, mask_file, pfile, # nparcels=200, method='ward_and_gkm') # Masking with a ROI so we just consider parcels inside # a certain area of the brain #if roi_mask_file is not None: if 0: #for ip in np.array([11, 51, 131, 194]): #ip = 200 #print 'Masking parcellation with roi_mask_file: ', roi_mask_file print 'Masking ROI: ', ip pfile_masked = op.join(parcellation_dir, 'parcellation_func_masked_roi' + str(ip) + '.nii') from pyhrf.ndarray import xndarray parcellation = xndarray.load(pfile) #m = xndarray.load(roi_mask_file) #parcels_to_keep = np.unique(parcellation.data * m.data) masked_parcellation = xndarray.xndarray_like(parcellation) #for ip in parcels_to_keep: # masked_parcellation.data[np.where(parcellation.data==ip)] = ip masked_parcellation.data[np.where(parcellation.data==ip)] = ip masked_parcellation.save(pfile_masked) from pyhrf.ndarray import xndarray for tmap in func_files: func_file_i = xndarray.load(tmap) func_data = func_file_i.data #func_file_i.data[np.where(func_data<0.1*func_data.max())] = 0 parcellation = xndarray.load(pfile) print func_data.max() print func_data.argmax() #ip = parcellation.data[func_data.argmax()] ip = parcellation.data[np.unravel_index(func_data.argmax(), parcellation.data.shape)] print ip.shape masked_parcellation = xndarray.xndarray_like(parcellation) print masked_parcellation.data.shape print parcellation.data.shape masked_parcellation.data[np.where(parcellation.data==ip)] = ip masked_parcellation.save(tmap[:-4] + '_parcelmax.nii') return pfile