def prepare_train_data(data): labels = [] flattened = [] features = [] images = [] images_norm = [0] * 7 for i in range(7): labels.append(np.array([int(el[0]) for el in data[0][i][0][0][1]])) # labels.append(1.0 * data[0][i][0][0][1]) images.append(1.0 * data[0][i][0][0][0]) #print images[0].shape #print labels[0].shape # images.append(1.0 * np.array([float(el[0]) for el in data[0][i][0][0][0]])) reshaped = np.reshape(images[i], (784, labels[i].shape[0])) reshaped = np.transpose(reshaped) #Center around 0, normalized mean = np.mean(reshaped, axis = 1) std = np.std(reshaped, axis =1) for k in xrange(std.shape[0]): if std[k] == 0: std[k] = 1 reshaped = reshaped - mean[:,None] reshaped_normalized = reshaped / std[:,None] #reshaped_normalized = 1.0 * reshaped / sum(reshaped) reshaped_normalized = np.transpose(reshaped_normalized) flattened.append(reshaped_normalized) features.append(np.transpose(flattened[i])) return (features, labels)
def read_abinit(filename): with open(filename) as f: abinit_in = AbinitIn(f.readlines()) tags = abinit_in.get_variables() acell = tags['acell'] rprim = tags['rprim'].T scalecart = tags['scalecart'] lattice = rprim * acell if scalecart is not None: for i in range(3): lattice[i] *= scalecart[i] if tags['xcart'] is not None: pos_bohr = np.transpose(tags['xcart']) positions = np.dot(np.linalg.inv(lattice), pos_bohr).T elif tags['xangst'] is not None: pos_bohr = np.transpose(tags['xangst']) / Bohr positions = np.dot(np.linalg.inv(lattice), pos_bohr).T elif tags['xred'] is not None: positions = tags['xred'] numbers = [tags['znucl'][x - 1] for x in tags['typat']] return Atoms(numbers=numbers, cell=lattice.T, scaled_positions=positions)
def spatial_batchnorm_backward(dout, cache): """ Computes the backward pass for spatial batch normalization. Inputs: - dout: Upstream derivatives, of shape (N, C, H, W) - cache: Values from the forward pass Returns a tuple of: - dx: Gradient with respect to inputs, of shape (N, C, H, W) - dgamma: Gradient with respect to scale parameter, of shape (C,) - dbeta: Gradient with respect to shift parameter, of shape (C,) """ dx, dgamma, dbeta = None, None, None ############################################################################# # TODO: Implement the backward pass for spatial batch normalization. # # # # HINT: You can implement spatial batch normalization using the vanilla # # version of batch normalization defined above. Your implementation should # # be very short; ours is less than five lines. # ############################################################################# N,C,H,W = dout.shape dout_temp = np.transpose(dout, [0, 2, 3, 1]).reshape(N*H*W, C) dx, dgamma, dbeta = batchnorm_backward(dout_temp, cache) dx = np.transpose(dx.reshape(N, H, W, C), [0, 3, 1, 2]) ############################################################################# # END OF YOUR CODE # ############################################################################# return dx, dgamma, dbeta
def store_models(self, specs, ivar): self.models = n.zeros( (specs.shape) ) for i in xrange(self.models.shape[0]): minloc = n.unravel_index( self.zchi2arr[i].argmin(), self.zchi2arr[i].shape ) pmat = n.zeros( (specs.shape[-1],self.npoly+1) ) this_temp = self.templates[minloc[:-1]] pmat[:,0] = this_temp[(minloc[-1]*self.npixstep) + self.pixoffset:(minloc[-1]*self.npixstep) + self.pixoffset+specs.shape[-1]] polyarr = poly_array(self.npoly, specs.shape[-1]) pmat[:,1:] = n.transpose(polyarr) ninv = n.diag(ivar[i]) try: # Some eBOSS spectra have ivar[i] = 0 for all i ''' f = n.linalg.solve( n.dot(n.dot(n.transpose(pmat),ninv),pmat), n.dot( n.dot(n.transpose(pmat),ninv),specs[i]) ) ''' f = nnls( n.dot(n.dot(n.transpose(pmat),ninv),pmat), n.dot( n.dot(n.transpose(pmat),ninv),specs[i]) );\ f = n.array(f)[0] self.models[i] = n.dot(pmat, f) except Exception as e: self.models[i] = n.zeros(specs.shape[-1]) print "Exception: %r" % r
def updateParameters(self, articlePicked, click, userID): self.counter +=1 self.Wlong = vectorize(self.W) featureDimension = len(articlePicked.featureVector) T_X = vectorize(np.outer(articlePicked.featureVector, self.W.T[userID])) self.A += np.outer(T_X, T_X) self.b += click*T_X self.AInv = np.linalg.inv(self.A) self.UserTheta = matrixize(np.dot(self.AInv, self.b), len(articlePicked.featureVector)) Xi_Matirx = np.zeros(shape = (featureDimension, self.userNum)) Xi_Matirx.T[userID] = articlePicked.featureVector W_X = vectorize( np.dot(np.transpose(self.UserTheta), Xi_Matirx)) self.batchGradient +=evaluateGradient(W_X, click, self.Wlong, self.lambda_, self.regu ) if self.counter%self.windowSize ==0: self.Wlong -= 1/(float(self.counter/self.windowSize)+1)*self.batchGradient self.W = matrixize(self.Wlong, self.userNum) self.W = normalize(self.W, axis=0, norm='l1') #print 'SVD', self.W self.batchGradient = np.zeros(self.userNum*self.userNum) # Use Ridge regression to fit W ''' plt.pcolor(self.W_b) plt.colorbar plt.show() ''' if self.W.T[userID].any() <0 or self.W.T[userID].any()>1: print self.W.T[userID] self.CoTheta = np.dot(self.UserTheta, self.W) self.BigW = np.kron(np.transpose(self.W), np.identity(n=len(articlePicked.featureVector))) self.CCA = np.dot(np.dot(self.BigW , self.AInv), np.transpose(self.BigW)) self.BigTheta = np.kron(np.identity(n=self.userNum) , self.UserTheta)
def detrend(dataset,tvalues,order=1): """Removes polinomial trends Given a linear input dataset, detrend it removing a polynomial of order N of type trend(t)=\sum_{i=0}^N a_i t^i Arguments: 'dataset' -- Numpy array with the data to be detrended (along its _first_ dimension) 'tvalues' -- Numpy array with the values of the time coordinate. Its length must be that of the first dimension of 'dataset' Optional arguments: 'order' -- The order of the polinomial to be removed. Defaults to 1 (linear) The function returns the detrended (LINEAR) dataset and the parameters of the trend in a tuple. Of course, when removing a polynomial of the mentioned type, the mean is also removed !!! """ T=numpy.ones((len(dataset),)+(order+1,),numpy.float64) for iord in xrange(1,len(T[0])): T[:,iord]=T[:,iord-1]*tvalues V=mm(numpy.transpose(T),T) Tx=mm(numpy.transpose(T),dataset) A=mm(numpy.linalg.inv(V),Tx) trendterm=mm(T,A) residual=dataset-trendterm return residual,A
def _add_solids(self, X, Y, C): """ Draw the colors using :meth:`~matplotlib.axes.Axes.pcolormesh`; optionally add separators. """ if self.orientation == "vertical": args = (X, Y, C) else: args = (np.transpose(Y), np.transpose(X), np.transpose(C)) kw = dict(cmap=self.cmap, norm=self.norm, alpha=self.alpha, edgecolors="None") # Save, set, and restore hold state to keep pcolor from # clearing the axes. Ordinarily this will not be needed, # since the axes object should already have hold set. _hold = self.ax.ishold() self.ax.hold(True) col = self.ax.pcolormesh(*args, **kw) self.ax.hold(_hold) # self.add_observer(col) # We should observe, not be observed... if self.solids is not None: self.solids.remove() self.solids = col if self.dividers is not None: self.dividers.remove() self.dividers = None if self.drawedges: linewidths = (0.5 * mpl.rcParams["axes.linewidth"],) self.dividers = collections.LineCollection( self._edges(X, Y), colors=(mpl.rcParams["axes.edgecolor"],), linewidths=linewidths ) self.ax.add_collection(self.dividers)
def test_arclength_half_circle(): """ Here we define the tests for the lenght computer of our ArcLengthParametrizer, we try it with a half a circle and a fan. We test it both in 2d and 3d.""" # Number of interpolation points minus one n = 5 toll = 1.e-6 points = np.linspace(0, 1, (n+1) ) R = 1 P = 1 control_points_2d = np.asmatrix(np.zeros([n+1,2]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)] control_points_2d[:,0] = np.transpose(np.matrix([R*np.cos(1 * i * np.pi / (n + 1))for i in range(n+1)])) control_points_2d[:,1] = np.transpose(np.matrix([R*np.sin(1 * i * np.pi / (n + 1))for i in range(n+1)])) control_points_3d = np.asmatrix(np.zeros([n+1,3]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)] control_points_3d[:,0] = np.transpose(np.matrix([R*np.cos(1 * i * np.pi / (n + 1))for i in range(n+1)])) control_points_3d[:,1] = np.transpose(np.matrix([R*np.sin(1 * i * np.pi / (n + 1))for i in range(n+1)])) control_points_3d[:,2] = np.transpose(np.matrix([P*i for i in range(n+1)])) vsl = AffineVectorSpace(UniformLagrangeVectorSpace(n+1),0,1) dummy_arky_2d = ArcLengthParametrizer(vsl, control_points_2d) dummy_arky_3d = ArcLengthParametrizer(vsl, control_points_3d) length2d = dummy_arky_2d.compute_arclength()[-1,1] length3d = dummy_arky_3d.compute_arclength()[-1,1] # print (length2d) # print (n * np.sqrt(2)) l2 = np.pi * R l3 = 2 * np.pi * np.sqrt(R * R + (P / (2 * np.pi)) * (P / (2 * np.pi))) print (length2d, l2) print (length3d, l3) assert (length2d - l2) < toll assert (length3d - l3) < toll
def test_reparametrization(): """ Here we define the tests for reparametrizer of our ArcLengthParametrizer, we try it with a half a circle and a fan. We test it both in 2d and 3d.""" R = 1 P = 1 toll = 1.e-6 n = 10 ii = [0.5,0.8,4,6,7,8,9,9.6,10,10.1,11] control_points_3d = np.asmatrix(np.zeros([n+1,3]))#[np.array([R*np.cos(5*i * np.pi / (n + 1)), R*np.sin(5*i * np.pi / (n + 1)), P * i]) for i in range(0, n+1)] control_points_3d[:,0] = np.transpose(np.matrix([R*np.cos(5*i * np.pi / (n + 1))for i in ii])) control_points_3d[:,1] = np.transpose(np.matrix([R*np.sin(5*i * np.pi / (n + 1))for i in ii])) control_points_3d[:,2] = np.transpose(np.matrix([P*i for i in range(n+1)])) #control_points_3d[3,:] += 32 #print control_points_3d[0] vsl = AffineVectorSpace(UniformLagrangeVectorSpace(n+1),0,1) arky = ArcLengthParametrizer(vsl, control_points_3d) new_control_points_3d = arky.reparametrize() new_arky = ArcLengthParametrizer(vsl, new_control_points_3d) new_new_control_points_3d = arky.reparametrize() tt = np.linspace(0, 1, 128) new_new_vals = vsl.element(new_new_control_points_3d)(tt) #print vals new_vals = vsl.element(new_control_points_3d)(tt) #print vals.shape, new_vals.shape assert np.amax(np.abs(new_new_vals-new_vals)) < toll
def display_grid(grid, **kwargs): fig = plt.figure() plt.axes().set_aspect('equal') if kwargs.get('mark_core_cells', True): core_cell_coords = grid._cell_nodes[1:-1, 1:-1] cellx, celly = core_cell_coords[:, :, 0], core_cell_coords[:, :, 1] plt.plot(cellx, celly, '-o', np.transpose(cellx), np.transpose(celly), '-o', color='red') if kwargs.get('mark_boundary_cells', True): boundary_cell_coords = grid._cell_nodes[0, :], \ grid._cell_nodes[-1, :], \ grid._cell_nodes[1:-1, 0], \ grid._cell_nodes[1:-1, -1] for coords in boundary_cell_coords: plt.plot(coords[:, 0], coords[:, 1], '-x', color='blue') if kwargs.get('show', False): plt.show() f = BytesIO() plt.savefig(f) return f
def sample_line_segment_mm_s(start_xy_mm, end_xy_mm, dt_s, mW=None, max_mm=5.0): """ Given a line segment in mm space, map it to galvo space. To make the line straight in mm space, samples may be added to more-closely approximate a straight line. Returns: An array of shape nx3 (if mW is None) or nx4 (if mW is not None) of points time deltas in mm and seconds, excluding start_xy_mm and including end_xy_mm, possibly including samples along the way. """ import FLP from numpy.linalg import norm dist_mm = norm(np.asarray(end_xy_mm) - start_xy_mm) if dist_mm <= max_mm: if mW is None: return np.array((tuple(end_xy_mm) + (dt_s,),)) # Just the end sample. else: return np.array((tuple(end_xy_mm) + (dt_s, mW),)) # Just the end sample. samples_s = np.linspace(0, dt_s, np.ceil(dist_mm / max_mm) + 1) timeRange_s = (0, dt_s) if mW is None: return np.transpose([np.interp(samples_s[1:], timeRange_s, (start_xy_mm[0], end_xy_mm[0])), np.interp(samples_s[1:], timeRange_s, (start_xy_mm[1], end_xy_mm[1])), np.diff(samples_s)]) else: return np.transpose([np.interp(samples_s[1:], timeRange_s, (start_xy_mm[0], end_xy_mm[0])), np.interp(samples_s[1:], timeRange_s, (start_xy_mm[1], end_xy_mm[1])), np.diff(samples_s), mW * np.ones_like(samples_s[1:])])
def test_cl(): ctx = cl.create_some_context() # (interactive=False) # print 'ctx', ctx queue = cl.CommandQueue(ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) f = open("part1.cl", "r") fstr = "".join(f.readlines()) program = cl.Program(ctx, fstr).build() mf = cl.mem_flags cameraPos = np.array([0, 6, -1, 0]) invView = la.inv(look_at((0, 6, -1), (0, 1, 1), (0, 1, 0))) invProj = la.inv(perspective(60, 1, 1, 1000)) print "view", invView print "proj", invProj viewParamsData = ( cameraPos.flatten().tolist() + np.transpose(invView).flatten().tolist() + np.transpose(invProj).flatten().tolist() ) # print 'vpd', viewParamsData viewParams = struct.pack("4f16f16f", *viewParamsData) viewParams_buf = cl.Buffer(ctx, mf.READ_ONLY | mf.COPY_HOST_PTR, hostbuf=viewParams) num_pixels = 1000 * 1000 # setup opencl dest = np.ndarray((1000, 1000, 4), dtype=np.float32) dest_buf = cl.Buffer(ctx, mf.WRITE_ONLY, dest.nbytes) local_shape = (8, 8) # run kernel evt = program.part1(queue, (dest.shape[0], dest.shape[1]), None, viewParams_buf, dest_buf) # evt = program.part1(queue, dest.shape, None, dest_buf) cl.enqueue_read_buffer(queue, dest_buf, dest).wait() print "time", (evt.profile.end - evt.profile.start) * 0.000001, "ms" return dest
def get_moments(v,m,n=100): """ Get the first n moments of a certain vector using the Chebychev recursion relations""" mus = np.array([0.0j for i in range(2*n)]) # empty arrray for the moments a = v.copy() # first vector am = v.copy() # zero vector a = m*v # vector number 1 bk = (np.transpose(np.conjugate(v))*v)[0,0] # scalar product bk1 = (np.transpose(np.conjugate(v))*a)[0,0] # scalar product mus[0] = bk # mu0 mus[1] = bk1 # mu1 for i in range(1,n): ap = 2*m*a - am # recursion relation bk = (np.transpose(np.conjugate(a))*a)[0,0] # scalar product bk1 = (np.transpose(np.conjugate(ap))*a)[0,0] # scalar product mus[2*i] = 2.*bk mus[2*i+1] = 2.*bk1 am = a +0. # new variables a = ap+0. # new variables mu0 = mus[0] # first mu1 = mus[1] # second for i in range(1,n): mus[2*i] += - mu0 mus[2*i+1] += -mu1 return mus
def _train(self, input_list, target_list): """Internal training method for our neural network.""" # 1) Get transposed arrays - one column. inputs = numpy.array(input_list, ndmin=2).T # 2) Forward propogation. hidden_inputs = numpy.dot(self.weights[ITH], inputs) hidden_outputs = self.activation_function(hidden_inputs) final_inputs = numpy.dot(self.weights[HTO], hidden_outputs) final_outputs = self.activation_function(final_inputs) # 3) Get error. targets = numpy.array(target_list, ndmin=2).T output_errors = targets - final_outputs hidden_errors = numpy.dot(self.weights[HTO].T, output_errors) self.weights[HTO] += self._learning_rate * numpy.dot( (output_errors * final_outputs * (1.0 - final_outputs)), numpy.transpose(hidden_outputs)) self.weights[ITH] += self._learning_rate * numpy.dot( (hidden_errors * hidden_outputs * (1.0 - hidden_outputs)), numpy.transpose(inputs))
def __new__(cls, input_array, tol=1e-3): """ Create an ElasticTensor object. The constructor throws an error if the shape of the input_matrix argument is not 3x3x3x3, i. e. in true tensor notation. Issues a warning if the input_matrix argument does not satisfy standard symmetries. Note that the constructor uses __new__ rather than __init__ according to the standard method of subclassing numpy ndarrays. Args: input_array (3x3x3x3 array-like): the 3x3x3x3 array-like representing the elastic tensor tol (float): tolerance for initial symmetry test of tensor """ obj = TensorBase(input_array).view(cls) if obj.shape != (3, 3, 3, 3): raise ValueError("Default elastic tensor constructor requires " "input to be the true 3x3x3x3 representation. " "To construct from an elastic tensor from " "6x6 Voigt array, use ElasticTensor.from_voigt") if not ((obj - np.transpose(obj, (1, 0, 2, 3)) < tol).all() and (obj - np.transpose(obj, (0, 1, 3, 2)) < tol).all() and (obj - np.transpose(obj, (1, 0, 3, 2)) < tol).all() and (obj - np.transpose(obj, (3, 2, 0, 1)) < tol).all()): warnings.warn("Input elasticity tensor does " "not satisfy standard symmetries") return obj
def search(self, text, isSvd = False): if isSvd: print "Search with SVD" else: print "Search with LDA" start = time.time() vector = text.split() cleanedVector = cleanVector(vector) #cleanedVector = self.wordsToAsk print cleanedVector if isSvd: bagOfWords, indices = createBagOfWordsFromVector(cleanedVector, self.amountOfWords, self.dictOfWords, self.idfs) b = fasterCorrelations(self.matrix, indices, bagOfWords, self.amountOfFiles) else: bagOfWords = createBagOfWordsForLDA(cleanedVector, self.amountOfWords, self.dictOfWords) #print 'bow shape:' #print bagOfWords.shape #print 'topic word shape:' #print np.transpose(self.model.topic_word_).shape res = np.dot(bagOfWords, np.transpose(self.model.topic_word_)) #print 'first mul shape:' #print res.shape #print res #print 'doc topic shape:' #print np.transpose(self.model.doc_topic_).shape res2 = np.dot(res, np.transpose(self.model.doc_topic_)) #print 'final shape:' #print res2.shape bestValues = sorted(list(res2[0]), reverse=True)[:5] b = [] for bestVal in bestValues: for ind, val in enumerate(list(res2[0])): if val == bestVal: b.append((ind, val)) results = [] dbMan = DatabaseManager() for x in b: results.append(dbMan.get_link(self.listOfArticles[x[0]])) for res in results: print res.url print res.title stop = time.time() return results, stop - start
def getTopOutlierIndexAsc(data, per): res = np.argsort(np.transpose(data)) resD = np.sort(np.transpose(data)) lenth = len(data) topNum = int(np.floor(per*lenth))+1 res = res[0] return res[:topNum]
def lms2rgb(lms_Nx3, conversionMatrix=None): """Convert from cone space (Long, Medium, Short) to RGB. Requires a conversion matrix, which will be generated from generic Sony Trinitron phosphors if not supplied (note that you will not get an accurate representation of the color space unless you supply a conversion matrix) usage:: rgb_Nx3 = lms2rgb(dkl_Nx3(el,az,radius), conversionMatrix) """ lms_3xN = numpy.transpose(lms_Nx3)#its easier to use in the other orientation! if conversionMatrix==None: cones_to_rgb = numpy.asarray([ \ #L M S [ 4.97068857, -4.14354132, 0.17285275],#R [-0.90913894, 2.15671326, -0.24757432],#G [-0.03976551, -0.14253782, 1.18230333]#B ]) logging.warning('This monitor has not been color-calibrated. Using default LMS conversion matrix.') else: cones_to_rgb=conversionMatrix rgb_to_cones = numpy.linalg.pinv(cones_to_rgb)#get inverse rgb = numpy.dot(cones_to_rgb, lms_3xN) return numpy.transpose(rgb)#return in the shape we received it
def test(self): # Expected in_channels = 3 in_dim = 11 out_channels = 5 out_dim = (in_dim/2 + 1) img = np.arange(0,in_dim*in_dim*in_channels*1, dtype=np.float32) img = np.reshape(img,[in_dim,in_dim,in_channels,1]) filter = np.arange(0,3*3*in_channels*out_channels, dtype=np.float32) filter = np.reshape(filter,[3,3,in_channels,out_channels]) bias = np.zeros([5]) expected = np.zeros([out_dim,out_dim,out_channels]) for och in range(out_channels): tmp = np.zeros([out_dim,out_dim,1]) for ich in range(in_channels): imgslice = np.reshape(img[:,:,ich,0],[in_dim,in_dim]) filterslice = np.reshape(filter[:,:,ich,och],[3,3]) tmp += np.reshape(convolve(imgslice,filterslice,mode='constant',cval = 0.0)[::2,::2] , [out_dim, out_dim, 1]) expected[:,:,och] = np.squeeze(tmp) + bias[och] # test owlimg = owl.from_numpy(np.transpose(img)) owlfilter = owl.from_numpy(np.transpose(filter)) owlbias = owl.from_numpy(bias) convolver = owl.conv.Convolver(1,1,2,2) test = convolver.ff(owlimg, owlfilter, owlbias) print 'Expected\n',expected print "Actual\n",test.to_numpy() self.assertTrue(np.allclose(expected, test))
def calc_alm_chisq_fromfits(almfile, clfile): alm = pyfits.open(almfile)[0].data cls = pyfits.open(clfile)[0].data numiter = alm.shape[0] numchain = alm.shape[1] alm = np.reshape(alm, (numiter*numchain,alm.shape[2], alm.shape[3], alm.shape[4], alm.shape[5])) alm = alm[:, :, :, 2:, :] cls = cls[1:] cls = np.reshape(cls, (numiter * numchain, cls.shape[2], cls.shape[3])) if alm.shape[1] == 3: cls = np.concatenate((cls[:, 0:1, :], cls[:, 3:4, :], cls[:, 5:6, :]), 1) elif alm.shape[1] == 1: cls = cls[:, 0:1, :] cls = cls[:, :, 2:] cls = np.transpose(cls).copy() alm = np.transpose(alm).copy() chisq = np.zeros(cls.shape) for i in range(cls.shape[0]): l = i + 2 for m in range(l): if m == 0: chisq[i, :, :] += alm[0, i, m, :, :] ** 2 else: chisq[i, :, :] += np.sum(2 * alm[:, i, m, :, :] ** 2, 0) chisq[i, :, :] = chisq[i, :, :] / cls[i, :, :] / (2 * l + 1) * (l * (l + 1)) / (2 * np.pi) return chisq
def permute_2d(m, p): """Performs 2D permutation of matrix m according to p.""" return m[p][:, p] # unused below m_t = np.transpose(m) r_t = np.take(m_t, p, axis=0) return np.take(np.transpose(r_t), p, axis=0)
def _csd_array(x, sfreq, window_fun, eigvals, freq_mask, freq_mask_mt, n_fft, mode, mt_adaptive): """Calculate Fourier transform using multitaper module. The arguments correspond to the values in `compute_csd_epochs` and `csd_array`. """ x_mt, _ = _mt_spectra(x, window_fun, sfreq, n_fft) if mt_adaptive: # Compute adaptive weights _, weights = _psd_from_mt_adaptive(x_mt, eigvals, freq_mask, return_weights=True) # Tiling weights so that we can easily use _csd_from_mt() weights = weights[:, np.newaxis, :, :] weights = np.tile(weights, [1, x_mt.shape[0], 1, 1]) else: # Do not use adaptive weights if mode == 'multitaper': weights = np.sqrt(eigvals)[np.newaxis, np.newaxis, :, np.newaxis] else: # Hack so we can sum over axis=-2 weights = np.array([1.])[:, np.newaxis, np.newaxis, np.newaxis] x_mt = x_mt[:, :, freq_mask_mt] # Calculating CSD # Tiling x_mt so that we can easily use _csd_from_mt() x_mt = x_mt[:, np.newaxis, :, :] x_mt = np.tile(x_mt, [1, x_mt.shape[0], 1, 1]) y_mt = np.transpose(x_mt, axes=[1, 0, 2, 3]) weights_y = np.transpose(weights, axes=[1, 0, 2, 3]) csds = _csd_from_mt(x_mt, y_mt, weights, weights_y) return csds
def wide_dataset_large(): print("Reading in Arcene training data for binomial modeling.") trainDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train_labels.labels"), delimiter=' ') trainDataResponse = np.where(trainDataResponse == -1, 0, 1) trainDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_train.data"), delimiter=' ') xtrain = np.transpose(trainDataFeatures).tolist() ytrain = trainDataResponse.tolist() trainData = h2o.H2OFrame.fromPython([ytrain]+xtrain) trainData[0] = trainData[0].asfactor() print("Run model on 3250 columns of Arcene with strong rules off.") model = H2OGeneralizedLinearEstimator(family="binomial", lambda_search=False, alpha=1) model.train(x=range(1,3250), y=0, training_frame=trainData) print("Test model on validation set.") validDataResponse = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid_labels.labels"), delimiter=' ') validDataResponse = np.where(validDataResponse == -1, 0, 1) validDataFeatures = np.genfromtxt(pyunit_utils.locate("smalldata/arcene/arcene_valid.data"), delimiter=' ') xvalid = np.transpose(validDataFeatures).tolist() yvalid = validDataResponse.tolist() validData = h2o.H2OFrame.fromPython([yvalid]+xvalid) prediction = model.predict(validData) print("Check performance of predictions.") performance = model.model_performance(validData) print("Check that prediction AUC better than guessing (0.5).") assert performance.auc() > 0.5, "predictions should be better then pure chance"
def prettyPrint(self, include_header=False, transpose_data=False, column_limit=None, col_sep="\t"): """Returns a string method of the data and character order. include_header: include charcter order or not transpose_data: data as is (rows are positions) or transposed (rows are characters) to line it up with an alignment column_limit = int, maximum number of columns displayed col_sep = string, column separator """ h = self.CharOrder d = self.Data if column_limit is None: max_col_idx = d.shape[1] else: max_col_idx = column_limit if include_header and not transpose_data: r = [h] + d.tolist() elif include_header and transpose_data: r = [[x] + y for x, y in zip(h, transpose(d).tolist())] elif transpose_data: r = transpose(d).tolist() else: r = d.tolist() # resize the result based on the column limit if column_limit is not None: r = [row[:column_limit] for row in r] # nicely format the table content, discard the header (already included) if r: new_header, formatted_res = formattedCells(r) else: formatted_res = r return "\n".join([col_sep.join(map(str, i)) for i in formatted_res])
def _forward( self, inputs ): """ Forward phase. Returns the calculated outputs. """ ones = -ny.ones( ( len( inputs ), 1 ) ) # Activation in hidden layer self.hidden = ny.dot( inputs, self.weights_layer1 ) self.hidden = 1.0 / ( 1.0 + ny.exp( -self.beta * self.hidden ) ) self.hidden = ny.concatenate( ( self.hidden, ones ), axis = 1 ) # Acitvation in output layer outputs = ny.dot( self.hidden, self.weights_layer2 ) if self.outtype == "linear": pass elif self.outtype == "logistic": outputs = 1.0 / ( 1.0 + ny.exp( -self.beta * outputs ) ) elif self.outtype == 'softmax': normalizers = ny.sum( ny.exp( outputs ), axis = 1 ) * ny.ones( ( 1, ny.shape( outputs )[0] ) ) outputs = ny.transpose( ny.transpose( ny.exp( outputs ) ) / normalizers ) else: print "ERROR: Unknown outtype = %s" % outtype return outputs
def test_putting_filters_back_together_from_fitted_basis_function_coefficients(self): """ Make sure we can properly put filters back together if we have found their coefficients. """ cc = np.concatenate tsm = time_series.Munger() tsm.delay = 10 # we will assume there is a 10 timestep delay between input and response # we will assume x1 is used directly and x2, x3, and y are filtered, and that their # filters are represented by sums of sinusoidal basis functions # (the details don't matter as we're just making sure all the arrays are getting # rearranged correctly) t = np.linspace(0, np.pi, 20) sin_basis = np.transpose([np.sin(t), np.sin(2*t), np.sin(3*t), np.sin(4*t)]) t_short = t[:10] sin_basis_short = np.transpose([np.sin(t_short), np.sin(2*t_short)]) tsm.basis_in = [None, sin_basis, sin_basis] tsm.basis_out = sin_basis_short coeffs = cc([[-1.], [.3], [1, 2, 3, 4], [5, 6, 7, 8], [-3, -4]]) constant, in_filters, out_filter = tsm.filters_from_coeffs(coeffs) self.assertAlmostEqual(constant, coeffs[0]) self.assertAlmostEqual(in_filters[0], coeffs[1]) np.testing.assert_array_almost_equal(in_filters[1], sin_basis.dot(coeffs[2:6])) np.testing.assert_array_almost_equal(in_filters[2], sin_basis.dot(coeffs[6:10])) np.testing.assert_array_almost_equal(out_filter, sin_basis_short.dot(coeffs[10:12]))
def open_file(self,filename): self.statusBar().showMessage('Loading {}'.format(os.path.basename(filename))) t=time.time() Tiff=tifffile.TiffFile(filename) try: metadata=Tiff[0].image_description metadata = self.txt2dict(metadata) except AttributeError: metadata=dict() tif=Tiff.asarray().astype(np.float64) Tiff.close() #tif=imread(filename,plugin='tifffile').astype(g.m.settings['internal_data_type']) if len(tif.shape)>3: # WARNING THIS TURNS COLOR movies TO BLACK AND WHITE BY AVERAGING ACROSS THE THREE CHANNELS tif=np.mean(tif,3) tif=np.squeeze(tif) #this gets rid of the meaningless 4th dimention in .stk files if len(tif.shape)==3: #this could either be a movie or a colored still frame if tif.shape[2]==3: #this is probably a colored still frame tif=np.mean(tif,2) tif=np.transpose(tif,(1,0)) # This keeps the x and y the same as in FIJI. else: tif=np.transpose(tif,(0,2,1)) # This keeps the x and y the same as in FIJI. elif len(tif.shape)==2: # I haven't tested whether this preserved the x y and keeps it the same as in FIJI. TEST THIS!! tif=np.transpose(tif,(0,1)) self.statusBar().showMessage('{} successfully loaded ({} s)'.format(os.path.basename(filename), time.time()-t)) return tif
def testImplicitLargeDiag(self): mu = np.array([[1., 2, 3], [11, 22, 33]]) # shape: [b, k] = [2, 3] u = np.array([[[1., 2], [3, 4], [5, 6]], [[0.5, 0.75], [1, 0.25], [1.5, 1.25]]]) # shape: [b, k, r] = [2, 3, 2] m = np.array([[0.1, 0.2], [0.4, 0.5]]) # shape: [b, r] = [2, 2] scale = np.stack([ np.eye(3) + np.matmul(np.matmul(u[0], np.diag(m[0])), np.transpose(u[0])), np.eye(3) + np.matmul(np.matmul(u[1], np.diag(m[1])), np.transpose(u[1])), ]) cov = np.stack([np.matmul(scale[0], scale[0].T), np.matmul(scale[1], scale[1].T)]) logging.vlog(2, "expected_cov:\n{}".format(cov)) with self.test_session(): mvn = ds.MultivariateNormalDiagPlusLowRank( loc=mu, scale_perturb_factor=u, scale_perturb_diag=m) self.assertAllClose(cov, mvn.covariance().eval(), atol=0., rtol=1e-6)
def fitt(XX, TT, DD, lam, p, fun=fmin_ncg): X = XX T = np.reshape(TT, (len(TT), 1)) D = np.reshape(DD, (1, len(DD))) sdi = sum(DD) ss = float(( np.dot(D,np.log(T)) ) * (p-1) + (math.log(lam*p))*sdi) #print ss, sdi ltpt = np.reshape(lam*np.power(T, p), (len(T), 1) ) ltp = np.transpose(ltpt) Dead = D Time = T DT = np.transpose(D) XT = np.transpose(X) def obj_grad (beta): K = np.dot(X , beta) v = ( DT - np.multiply( ltpt , derexpp(K).reshape( (len(T), 1) ) ) ) vl = - (np.dot (XT , v) ) return vl[:,0] def obj_value (beta): K = np.dot(X , beta) vl= -(ss + np.dot(Dead , K) - np.dot(ltp,expp(K))) return float(vl) bopt = fun(obj_value, np.zeros( np.shape(X)[1] ), fprime=obj_grad) return bopt
def one_schedule_var(P, mu, K, q, N, ucb_avg, l, R): # one scheduling step # P : permutation matrix # mu : actual transmission rate # K : number of queues # q : current queuelengths # N : number of samplings to be updates # avg : number of samplings # R : maximum alphabet size rates = np.diag(P * np.transpose(mu)) # print P # print "rates", rates for i in range(K): bit = random.random() if bit <= l[i]: q[i] = q[i] + 1 nodes = P * np.transpose(np.matrix(range(K))) # print nodes for i in range(K): bit = random.random() if bit <= float(rates[i] / R): # print "testing" if q[i] - R >= 0: q[i] = q[i] - R else: q[i] = 0 ucb_avg[i, int(nodes[i])] = (N[i, int(nodes[i])] * ucb_avg[i, int(nodes[i])] + 1) / ( N[i, int(nodes[i])] + 1 ) N[i, int(nodes[i])] = N[i, int(nodes[i])] + 1 else: ucb_avg[i, int(nodes[i])] = (N[i, int(nodes[i])] * ucb_avg[i, int(nodes[i])]) / (N[i, int(nodes[i])] + 1) N[i, int(nodes[i])] = N[i, int(nodes[i])] + 1
def imshow(img): img = img / 2 + 0.5 # unnormalize npimg = img.numpy() plt.imshow(np.transpose(npimg, (1, 2, 0))) plt.show()
def _augment_images_with_patch( x: np.ndarray, patch: np.ndarray, random_location: bool, channels_first: bool, mask: Optional[np.ndarray] = None, ) -> Tuple[np.ndarray, List[Dict[str, int]]]: """ Augment images with patch. :param x: Sample images. :param patch: The patch to be applied. :param random_location: If True apply patch at randomly shifted locations, otherwise place patch at origin (top-left corner). :param channels_first: Set channels first or last. :param mask: An boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x` (N, H, W) without their channel dimensions. Any features for which the mask is True can be the center location of the patch during sampling. :type mask: `np.ndarray` """ transformations = list() x_copy = x.copy() patch_copy = patch.copy() if channels_first: x_copy = np.transpose(x_copy, (0, 2, 3, 1)) patch_copy = np.transpose(patch_copy, (1, 2, 0)) for i_image in range(x.shape[0]): if random_location: if mask is None: i_x_1 = random.randint(0, x_copy.shape[1] - 1 - patch_copy.shape[0]) i_y_1 = random.randint(0, x_copy.shape[2] - 1 - patch_copy.shape[1]) else: if mask.shape[0] == 1: mask_2d = mask[0, :, :] else: mask_2d = mask[i_image, :, :] edge_x_0 = patch_copy.shape[0] // 2 edge_x_1 = patch_copy.shape[0] - edge_x_0 edge_y_0 = patch_copy.shape[1] // 2 edge_y_1 = patch_copy.shape[1] - edge_y_0 mask_2d[0:edge_x_0, :] = False mask_2d[-edge_x_1:, :] = False mask_2d[:, 0:edge_y_0] = False mask_2d[:, -edge_y_1:] = False num_pos = np.argwhere(mask_2d).shape[0] pos_id = np.random.choice(num_pos, size=1) pos = np.argwhere(mask_2d > 0)[pos_id[0]] i_x_1 = pos[0] - edge_x_0 i_y_1 = pos[1] - edge_y_0 else: i_x_1 = 0 i_y_1 = 0 i_x_2 = i_x_1 + patch_copy.shape[0] i_y_2 = i_y_1 + patch_copy.shape[1] transformations.append({"i_x_1": i_x_1, "i_y_1": i_y_1, "i_x_2": i_x_2, "i_y_2": i_y_2}) x_copy[i_image, i_x_1:i_x_2, i_y_1:i_y_2, :] = patch_copy if channels_first: x_copy = np.transpose(x_copy, (0, 3, 1, 2)) return x_copy, transformations
def grad_loss(X, w, y, alpha): predictions = probability_prediction(X, w) return (1.0 / y.shape[0]) * np.dot(np.transpose(X), (predictions-y)) + alpha * w
def eval_loss(X, w, y, alpha): predictions = probability_prediction(X, w) return ((-1.0 / y.shape[0]) * np.sum(np.log((1 - y) * (1 - predictions) + (y) * (predictions)))) + ((alpha / (2.0*y.shape[0])) * (np.dot(np.transpose(w), w)))
def evaluate(self, outputs, res_folder, metric='PCKh', **kwargs): """Evaluate PCKh for MPII dataset. Adapted from https://github.com/leoxiaobin/deep-high-resolution- net.pytorch Original licence: Copyright (c) Microsoft, under the MIT License. Note: batch_size: N num_keypoints: K heatmap height: H heatmap width: W Args: outputs(list(preds, boxes, image_path, output_heatmap)): * preds(np.ndarray[1,K,3]): The first two dimensions are coordinates, score is the third dimension of the array. * boxes(np.ndarray[1,6]): [center[0], center[1], scale[0] , scale[1],area, score] * image_path(list[str]): For example, ['0', '0', '0', '0', '0', '1', '1', '6', '3', '.', 'j', 'p', 'g'] * output_heatmap (np.ndarray[N, K, H, W]): model outputs. res_folder(str): Path of directory to save the results. metric (str | list[str]): Metrics to be performed. Defaults: 'PCKh'. Returns: dict: PCKh for each joint """ metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['PCKh'] for metric in metrics: if metric not in allowed_metrics: raise KeyError(f'metric {metric} is not supported') preds = np.stack([kpts[0] for kpts, _, _, _ in outputs]) # convert 0-based index to 1-based index, # and get the first two dimensions. preds = preds[..., :2] + 1.0 if res_folder: pred_file = os.path.join(res_folder, 'pred.mat') savemat(pred_file, mdict={'preds': preds}) SC_BIAS = 0.6 threshold = 0.5 gt_file = os.path.join(os.path.dirname(self.ann_file), 'mpii_gt_val.mat') gt_dict = loadmat(gt_file) dataset_joints = gt_dict['dataset_joints'] jnt_missing = gt_dict['jnt_missing'] pos_gt_src = gt_dict['pos_gt_src'] headboxes_src = gt_dict['headboxes_src'] pos_pred_src = np.transpose(preds, [1, 2, 0]) head = np.where(dataset_joints == 'head')[1][0] lsho = np.where(dataset_joints == 'lsho')[1][0] lelb = np.where(dataset_joints == 'lelb')[1][0] lwri = np.where(dataset_joints == 'lwri')[1][0] lhip = np.where(dataset_joints == 'lhip')[1][0] lkne = np.where(dataset_joints == 'lkne')[1][0] lank = np.where(dataset_joints == 'lank')[1][0] rsho = np.where(dataset_joints == 'rsho')[1][0] relb = np.where(dataset_joints == 'relb')[1][0] rwri = np.where(dataset_joints == 'rwri')[1][0] rkne = np.where(dataset_joints == 'rkne')[1][0] rank = np.where(dataset_joints == 'rank')[1][0] rhip = np.where(dataset_joints == 'rhip')[1][0] jnt_visible = 1 - jnt_missing uv_error = pos_pred_src - pos_gt_src uv_err = np.linalg.norm(uv_error, axis=1) headsizes = headboxes_src[1, :, :] - headboxes_src[0, :, :] headsizes = np.linalg.norm(headsizes, axis=0) headsizes *= SC_BIAS scale = headsizes * np.ones((len(uv_err), 1), dtype=np.float32) scaled_uv_err = uv_err / scale scaled_uv_err = scaled_uv_err * jnt_visible jnt_count = np.sum(jnt_visible, axis=1) less_than_threshold = (scaled_uv_err <= threshold) * jnt_visible PCKh = 100. * np.sum(less_than_threshold, axis=1) / jnt_count # save rng = np.arange(0, 0.5 + 0.01, 0.01) pckAll = np.zeros((len(rng), 16), dtype=np.float32) for r, threshold in enumerate(rng): less_than_threshold = (scaled_uv_err <= threshold) * jnt_visible pckAll[r, :] = 100. * np.sum(less_than_threshold, axis=1) / jnt_count PCKh = np.ma.array(PCKh, mask=False) PCKh.mask[6:8] = True jnt_count = np.ma.array(jnt_count, mask=False) jnt_count.mask[6:8] = True jnt_ratio = jnt_count / np.sum(jnt_count).astype(np.float64) name_value = [('Head', PCKh[head]), ('Shoulder', 0.5 * (PCKh[lsho] + PCKh[rsho])), ('Elbow', 0.5 * (PCKh[lelb] + PCKh[relb])), ('Wrist', 0.5 * (PCKh[lwri] + PCKh[rwri])), ('Hip', 0.5 * (PCKh[lhip] + PCKh[rhip])), ('Knee', 0.5 * (PCKh[lkne] + PCKh[rkne])), ('Ankle', 0.5 * (PCKh[lank] + PCKh[rank])), ('PCKh', np.sum(PCKh * jnt_ratio)), ('[email protected]', np.sum(pckAll[11, :] * jnt_ratio))] name_value = OrderedDict(name_value) return name_value
def on_enter(pts): x, y = np.transpose(pts) print("length = %0.2f" % np.sqrt(np.diff(x)**2 + np.diff(y)**2))
def makeSubmissionScript(featureMatrixPath, outputSubmissionPath, trainFakeTrips=200, digits=5, foldsize=10): #Read Feature Matrix featureMatrix = readFeatureMatrix.totalFeatureMatrix(featureMatrixPath) #ShortCut, to make loading faster, change the path if this is used. #featureMatrix = np.load('D:\\Documents\\Data\\MLiP\\features.npy') #np.save('D:\\Documents\\Data\\MLiP\\features', featureMatrix) print(np.shape(featureMatrix)) #some features that are not very informative, so they are ignored featureMatrix = np.delete( featureMatrix, [17, 39, 42, 46, 49, 52, 53, 85, 86, 87, 126, 138, 144, 148], 0) #Get the driver numbers as the names of the csv files in the feature matrix path drivernrs = readFeatureMatrix.getdrivernrs(featureMatrixPath) print('Done Reading Feature matrix!') print(np.shape(featureMatrix)) numFeat, _, numDrivers = np.shape(featureMatrix) numTrips = 200 #The number of trips to make the submission out of, always 200 numfolds = float( numTrips / foldsize ) #the number of folds that will be made, aka the number of models per driver importances = np.zeros( (numFeat, numDrivers)) #Storage for feature importances #The probabilities that a trip belongs to a driver probabilities = np.zeros((numTrips, 2, numDrivers)) for i in range(numDrivers): #First get the trainingtrips from the featurematrix. It is assumed that the real trips come first trainTrips = np.transpose(featureMatrix[:, :(numTrips + trainFakeTrips), i]) realTrips = trainTrips[:numTrips, :] trainLabels = np.hstack( (np.ones(numTrips - foldsize), np.zeros(trainFakeTrips))) #Training the model for each fold for b, e in [(k, k + foldsize) for k in np.arange(0, numTrips, foldsize)]: testIndex = np.arange(b, e) trainIndex = np.hstack( (np.arange(0, b), np.arange(e, (numTrips + trainFakeTrips)))) #Train the model, then predict the classes for that model model = learn.trainModel(trainTrips[trainIndex], trainLabels, criterion='entropy', n_trees=300, n_jobs=-1) #Keep track of the importance of each feature for this driver importances[:, i] = importances[:, i] + np.divide( model.feature_importances_, numfolds) tempprobs = learn.predictClass(model, realTrips[testIndex]) #Append the tripnrs, then add to probability table probabilities[testIndex, :, i] = np.transpose( np.vstack((testIndex + 1, tempprobs))) #Progress report, it's so slow that we give a report every driver if i % 1 == 0: print("Done learning driver " + ` i `) #Appending the output to a file. This is reccomended when doing a long calculation because you can halt #the computation and still have the results for the drivers so far, but it's not needed. #CreateSubmission.appendProbabilities(outputSubmissionPath, drivernrs[i], probabilities[:,:,i],'%0.' + `digits` + 'f') print('Done calculating probabilities!') #Makes submission file fmtstring = '%0.' + ` digits ` + 'f' CreateSubmission.createSubmissionfileFrom3D(outputSubmissionPath, probabilities, drivernrs=drivernrs, fmtstring=fmtstring) return importances
plt.plot(G_losses,label="G") plt.plot(D_losses,label="C") plt.xlabel("iterations") plt.ylabel("Loss") plt.legend() plt.savefig(str(filestr + "_loss_plot.png")) plt.close() # Grab a batch of real images from the dataloader real_batch = next(iter(test_loader)) # Plot the real images plt.figure(figsize=(15,15)) plt.subplot(1,3,1) plt.axis("off") plt.title("Real Images") plt.imshow(np.transpose(vutils.make_grid(real_batch[0].to(device)[:64], padding=5, normalize=True).cpu(),(1,2,0))) plt.subplot(1,3,2) plt.axis("off") plt.title("Fake Images after " + str(num_epochs//2) + " epochs") plt.imshow(np.transpose(img_list[num_epochs//2 +1],(1,2,0))) # Plot the fake images from the last epoch plt.subplot(1,3,3) plt.axis("off") plt.title("Fake Images after " + str(num_epochs) + " epochs") plt.imshow(np.transpose(img_list[-1],(1,2,0))) plt.savefig(str(filestr + "real_v_fake.png")) plt.close()
def sample_random_candidates(id, df, n_sample,VOXEL_SIZE, dfcandidates,fp_only,rotate=False): #choose random voxels from this id, #if they contain a nodule, return the index of this nodule in the dataframe #from that we can determine size, attributes #TODO: TRIM IMAGE df['ix'] = range(df.shape[0]) dfcandidates = dfcandidates[dfcandidates['seriesuid'] == id] dfsub = df[df['seriesuid']==id] nodule_coords = [] nodule_sizes = [] nodule_ixs = [] if len(dfsub) > 0: for i in range(dfsub.shape[0]): row = dfsub.iloc[i] nodule_coords.append((row['coordZ'], row['coordY'], row['coordX'])) nodule_sizes.append(row['diameter_mm']) img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', id + '.mhd')) voxel_coords = [worldToVoxelCoord(c,origin,spacing) for c in nodule_coords] # zoom = np.random.uniform(.99,1.01) numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0]) # assert numZpix > 10, 'too few z pixels' numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1]) # assert numYpix > 10, 'too few y pixels' numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2]) # assert numXpix > 10, 'too few x pixels' voxels = [] ixs = [] for i in range(n_sample): #choose a random row from the candidates file #OR choose a random voxel. 50-50. if dfcandidates.shape[0] > 0 and (np.random.randint(0,2) == 0 or fp_only): row = dfcandidates.iloc[np.random.choice(dfcandidates.shape[0])] x_center_raw = row['coordX'] * np.random.uniform(.9,1.1) y_center_raw = row['coordY'] * np.random.uniform(.9,1.1) z_center_raw = row['coordZ'] * np.random.uniform(.9,1.1) vcords = worldToVoxelCoord((z_center_raw, y_center_raw, x_center_raw), origin, spacing) x_center = vcords[2] y_center = vcords[1] z_center = vcords[0] ix = -2 #now - if the coordinates are too close to an edge just default to random ones if x_center < numXpix/2 or x_center > img.shape[2]-numXpix/2 or \ y_center < numYpix/2 or y_center > img.shape[1]-numYpix/2 or \ z_center < numZpix/2 or z_center > img.shape[0]-numZpix/2: x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2) y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2) z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2) ix = -1 else: #print 'no candidates for id', id x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2) y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2) z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2) ix = -1 z_start = np.clip(z_center-numZpix/2, 0, img.shape[0]) z_end = np.clip(z_center+numZpix/2, 0, img.shape[0]) y_start = np.clip(y_center-numYpix/2, 0, img.shape[1]) y_end = np.clip(y_center+numYpix/2, 0, img.shape[1]) x_start = np.clip(x_center-numXpix/2, 0, img.shape[2]) x_end = np.clip(x_center+numXpix/2, 0, img.shape[2]) voxel = img[z_start:z_end,y_start:y_end,x_start:x_end] voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE)) voxel_norm = np.clip(voxel_norm, -1000, 400) if rotate: voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(1,0), order=1,reshape=False,cval=-1000,mode='nearest') voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(2,1), order=1,reshape=False,cval=-1000,mode='nearest') #apply a random rotation #determine index (if applicable) #if no match put -1. largest_diam = 0 for i,(coord,size) in enumerate(zip(voxel_coords,nodule_sizes)): if (x_start < coord[2] < x_end) and (y_start < coord[1] < y_end ) and (z_start < coord[0] < z_end): #we got one if dfsub.iloc[i]['diameter_mm'] > largest_diam: largest_diam = dfsub.iloc[i]['diameter_mm'] ix = dfsub.iloc[i]['ix'] #target = size voxels.append(np.transpose(voxel_norm, (2,1,0))) ixs.append(ix) return np.stack(voxels),np.stack(ixs)
x_mesh_centralise = x_mesh_centralise_all y_mesh_centralise = y_mesh_centralise_all print('Data Collection and Binning Completed') # ------------------------------------------End of SELECTION FOR EXCLUSION OF ZERO POINTS # ------------------------------------------ Tabulate Poisson Likelihood # Use the k_quad and latent intensity v array to obtain likelihood # The k_quad now contains data from year 2014 # Latent_v_array was optimized based on # Generate Objective Function: log(P(D|v)) exp_term = -1 * np.sum(np.exp(latent_v_array)) product_term = np.matmul(latent_v_array, np.transpose(k_quad)) factorial_k = scispec.gamma(k_quad + np.ones_like(k_quad)) # factorial_term = - np.sum(np.log(factorial_k)) # summation of logs = log of product factorial_term = - np.sum(fn.log_special(factorial_k)) # summation of logs = log of product log_p_likelihood = exp_term + product_term + factorial_term print('The Log Poisson Likelihood is', log_p_likelihood) # ------------------------------------------------------- Tabulate the GP Log Marginal Likelihood # The Covariance Matrix and Prior mean are created here as a component of the objective function prior_mean = mean_func_scalar(mean_optimal, xy_quad) # Select Kernel and Construct Covariance Matrix
Level_ratio[car_id*(num_cars-1)+count, 1] = Level_ratio[car_id*(num_cars-1) + count, 1] + 0.5 Level_ratio[car_id*(num_cars-1)+count,:] = Level_ratio[car_id*(num_cars-1)+count,:]/ sum(Level_ratio[car_id*(num_cars-1)+count,:]) # normalizing count = count+1 print(Level_ratio) # State update X_new, R = Environment_Multi_Sim.Environment_Multi_Sim(X_old, Action_id, params.t_step_Sim, params) X_old = X_new # Reward plot color = ['b','r','m','g'] R_history[episode, :, step] = np.transpose(R) # plt.figure(2) # plt.plot(step,R[0],color='b',marker='.',markersize=16) # plt.plot(step,R[1],color='r',marker='.',markersize=16) # Plot the level_history ego_car_id = 1 # AV opp_car_id = 3 # Car 4 plot_level_ratio.plot_level_ratio(Level_ratio_history, ego_car_id, opp_car_id, params, step, episode, max_step, fig_lh) # Timer t1 = time.time() time_per_step = t1 - t0 # print(time_per_step)
def get_bounding_voxels_new(patient, df, n,VOXEL_SIZE,rotate=False): #given the nodule index and the nodule dataframe #return n jittered views of the nodule and n copies of the row index #from the row index we can look up the nodule size, malignancy, etc. df['ix'] = range(df.shape[0]) dfsub = df[df['seriesuid']==patient] if dfsub.shape[0] == 0: return None img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', patient + '.mhd')) #now pick out VOXEL_SIZE mm of pixels in each dimension. # zoom = np.random.uniform(.99,1.01) numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0]) # assert numZpix > 10, 'too few z pixels' numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1]) # assert numYpix > 10, 'too few y pixels' numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2]) # assert numXpix > 10, 'too few x pixels' voxels = [] indices = [] for i in range(n): #choose a random nodule from this patient row = dfsub.iloc[ np.random.choice(dfsub.shape[0]) ] coords = (row['coordZ'], row['coordY'], row['coordX']) diameter_mm = row['diameter_mm'] voxel_coords = worldToVoxelCoord(coords, origin, spacing) voxel_coords = np.round(voxel_coords) #fuzz max_z_fuzz = int((numZpix/2) * (1 - diameter_mm / VOXEL_SIZE)) max_y_fuzz = int((numYpix/2) * (1 - diameter_mm / VOXEL_SIZE)) max_x_fuzz = int((numXpix/2) * (1 - diameter_mm / VOXEL_SIZE)) zfuzz = np.random.randint(-max_z_fuzz, max_z_fuzz+1) if max_z_fuzz > 0 else 0 yfuzz = np.random.randint(-max_y_fuzz, max_y_fuzz+1) if max_y_fuzz > 0 else 0 xfuzz = np.random.randint(-max_x_fuzz, max_x_fuzz+1) if max_x_fuzz > 0 else 0 z_start = np.clip(voxel_coords[0] + zfuzz - numZpix/2, 0, img.shape[0]) z_end = np.clip(voxel_coords[0] + zfuzz +numZpix/2, 0, img.shape[0]) y_start = np.clip(voxel_coords[1]+ yfuzz -numYpix/2, 0, img.shape[1]) y_end = np.clip(voxel_coords[1] + yfuzz +numYpix/2, 0, img.shape[1]) x_start = np.clip(voxel_coords[2]+xfuzz-numXpix/2, 0, img.shape[2]) x_end = np.clip(voxel_coords[2]+xfuzz +numXpix/2, 0, img.shape[2]) #now let's see if this voxel contains more than one nodule num_nodules = 0 maxdiam_ix = -1 maxdiam = 0 for j in range(dfsub.shape[0]): row_j = dfsub.iloc[j] row_coords = (row_j['coordZ'], row_j['coordY'], row_j['coordX']) row_voxel_coords = worldToVoxelCoord(row_coords, origin, spacing) if (row_voxel_coords[2] > x_start+0 and row_voxel_coords[2] < x_end-0) and \ (row_voxel_coords[1] > y_start+0 and row_voxel_coords[1] < y_end-0) and \ (row_voxel_coords[0] > z_start+0 and row_voxel_coords[0] < z_end-0): #found one num_nodules += 1 if row_j['diameter_mm'] > maxdiam: maxdiam_ix = row_j['ix'] maxdiam = row_j['diameter_mm'] if num_nodules == 0: # print 'no nodules in region!' # print x_start, x_end, y_start, y_end, z_start, z_end # print voxel_coords maxdiam_ix = -1 # assert num_nodules > 0, 'no nodules in region' # if num_nodules > 1: # print 'multiple nodules found in voxel. choosing largest' indices.append(maxdiam_ix) voxel = img[z_start:z_end,y_start:y_end,x_start:x_end] # print voxel.shape, spacing voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE)) if rotate: voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(1,0), order=1,reshape=False,cval=-1000,mode='nearest') voxel_norm = ndimage.interpolation.rotate(voxel_norm, np.random.uniform(-10, 10), axes=(2,1), order=1,reshape=False,cval=-1000,mode='nearest') # halfsize = size/2 voxel_norm = np.clip(voxel_norm, -1000, 400) voxel_norm = np.transpose(voxel_norm, (2,1,0)) #X,Y,Z voxels.append(voxel_norm) return np.stack(voxels), np.array(indices).astype('int32')
def fit(self, images, indices=[slice(None), slice(None)]): """ This method uses the cnmf algorithm to find sources in data. it is calling every function from the cnmf folder you can find out more at how the functions are called and how they are laid out at the ipython notebook Args: images : mapped np.ndarray of shape (t,x,y[,z]) containing the images that vary over time. indices: list of slice objects along dimensions (x,y[,z]) for processing only part of the FOV Returns: self: updated using the cnmf algorithm with C,A,S,b,f computed according to the given initial values Raises: Exception 'You need to provide a memory mapped file as input if you use patches!!' See Also: ..image::docs/img/quickintro.png http://www.cell.com/neuron/fulltext/S0896-6273(15)01084-3 """ # Todo : to compartment if isinstance(indices, slice): indices = [indices] indices = [slice(None)] + indices if len(indices) < len(images.shape): indices = indices + [slice(None) ] * (len(images.shape) - len(indices)) dims_orig = images.shape[1:] dims_sliced = images[tuple(indices)].shape[1:] is_sliced = (dims_orig != dims_sliced) if self.params.get('patch', 'rf') is None and (is_sliced or 'ndarray' in str(type(images))): images = images[tuple(indices)] self.dview = None logging.warning("Parallel processing in a single patch " "is not available for loaded in memory or sliced" + " data.") T = images.shape[0] self.params.set('online', {'init_batch': T}) self.dims = images.shape[1:] #self.params.data['dims'] = images.shape[1:] Y = np.transpose(images, list(range(1, len(self.dims) + 1)) + [0]) Yr = np.transpose(np.reshape(images, (T, -1), order='F')) if np.isfortran(Yr): raise Exception( 'The file is in F order, it should be in C order (see save_memmap function)' ) logging.info((T, ) + self.dims) # Make sure filename is pointed correctly (numpy sets it to None sometimes) try: Y.filename = images.filename Yr.filename = images.filename except AttributeError: # if no memmapping cause working with small data pass # update/set all options that depend on data dimensions # number of rows, columns [and depths] self.params.set( 'spatial', { 'medw': (3, ) * len(self.dims), 'se': np.ones((3, ) * len(self.dims), dtype=np.uint8), 'ss': np.ones((3, ) * len(self.dims), dtype=np.uint8) }) logging.info(('Using ' + str(self.params.get('patch', 'n_processes')) + ' processes')) if self.params.get('preprocess', 'n_pixels_per_process') is None: avail_memory_per_process = psutil.virtual_memory( )[1] / 2.**30 / self.params.get('patch', 'n_processes') mem_per_pix = 3.6977678498329843e-09 npx_per_proc = np.int(avail_memory_per_process / 8. / mem_per_pix / T) npx_per_proc = np.int( np.minimum( npx_per_proc, np.prod(self.dims) // self.params.get('patch', 'n_processes'))) self.params.set('preprocess', {'n_pixels_per_process': npx_per_proc}) self.params.set( 'spatial', { 'n_pixels_per_process': self.params.get('preprocess', 'n_pixels_per_process') }) logging.info( 'using ' + str(self.params.get('preprocess', 'n_pixels_per_process')) + ' pixels per process') logging.info('using ' + str(self.params.get('spatial', 'block_size_spat')) + ' block_size_spat') logging.info('using ' + str(self.params.get('temporal', 'block_size_temp')) + ' block_size_temp') if self.params.get('patch', 'rf') is None: # no patches logging.info('preprocessing ...') Yr = self.preprocess(Yr) if self.estimates.A is None: logging.info('initializing ...') self.initialize(Y) if self.params.get( 'patch', 'only_init'): # only return values after initialization if not (self.params.get('init', 'method_init') == 'corr_pnr' and self.params.get('init', 'ring_size_factor') is not None): self.compute_residuals(Yr) self.estimates.bl = None self.estimates.c1 = None self.estimates.neurons_sn = None if self.remove_very_bad_comps: logging.info('removing bad components : ') final_frate = 10 r_values_min = 0.5 # threshold on space consistency fitness_min = -15 # threshold on time variability fitness_delta_min = -15 Npeaks = 10 traces = np.array(self.C) logging.info('estimating the quality...') idx_components, idx_components_bad, fitness_raw,\ fitness_delta, r_values = estimate_components_quality( traces, Y, self.estimates.A, self.estimates.C, self.estimates.b, self.estimates.f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True, N=5) logging.info( ('Keeping ' + str(len(idx_components)) + ' and discarding ' + str(len(idx_components_bad)))) self.estimates.C = self.estimates.C[idx_components] self.estimates.A = self.estimates.A[:, idx_components] # type: ignore # not provable that self.initialise provides a value self.estimates.YrA = self.estimates.YrA[idx_components] self.estimates.normalize_components() return self logging.info('update spatial ...') self.update_spatial(Yr, use_init=True) logging.info('update temporal ...') if not self.skip_refinement: # set this to zero for fast updating without deconvolution self.params.set('temporal', {'p': 0}) else: self.params.set('temporal', {'p': self.params.get('preprocess', 'p')}) logging.info('deconvolution ...') self.update_temporal(Yr) if not self.skip_refinement: logging.info('refinement...') if self.params.get('merging', 'do_merge'): logging.info('merging components ...') self.merge_comps(Yr, mx=50, fast_merge=True) logging.info('Updating spatial ...') self.update_spatial(Yr, use_init=False) # set it back to original value to perform full deconvolution self.params.set('temporal', {'p': self.params.get('preprocess', 'p')}) logging.info('update temporal ...') self.update_temporal(Yr, use_init=False) # else: # todo : ask for those.. # C, f, S, bl, c1, neurons_sn, g1, YrA, lam = self.estimates.C, self.estimates.f, self.estimates.S, self.estimates.bl, self.estimates.c1, self.estimates.neurons_sn, self.estimates.g, self.estimates.YrA, self.estimates.lam # embed in the whole FOV if is_sliced: FOV = np.zeros(dims_orig, order='C') FOV[indices[1:]] = 1 FOV = FOV.flatten(order='F') ind_nz = np.where(FOV > 0)[0].tolist() self.estimates.A = self.estimates.A.tocsc() A_data = self.estimates.A.data A_ind = np.array(ind_nz)[self.estimates.A.indices] A_ptr = self.estimates.A.indptr A_FOV = scipy.sparse.csc_matrix( (A_data, A_ind, A_ptr), shape=(FOV.shape[0], self.estimates.A.shape[-1])) b_FOV = np.zeros((FOV.shape[0], self.estimates.b.shape[-1])) b_FOV[ind_nz] = self.estimates.b self.estimates.A = A_FOV self.estimates.b = b_FOV else: # use patches if self.params.get('patch', 'stride') is None: self.params.set('patch', { 'stride': np.int(self.params.get('patch', 'rf') * 2 * .1) }) logging.info( ('Setting the stride to 10% of 2*rf automatically:' + str(self.params.get('patch', 'stride')))) if type(images) is np.ndarray: raise Exception( 'You need to provide a memory mapped file as input if you use patches!!' ) self.estimates.A, self.estimates.C, self.estimates.YrA, self.estimates.b, self.estimates.f, \ self.estimates.sn, self.estimates.optional_outputs = run_CNMF_patches( images.filename, self.dims + (T,), self.params, dview=self.dview, memory_fact=self.params.get('patch', 'memory_fact'), gnb=self.params.get('init', 'nb'), border_pix=self.params.get('patch', 'border_pix'), low_rank_background=self.params.get('patch', 'low_rank_background'), del_duplicates=self.params.get('patch', 'del_duplicates'), indices=indices) self.estimates.bl, self.estimates.c1, self.estimates.g, self.estimates.neurons_sn = None, None, None, None logging.info("merging") self.estimates.merged_ROIs = [0] if self.params.get('init', 'center_psf'): # merge taking best neuron if self.params.get('patch', 'nb_patch') > 0: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf, fast_merge=True) logging.info("update temporal") self.update_temporal(Yr, use_init=False) self.params.set( 'spatial', { 'se': np.ones( (1, ) * len(self.dims), dtype=np.uint8) }) logging.info('update spatial ...') self.update_spatial(Yr, use_init=False) logging.info("update temporal") self.update_temporal(Yr, use_init=False) else: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf, fast_merge=True) if len(self.estimates.merged_ROIs) > 0: not_merged = np.setdiff1d( list(range(len(self.estimates.YrA))), np.unique( np.concatenate( self.estimates.merged_ROIs))) self.estimates.YrA = np.concatenate([ self.estimates.YrA[not_merged], np.array([ self.estimates.YrA[m].mean(0) for ind, m in enumerate(self.estimates.merged_ROIs) if not self.empty_merged[ind] ]) ]) if self.params.get('init', 'nb') == 0: self.estimates.W, self.estimates.b0 = compute_W( Yr, self.estimates.A.toarray(), self.estimates.C, self.dims, self.params.get('init', 'ring_size_factor') * self.params.get('init', 'gSiz')[0], ssub=self.params.get('init', 'ssub_B')) if len(self.estimates.C): self.deconvolve() else: self.estimates.S = self.estimates.C else: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf) logging.info("update temporal") self.update_temporal(Yr, use_init=False) self.estimates.normalize_components() return self
def sample_random_voxels(id, df, n_sample,VOXEL_SIZE): #choose random voxels from this id, #if they contain a nodule, return the index of this nodule in the dataframe #from that we can determine size, attributes #TODO: TRIM IMAGE df['ix'] = range(df.shape[0]) dfsub = df[df['seriesuid']==id] nodule_coords = [] nodule_sizes = [] nodule_ixs = [] if len(dfsub) > 0: for i in range(dfsub.shape[0]): row = dfsub.iloc[i] nodule_coords.append((row['coordZ'], row['coordY'], row['coordX'])) nodule_sizes.append(row['diameter_mm']) img,origin,spacing = load_itk_image(os.path.join(r'D:\dlung\LUNA\imgs', id + '.mhd')) voxel_coords = [worldToVoxelCoord(c,origin,spacing) for c in nodule_coords] # zoom = np.random.uniform(.99,1.01) numZpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[0]) # assert numZpix > 10, 'too few z pixels' numYpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[1]) # assert numYpix > 10, 'too few y pixels' numXpix = np.round(np.random.uniform(.99,1.01) *float(VOXEL_SIZE) / spacing[2]) # assert numXpix > 10, 'too few x pixels' voxels = [] ixs = [] for i in range(n_sample): x_center = np.random.randint(low=numXpix/2,high=img.shape[2]-numXpix/2) y_center = np.random.randint(low=numYpix/2,high=img.shape[1]-numYpix/2) z_center = np.random.randint(low=numZpix/2,high=img.shape[0]-numZpix/2) z_start = np.clip(z_center-numZpix/2, 0, img.shape[0]) z_end = np.clip(z_center+numZpix/2, 0, img.shape[0]) y_start = np.clip(y_center-numYpix/2, 0, img.shape[1]) y_end = np.clip(y_center+numYpix/2, 0, img.shape[1]) x_start = np.clip(x_center-numXpix/2, 0, img.shape[2]) x_end = np.clip(x_center+numXpix/2, 0, img.shape[2]) voxel = img[z_start:z_end,y_start:y_end,x_start:x_end] voxel_norm = resize_voxel(voxel, (VOXEL_SIZE, VOXEL_SIZE, VOXEL_SIZE)) voxel_norm = np.clip(voxel_norm, -1000, 400) #determine index (if applicable) #if no match put -1. ix = -1 for i,(coord,size) in enumerate(zip(voxel_coords,nodule_sizes)): if (x_start + size*1.5 <= coord[2] <= x_end - size*1.5) and (y_start + size*1.5 <= coord[1] <= y_end - size*1.5) and (z_start + size*1.5 <= coord[0] <= z_end - size*1.5): #we got one ix = dfsub.iloc[i]['ix'] # pdb.set_trace() #target = size voxels.append(np.transpose(voxel_norm, (2,1,0))) ixs.append(ix) return np.stack(voxels),np.stack(ixs)
def initialize_variables(self): # initialization of V doesn't matter as we learn V on the first step conditioned on U self.V_mu = np.zeros((self.K, self.num_studies)) self.V_mu = np.random.randn(self.K, self.num_studies) print('start') pca = sklearn.decomposition.PCA(n_components=self.K, whiten=True) pca.fit(np.random.randn(self.num_studies, 200).T) self.V_mu = np.transpose(pca.components_.T) for kk in range(self.K): self.V_mu[kk, :] = (self.V_mu[kk, :] - np.mean( self.V_mu[kk, :])) / np.std(self.V_mu[kk, :]) print('end') self.V_var = np.ones((self.K, self.num_studies)) * 1.0 # Initialization of intercept doesn't matter as we learn intercept in the first step, conditioned on U self.intercept_mu = np.zeros(self.num_studies) self.intercept_var = np.ones(self.num_studies) * 1.0 # Initialization of U DOES matter self.U_mu = np.random.randn(self.num_snps, self.K) self.U_var = np.ones((self.num_snps, self.K)) * 1.0 self.S_U = np.ones((self.num_snps, self.K)) # Smart init for Residual variance ld_score = generate_ld_scores(np.ones( (self.U_mu.shape[0], 1)), self.pairwise_ld_files, self.pairwise_ld_indices_files) # Perform regression analysis independently for each study resid_varz = [] for study_num in range(self.num_studies): # Extract relevent data for this study study_chi_sq = np.load(self.chi_squared_files[study_num]) study_sample_size = self.study_sample_sizes[study_num] # Simple error checking if len(study_chi_sq) != ld_score.shape[0]: print('assumption error') # Fit regression model #clf = Lasso(alpha=.1) reg = LinearRegression().fit(ld_score * study_sample_size, study_chi_sq - 1.0) residual_var = np.sum( np.square( reg.predict(ld_score * study_sample_size) - (study_chi_sq - 1.0))) / (len(study_chi_sq) - 2) resid_varz.append(residual_var) mean_resid_var = np.mean(resid_varz) / 10.0 print(mean_resid_var) # Variance params # resid var #self.tau_alpha = np.ones(self.num_studies) #self.tau_beta = np.ones(self.num_studies) #self.tau_beta = np.asarray(resid_varz) self.tau_alpha = 1.0 self.tau_beta = mean_resid_var # U var self.gamma_U_alpha = np.ones(self.K) * 1.0 self.gamma_U_beta = np.ones(self.K) * 10.0 # V var self.gamma_V_alpha = np.ones(self.K) * 1.0 self.gamma_V_beta = np.ones(self.K) * 10.0 # Sparsity parameters self.theta_U_a = np.ones(self.K) self.theta_U_b = np.ones(self.K)
def step_centertrack(self, results, time_lag): if len(results) == 0: self.tracks = [] return [] else: temp = [] for det in results: # filter out classes not evaluated for tracking if det['detection_name'] not in TRACKING_NAMES: continue det['ct'] = np.array(det['translation'][:2]) det['tracking'] = np.array(det['velocity'][:2]) * -1 * time_lag det['label_preds'] = TRACKING_NAMES.index(det['detection_name']) temp.append(det) results = temp N = len(results) M = len(self.tracks) # N X 2 if 'tracking' in results[0]: dets = np.array( [det['ct'] + det['tracking'].astype(np.float32) for det in results], np.float32) else: dets = np.array( [det['ct'] for det in results], np.float32) item_cat = np.array([item['label_preds'] for item in results], np.int32) # N track_cat = np.array([track['label_preds'] for track in self.tracks], np.int32) # M max_diff = np.array([self.CLS_VELOCITY_ERROR[box['detection_name']] for box in results], np.float32) tracks = np.array( [pre_det['ct'] for pre_det in self.tracks], np.float32) # M x 2 if len(tracks) > 0: # NOT FIRST FRAME dist = (((tracks.reshape(1, -1, 2) - dets.reshape(-1, 1, 2)) ** 2).sum(axis=2)) # N x M dist = np.sqrt(dist) # absolute distance in meter invalid = ((dist > max_diff.reshape(N, 1)) + (item_cat.reshape(N, 1) != track_cat.reshape(1, M))) > 0 dist = dist + invalid * 1e18 if self.hungarian: dist[dist > 1e18] = 1e18 matched_indices = linear_sum_assignment(copy.deepcopy(dist)) matched_indices = np.transpose(np.asarray(matched_indices)) else: matched_indices = greedy_assignment(copy.deepcopy(dist)) else: # first few frame assert M == 0 matched_indices = np.array([], np.int32).reshape(-1, 2) matched_indices = np.array(matched_indices).reshape(-1, 2) unmatched_dets = [d for d in range(dets.shape[0]) if not (d in matched_indices[:, 0])] unmatched_tracks = [d for d in range(tracks.shape[0]) if not (d in matched_indices[:, 1])] if self.hungarian: matches = [] for m in matched_indices: if dist[m[0], m[1]] > 1e16: unmatched_dets.append(m[0]) else: matches.append(m) matches = np.array(matches).reshape(-1, 2) else: matches = matched_indices ret = [] for m in matches: track = results[m[0]] track['tracking_id'] = self.tracks[m[1]]['tracking_id'] track['age'] = 1 track['active'] = self.tracks[m[1]]['active'] + 1 ret.append(track) for i in unmatched_dets: track = results[i] self.id_count += 1 track['tracking_id'] = self.id_count track['age'] = 1 track['active'] = 1 ret.append(track) # still store unmatched tracks if its age doesn't exceed max_age, however, we shouldn't output # the object in current frame for i in unmatched_tracks: track = self.tracks[i] if track['age'] < self.max_age: track['age'] += 1 track['active'] = 0 ct = track['ct'] # movement in the last second if 'tracking' in track: offset = track['tracking'] * -1 # move forward track['ct'] = ct + offset ret.append(track) self.tracks = ret return ret
def predict(encoder, decoder): test_df = pd.read_csv(args.test_data, names=['input']) test_df['target_pos'] = test_df['input'].apply(lambda x: x.split('EOS')[ 1].split('NOP')[0].strip().split(' ')[:MAX_TGT_LEN]) test_df['target_rhyme'] = test_df['input'].apply( lambda x: x.split('NOP')[1].split('NOE')[0].strip()) test_df['target_length'] = test_df['input'].apply( lambda x: x.split('NOE')[1].split('NOR')[0].strip()) test_df['input'] = test_df['input'].apply( lambda x: x.split('EOS')[0].strip().split(' ')[1:MAX_INPUT_LEN + 1]) test_df['input'] = test_df['input'].apply( lambda x: x + [EOS_TOKEN] + [PAD_TOKEN for i in range(MAX_INPUT_LEN - len(x) - 1)]) # add EOS at the end target_rhyme = np.copy(test_df.target_rhyme.values) target_length = np.copy(test_df.target_length.values.astype(int)) target_pos = np.copy(test_df.target_pos.values) test_df['target_rhyme'] = test_df['target_rhyme'].apply( lambda x: [x, "NOE"]) test_df['target_length'] = test_df['target_length'].apply( lambda x: [x, "NOR"]) test_df['target_pos'] = test_df['target_pos'].apply( lambda x: x + ["POS_PAD" for i in range(MAX_TGT_LEN - len(x))] + ["NOP"]) test_df['input'] = test_df['input'] + test_df["target_pos"] + test_df[ "target_rhyme"] + test_df["target_length"] test_df['input'] = test_df['input'].apply( lambda x: [word2idx_mapping.get(i, 0) for i in x]) test_data = test_df['input'].values test_data = torch.LongTensor(test_data.tolist()).t().to(device) encoder = encoder.eval().to(device) decoder = decoder.eval().to(device) # predict batch_size = 1000 result = [[] for i in range(MAX_TGT_LEN)] for b in tqdm(range(test_data.shape[1] // batch_size)): start = batch_size * b end = None if (b == (test_data.shape[1] // batch_size) - 1) else batch_size * (b + 1) encoder_output, encoder_hidden = encoder(test_data[:, start:end]) decoder_hidden = encoder_hidden decoder_input = torch.LongTensor([[ word2idx_mapping[SOS_TOKEN] ]]).repeat(1, encoder_output.shape[1]).to(device) for i in range(MAX_TGT_LEN): if args.attn: decoder_output, decoder_hidden, attn_weights = decoder( decoder_input, decoder_hidden, encoder_output, tmp_tgt_lengths) _, top1 = decoder_output.topk(1, dim=1) decoder_input = top1.squeeze(-1).unsqueeze(0) else: decoder_output, decoder_hidden = decoder( decoder_input, decoder_hidden) _, top1 = decoder_output.topk(1, dim=2) decoder_input = top1.squeeze(-1) result[i].extend(top1.cpu().numpy()[0].tolist()) #_, top1 = decoder_output.topk(70, dim=2) # output top 70 for rhyme #decoder_input = top1[:,:,0].squeeze(-1) #result[i].extend(top1.cpu().numpy()[0].tolist()) ipdb.set_trace() # output result = np.transpose(np.array(result), (1, 0, 2)) res_len = np.zeros(result.shape[0]) new_result = [[] for i in range(result.shape[0])] cnt = 0 all_pos_cnt = 0 correct_pos_cnt = 0 with open(args.output_file, "w") as f: for i, res in enumerate(result): for j, w in enumerate(res): if (idx2word_mapping[w[0]] == EOS_TOKEN) or (j == len(res) - 1): res_len[i] = j # pick correct rhyme #for k, candidate in enumerate(last): # if j > 0 and rhyme_corpus.get(idx2word_mapping.get(candidate, 0), 0) == tgt_rhyme[i]: # new_result[i][j-1] = idx2word_mapping.get(candidate, 0) # cnt += 1 # break if j > 0 and rhyme_corpus.get( idx2word_mapping.get(last[0], 0), 0) == target_rhyme[i]: cnt += 1 # calculate POS accuracy all_pos_cnt += len(target_pos[i]) if len(new_result[i]) > 0: _, pos = zip(*jieba.posseg.cut(''.join(new_result[i]))) pos_len = min(len(pos), len(target_pos[i])) #pos = np.array([pos2idx_mapping.get(i, 0) for i in pos]) correct_pos_cnt += sum( np.array(pos[:pos_len]) == np.array(target_pos[i] [:pos_len])) else: print("", file=f) break for k, c in enumerate(new_result[i]): if k == len(new_result[i]) - 1: print(c, file=f) else: print(c, end=' ', file=f) break new_result[i].append(idx2word_mapping[w[0]]) last = w # save last timestep's output print('pos:', correct_pos_cnt / all_pos_cnt) print('rhyme:', cnt / result.shape[0]) print("length: ", sum(res_len == target_length) / res_len.shape[0]) output = subprocess.run([ 'python3', '/home/victai/SDML/HW3/trigram_model/trigram_evaluate.py', '--testing_data', args.output_file ], stderr=subprocess.STDOUT)
def __init__(self, tflist_arr, x_arr): if x_arr is not None: ndim = x_arr.ndim else: ndim = 0 if ndim == 0: self.rs = tflist_arr[0].rs self.in_eng = tflist_arr[0].in_eng self.eng = tflist_arr[0].eng self.dlnz = tflist_arr[0].dlnz self.x = x_arr self.spec_type = tflist_arr[0].spec_type self.tflist_arr = tflist_arr self._grid_vals = tflist_arr[0].grid_vals if tflist_arr[0].tftype == 'eng': # grid_vals should have indices corresponding to # (rs, in_eng, eng). grid_vals = np.transpose(grid_vals, (1, 0, 2)) # grid_vals are now (rs, in_eng, eng). if self.rs[0] - self.rs[1] > 0: # data points have been stored in decreasing rs. self.rs = np.flipud(self.rs) self._grid_vals = np.flip(self._grid_vals, 1) elif ndim == 1: if not arrays_equal([tflist.rs for tflist in tflist_arr]): raise TypeError('All redshift bins must be identical.') if not arrays_equal([tflist.in_eng for tflist in tflist_arr]): raise TypeError('All in_eng bins must be identical.') if not arrays_equal([tflist.eng for tflist in tflist_arr]): raise TypeError('All eng bins must be identical.') if len(set([tflist.dlnz for tflist in tflist_arr])) != 1: raise TypeError('All dlnz steps must be identical.') if len(set([tflist.spec_type for tflist in tflist_arr])) != 1: raise TypeError('All spec_type must be identical.') if len(set([tflist.tftype for tflist in tflist_arr])) != 1: raise TypeError('All tftype must be the same.') self.rs = tflist_arr[0].rs self.in_eng = tflist_arr[0].in_eng self.eng = tflist_arr[0].eng self.dlnz = tflist_arr[0].dlnz self.x = x_arr self.spec_type = tflist_arr[0].spec_type self.tflist_arr = tflist_arr grid_vals = np.array(np.stack( [tflist.grid_vals for tflist in tflist_arr]), ndmin=4) if tflist_arr[0].tftype == 'eng': # grid_vals should have indices corresponding to # (xH, rs, in_eng, eng). grid_vals = np.transpose(grid_vals, (0, 2, 1, 3)) # grid_vals are now (xH, rs, in_eng, eng). self._grid_vals = grid_vals if self.rs[0] - self.rs[1] > 0: # data points have been stored in decreasing rs. self.rs = np.flipud(self.rs) self._grid_vals = np.flip(self._grid_vals, 1) # Now, data is stored in *increasing* rs. elif ndim == 3: if not all( arrays_equal([tflist.rs for tflist in tflist_xHe_arr]) for tflist_xHe_arr in tflist_arr): raise TypeError('All redshift bins must be identical.') if not all( arrays_equal([tflist.in_eng for tflist in tflist_xHe_arr]) for tflist_xHe_arr in tflist_arr): raise TypeError('All in_eng bins must be identical.') if not all( arrays_equal([tflist.eng for tflist in tflist_xHe_arr]) for tflist_xHe_arr in tflist_arr): raise TypeError('All eng bins must be identical.') if len( set(tflist.dlnz for tflist_xHe_arr in tflist_arr for tflist in tflist_xHe_arr)) != 1: raise TypeError('All dlnz must be identical.') if len( set(tflist.spec_type for tflist_xHe_arr in tflist_arr for tflist in tflist_xHe_arr)) != 1: raise TypeError('All spec_type must be identical.') if len( set(tflist.tftype for tflist_xHe_arr in tflist_arr for tflist in tflist_xHe_arr)) != 1: raise TypeError('All tftype must be identical.') self.rs = tflist_arr[0][0].rs self.in_eng = tflist_arr[0][0].in_eng self.eng = tflist_arr[0][0].eng self.dlnz = tflist_arr[0][0].dlnz self.x = x_arr self.spec_type = tflist_arr[0][0].spec_type self.tflist_arr = tflist_arr grid_vals = np.array(np.stack([ np.stack([tflist.grid_vals for tflist in tflist_xHe_arr]) for tflist_xHe_arr in tflist_arr ]), ndmin=5) if tflist_arr[0][0].tftype == 'eng': # grid_vals should have indices corresponding to # (xH, xHe, rs, in_eng, eng). grid_vals = np.transpose(grid_vals, (0, 1, 3, 2, 4)) # grid_vals are now (xH, xHe, rs, in_eng, eng) self._grid_vals = grid_vals if self.rs[0] - self.rs[1] > 0: # data points have been stored in decreasing rs. self.rs = np.flipud(self.rs) self._grid_vals = np.flip(self._grid_vals, 2) # Now, data is stored in *increasing* rs. else: raise TypeError( 'x_arr dimensions is anomalous (and not in the good QFT way).') def __iter__(self): return iter(self.tflist_arr) def __getitem__(self, key): return self.tflist_arr[key] def __setitem__(self, key, value): self.tflist_arr[key] = value
def data_loader(data_name): if data_name == "starplus": """ starplus = sio.loadmat("data-starplus-04847-v7.mat") metadata = starplus['meta'][0,0] #meta.study gives the name of the fMRI study #meta.subject gives the identifier for the human subject #meta.ntrials gives the number of trials in this dataset #meta.nsnapshots gives the total number of images in the dataset #meta.nvoxels gives the number of voxels (3D pixels) in each image #meta.dimx gives the maximum x coordinate in the brain image. The minimum x coordinate is x=1. meta.dimy and meta.dimz give the same information for the y and z coordinates. #meta.colToCoord(v,:) gives the geometric coordinate (x,y,z) of the voxel corresponding to column v in the data #meta.coordToCol(x,y,z) gives the column index (within the data) of the voxel whose coordinate is (x,y,z) #meta.rois is a struct array defining a few dozen anatomically defined Regions Of Interest (ROIs) in the brain. Each element of the struct array defines on of the ROIs, and has three fields: "name" which gives the ROI name (e.g., 'LIFG'), "coords" which gives the xyz coordinates of each voxel in that ROI, and "columns" which gives the column index of each voxel in that ROI. #meta.colToROI{v} gives the ROI of the voxel corresponding to column v in the data. study = metadata['study'] subject = metadata['subject'] ntrials = metadata['ntrials'][0][0] nsnapshots = metadata['nsnapshots'][0][0] dimx = metadata['dimx'][0][0] colToCoord = metadata['colToCoord'] coordToCol = metadata['coordToCol'] rois = metadata['rois'] colToROI = metadata['colToROI'] info = starplus['info'][0] #info: This variable defines the experiment in terms of a sequence of 'trials'. 'info' is a 1x54 struct array, describing the 54 time intervals, or trials. Most of these time intervals correspond to trials during which the subject views a single picture and a single sentence, and presses a button to indicate whether the sentence correctly describes the picture. Other time intervals correspond to rest periods. The relevant fields of info are illustrated in the following example: #info(18) mint: 894 maxt: 948 cond: 2 firstStimulus: 'P' sentence: ''It is true that the star is below the plus.'' sentenceRel: 'below' sentenceSym1: 'star' sentenceSym2: 'plus' img: sap actionAnswer: 0 actionRT: 3613 #info.mint gives the time of the first image in the interval (the minimum time) #info.maxt gives the time of the last image in the interval (the maximum time) #info.cond has possible values 0,1,2,3. Cond=0 indicates the data in this segment should be ignored. Cond=1 indicates the segment is a rest, or fixation interval. Cond=2 indicates the interval is a sentence/picture trial in which the sentence is not negated. Cond=3 indicates the interval is a sentence/picture trial in which the sentence is negated. #info.firstStimulus: is either 'P' or 'S' indicating whether this trail was obtained during the session is which Pictures were presented before sentences, or during the session in which Sentences were presented before pictures. The first 27 trials have firstStimulus='P', the remained have firstStimulus='S'. Note this value is present even for trials that are rest trials. You can pick out the trials for which sentences and pictures were presented by selecting just the trials trials with info.cond=2 or info.cond=3. #info.sentence gives the sentence presented during this trial. If none, the value is '' (the empty string). The fields info.sentenceSym1, info.sentenceSym2, and info.sentenceRel describe the two symbols mentioned in the sentence, and the relation between them. #info.img describes the image presented during this trial. For example, 'sap' means the image contained a 'star above plus'. Each image has two tokens, where one is above the other. The possible tokens are star (s), plus (p), and dollar (d). #info.actionAnswer: has values -1 or 0. A value of 0 indicates the subject is expected to press the answer button during this trial (either the 'yes' or 'no' button to indicate whether the sentence correctly describes the picture). A value of -1 indicates it is inappropriate for the subject to press the answer button during this trial (i.e., it is a rest, or fixation trial). #info.actionRT: gives the reaction time of the subject, measured as the time at which they pressed the answer button, minus the time at which the second stimulus was presented. Time is in milliseconds. If the subject did not press the button at all, the value is 0. data = starplus['data'] #data: This variable contains the raw observed data. The fMRI data is a sequence of images collected over time, one image each 500 msec. The data structure 'data' is a [54x1] cell array, with one cell per 'trial' in the experiment. Each element in this cell array is an NxV array of observed fMRI activations. The element data{x}(t,v) gives the fMRI observation at voxel v, at time t within trial x. Here t is the within-trial time, ranging from 1 to info(x).len. The full image at time t within trial x is given by data{x}(t,:). #Note the absolute time for the first image within trial x is given by info(x).mint. MIN_LOSS_CHANGE=0.0001 maxFeatures = max([data[i][0].flatten().shape[0] for i in range(data.shape[0])]) # Inputs X = np.zeros((ntrials, maxFeatures + 1)) for i in range(data.shape[0]): f = data[i][0].flatten() X[i, :f.shape[0]] = f X[i, f.shape[0]] = 1 # Bias # Outputs (+1 = Picture, -1 = Sentence) Y = np.ones(ntrials) Y[np.array([info[i]['firstStimulus'][0] != 'P' for i in range(ntrials)])] = -1 # Randomly permute the data np.random.seed(1) # Seed the random number generator to preserve the dev/test split permutation = np.random.permutation(ntrials) permutation = np.random.permutation(X.shape[0]) X = X[permutation,] Y = Y[permutation,] X_train = X[:40] X_test = X[40:] Y_train = Y[:40] Y_test = Y[40:] np.savez('Starplus.npz', X_train=X_train, X_test=X_test, Y_train=Y_train,Y_test=Y_test) """ data = np.load('Starplus.npz') X_train = data['X_train'] / 10.0 Y_train = data['Y_train'] X_test = data['X_test'] / 10.0 Y_test = data['Y_test'] return np.transpose(X_train), Y_train.reshape(-1, 1), np.transpose(X_test), Y_test.reshape(-1, 1) else: X_train = [[1.0, 1.0, 1.0], [2.0, 2.0, 1.0], [4.0, 4.0, 1.0], [5.0, 5.0, 1.0], [1.0, 3.0, 1.0], [2.0, 4.0, 1.0], [4.0, 6.0, 1.0], [5.0, 7.0, 1.0]] X_test = [[3.0, 3.0, 1.0], [3.0, 5.0, 1.0]] Y_train = [[1.0], [1.0], [1.0], [1.0], [-1.0], [-1.0], [-1.0], [-1.0]] Y_test = [[1.0], [-1.0]] return np.transpose(np.asarray(X_train)), np.asarray(Y_train), np.transpose(np.asarray(X_test)), np.asarray(Y_test)
def setupParams(self, root, f): args = parse_args(root, f + '.avi') params = args['params'] # 50Hz gps data gps_name = args['gps_mark1'] print gps_name gps_reader = GPSReader(gps_name) GPSData = gps_reader.getNumericData() if GPSData.shape[0] == 0: print 'empty gps1 log. skipping...' return False self.imuTransforms = IMUTransforms(GPSData) self.GPSTime = utc_from_gps_log_all(GPSData) # Video-synced gps data gps_name2 = args['gps_mark2'] gps_reader2 = GPSReader(gps_name2) GPSData2 = gps_reader2.getNumericData() if GPSData2.shape[0] == 0: print 'empty gps2 log. skipping...' return False self.imuTransforms2 = IMUTransforms(GPSData2) # grab the initial time off the gps log and compute start and end times self.start_time = self.GPSTime[0] + 1 * 1e6 self.end_time = self.GPSTime[0] + 600 * 1e6 if self.end_time > self.GPSTime[-1]: self.end_time = self.GPSTime[-1] self.step_time = 0.5 self.scan_window = 0.1 self.lidar_loader = LDRLoader(args['frames']) self.T_from_l_to_i = params['lidar']['T_from_l_to_i'] self.lidar_height = params['lidar']['height'] print 'gps: ' + gps_name map_name = gps_name[0:-self.name_offset] + '.map' print 'map: ' + map_name self.map_outname = os.path.join( self.targetfolder, (gps_name[0:-self.name_offset] + '_lidarmap.pickle')) self.lane_outname = os.path.join( self.targetfolder, (gps_name[0:-self.name_offset] + '_interp_lanes.pickle')) print 'out: ' + self.lane_outname if os.path.isfile(self.lane_outname): print self.lane_outname + ' already exists, skipping...' return False total_num_frames = GPSData.shape[0] velocities = GPSData[:, 4:7] velocities[:, [0, 1]] = velocities[:, [1, 0]] vel_start = ENU2IMUQ50(np.transpose(velocities), GPSData[0, :]) # sideways vector wrt starting imu frame sideways_start = np.cross(vel_start.transpose(), self.imuTransforms[:, 0:3, 2], axisa=1, axisb=1, axisc=1) self.sideways = sideways_start / (np.sqrt( (sideways_start**2).sum(1))[..., np.newaxis] ) # normalize to unit vector # initialize empty data holders self.left_data = [] self.right_data = [] self.left_time = [] self.right_time = [] self.all_data = dict() self.all_time = dict() return True
def get_path(self, req): start = req.start end = req.goal tolerance = req.tolerance resp = self.map_client() map_ = MyMap() map_.from_msg(resp.map) my_down = MyMap(grid=map_.downscale(map_.grid, 20), resolution=1 / 20) explored_map = my_down.binary cv2.imshow("Map BW", map_.to_img(True)) voronoi_graph = generate_voronoi(explored_map) cv2.imshow("Voronoi", voronoi_graph) cv2.waitKey(0) cv2.destroyAllWindows() free_points = np.transpose(np.where(voronoi_graph == 0)) # Points in white in the voronoi min_start = 100 min_end = 100 # closest_start = min(free_points, key=lambda p: abs(p[0] - start.pose.position.x) + abs(p[1] - start.pose.position.y)) # closest_end = min(free_points, key=lambda p: abs(p[0] - end.pose.position.x) + abs(p[1] - end.pose.position.y)) # Get the closest point in the graph for start and end for point in free_points: tmp_start = (abs(point[0] - start.pose.position.x*10) + abs(point[1] - start.pose.position.y*10)) tmp_end = (abs(point[0] - end.pose.position.x*10) + abs(point[1] - end.pose.position.y*10)) if tmp_start < min_start: min_start = tmp_start start_point = point if tmp_end < min_end: min_end = tmp_end end_point = point path_list = best_first_search(voronoi_graph, (start_point[0], start_point[1]), (end_point[0], end_point[1]), 0) path_list = list(map(lambda i: (float(i[0])/10, float(i[1])/10), path_list)) path_list.append((end.pose.position.x, end.pose.position.y)) path_list.insert(0, (start.pose.position.x, start.pose.position.y)) self.is_path = True self.path = Path() self.path.header.frame_id = "map" filtered_path = [] filtered_path = np.copy(path_list[::2]) if path_list[-1][0] != filtered_path[-1][0] or path_list[-1][1] != filtered_path[-1][1]: filtered_path = np.vstack([filtered_path, path_list[-1]]) filtered_path = np.reshape(filtered_path, (-1,2)) no_obstacles = True n = 1 while no_obstacles and n<20: try_path = self.interpolate_path(filtered_path,0.2*n) n += 1 no_obstacles = self.check_obstacles(try_path) if no_obstacles and len(try_path) >= 2: smooth_path = np.copy(try_path) print('smoothened trajectory is valid') elif len(try_path) < 2: break if path_list[-1][0] != smooth_path[-1][0] or path_list[-1][1] != smooth_path[-1][1]: smooth_path = np.vstack([smooth_path, path_list[-1]]) for p in smooth_path: point = PoseStamped() point.pose.position.x = p[0] point.pose.position.y = p[1] self.path.poses.append(point) return self.path
#!/usr/bin/env python2.7 """ Extract the Cls from the chain CHAIN_FILE into a numpy file named CHAIN_FILE.cls.npy Usage: extract_cls.py CHAIN_FILE """ import sys from cosmoslik_plugins.samplers.metropolis_hastings import load_chain import numpy as np burnin = 1000 thin = 10 chain = load_chain(sys.argv[1]).burnin(burnin).join().thin(thin) # chain = load_chain('data1/run1_lcdm.chain').burnin(burnin).join().thin(thin) cls = np.array([x['cl_TT'] for x in chain['cl_TT']]) parnames = np.array(chain.keys()) parvls = np.transpose(np.array([chain[pn] for pn in parnames if pn!="cl_TT" ])) parnames_iterable = np.array([pn for pn in parnames if pn!="cl_TT" ]) np.save(sys.argv[1]+'.cls',cls) np.save(sys.argv[1]+'.parnames',parnames_iterable) np.save(sys.argv[1]+'.parvls',parvls)
# Plot data file = filename[0] with open(file) as f: line = f.readline() line = line.split(',') name = [] for i in range(len(line)): name.append(line[i]) fig1 = plt.figure(figsize=(15,9)) ax1 = fig1.add_subplot(211) ax2 = fig1.add_subplot(212) for i,file in enumerate(filename): data = np.loadtxt(file,delimiter=',',comments='#',skiprows=1) data = np.transpose(data) x = data[0] for k in range(len(name)): if k in [4,6,8,9]: ax1.plot(x,data[k],label=name[k],ls='-') ax2.plot(x,data[1],label=name[1],ls='-',c='k') # Fig ax1.legend(loc=0) ax2.legend(loc=0) x_ticks = np.linspace(x[0], x[-1], 5) ax1.set_xticks(x_ticks) # ax1.set_ylim(-20, 60) ax1.set_xlabel(r'$x$ (m)') ax1.set_ylabel(r'$\dot{\omega}$ (kg/m3 s)')
def make_outer_surf( orig_pial: Union[str, Path], image: Union[str, Path], output_fpath: Union[str, Path], outer_surface_sphere: float = 15, ): """Create outer surface of a pial volume. Make outer surface based on a pial volume and radius, write to surface in outfile. Parameters ---------- orig_pial : str | pathlib.Path Pial surface (e.g. lh.pial) image : str | pathlib.Path Filled lh or rh pial image (e.g. lh.pial.filled.mgz) output_fpath : str | pathlib.Path surface file to write data to outer_surface_sphere : float | None radius for smoothing in mm (default=15). diameter of the sphere used by make_outer_surface to close the sulci using morphological operations. Ignored currently. Corresponds to ``se=strel('sphere',se_diameter);`` in Matlab. See [1]. References ---------- .. [1] See FieldTrip Toolbox ``make_outer_surface`` function inside ``prepare_mesh_cortexhull.m`` file. .. [2] https://github.com/aestrivex/ielu """ from scipy.signal import convolve from scipy.ndimage.morphology import grey_closing, generate_binary_structure from mne import write_surface from mcubes import marching_cubes # radius information is currently ignored # it is a little tougher to deal with the morphology in python # load original pial surface to get the volume information pial_surf = nb.freesurfer.read_geometry(orig_pial, read_metadata=True) volume_info = pial_surf[2] # load filled pial image fill = nb.load(image) filld = fill.get_data() filld[filld == 1] = 255 # apply a very soft Gaussian filter with sigma = 1mm to # facilitate the closing gaussian = np.ones((2, 2)) * 0.25 # initialize image cube array image_f = np.zeros((256, 256, 256)) # initialize a thresholded image image2 = np.zeros((256, 256, 256)) # for each slice, convolve the Gaussian filter on the # filled image for slice in range(256): temp = filld[:, :, slice] image_f[:, :, slice] = convolve(temp, gaussian, "same") # thresholded image based on value of 25 image2[np.where(image_f <= 25)] = 0 image2[np.where(image_f > 25)] = 255 strel15 = generate_binary_structure(3, 1) # run multi-dimensional grayscale closing of the image BW2 = grey_closing(image2, structure=strel15) thresh = np.max(BW2) / 2 BW2[np.where(BW2 <= thresh)] = 0 BW2[np.where(BW2 > thresh)] = 255 # apply marching cubes algorithm to get # vertices and faces v, f = marching_cubes(BW2, 100) # in order to cope with the different orientation v2 = np.transpose( np.vstack(( 128 - v[:, 0], v[:, 2] - 128, 128 - v[:, 1], ))) write_surface(output_fpath, v2, f, volume_info=volume_info)
# incrementar fila print('VERDADERO') truth_table[filas, columna] = 'VERDADERO' arreglo.append('VERDADERO') cont=cont-1 bandera = False x_out = [] for col in range(number_of_variables): print('+++++++++++++++') print('columna', col) for i in range(0,number_of_states,(2**(col+1))): print(i,':') imprimir_verdadero_falso(2**col, col, x_out) print(x_out) y = np.reshape(x_out,(number_of_variables,number_of_states)) # changes from array to matrix format y2 = np.transpose(y) #print('************\n',y2) y3 = np.fliplr(y2) #changes order of columns print('************\n',y3) np.savetxt('Tabla_Verdad.csv',y3, fmt = "%s", delimiter=',') #np.savetxt('Tabla_Verdad.csv',y3, object, delimiter=',',header='v1,v2,v3,v4,v5,v6,v7,v8')
def __getitem__(self, index): #--------- # Image #--------- img_path = self.img_files[index % len(self.img_files)].rstrip() img = np.array(Image.open(img_path)) # Handles images with less than three channels while len(img.shape) != 3: index += 1 img_path = self.img_files[index % len(self.img_files)].rstrip() img = np.array(Image.open(img_path)) h, w, _ = img.shape dim_diff = np.abs(h - w) # Upper (left) and lower (right) padding pad1, pad2 = dim_diff // 2, dim_diff - dim_diff // 2 # Determine padding pad = ((pad1, pad2), (0, 0), (0, 0)) if h <= w else ((0, 0), (pad1, pad2), (0, 0)) # Add padding input_img = np.pad(img, pad, 'constant', constant_values=128) / 255. padded_h, padded_w, _ = input_img.shape # Resize and normalize input_img = resize(input_img, (*self.img_shape, 3), mode='reflect') # Channels-first input_img = np.transpose(input_img, (2, 0, 1)) # As pytorch tensor input_img = torch.from_numpy(input_img).float() #--------- # Label #--------- label_path = self.label_files[index % len(self.img_files)].rstrip() labels = None if os.path.exists(label_path): labels = np.loadtxt(label_path).reshape(-1, 5) # Extract coordinates for unpadded + unscaled image x1 = w * (labels[:, 1] - labels[:, 3]/2) y1 = h * (labels[:, 2] - labels[:, 4]/2) x2 = w * (labels[:, 1] + labels[:, 3]/2) y2 = h * (labels[:, 2] + labels[:, 4]/2) # Adjust for added padding x1 += pad[1][0] y1 += pad[0][0] x2 += pad[1][0] y2 += pad[0][0] # Calculate ratios from coordinates labels[:, 1] = ((x1 + x2) / 2) / padded_w labels[:, 2] = ((y1 + y2) / 2) / padded_h labels[:, 3] *= w / padded_w labels[:, 4] *= h / padded_h # Fill matrix filled_labels = np.zeros((self.max_objects, 5)) if labels is not None: filled_labels[range(len(labels))[:self.max_objects]] = labels[:self.max_objects] filled_labels = torch.from_numpy(filled_labels) return img_path, input_img, filled_labels
def rescaler(Xs, Ys, rescale_factor): Xs = np.asarray([tf.rescale(np.transpose(Xs[i], (2, 1, 0)), scale=rescale_factor) for i in range(0, Xs.shape[0])]) Xs = Xs.transpose((0, 3, 2, 1)) Ys = np.asarray([tf.rescale(np.transpose(Ys[i], (2, 1, 0)), scale=rescale_factor) for i in range(0, Ys.shape[0])]) Ys = Ys.transpose((0, 3, 2, 1)) return Ys, Xs
txt = plt.suptitle("-", fontsize=36, fontweight='bold') pred_text = str(all_labels[pred[0]]) true_text = str(all_labels[true[0]]) txt.set_text('PREDICTION: %s\nTRUTH: %s' % (pred_text, true_text)) plt.subplots_adjust(top=0.7) f_glimpse_images = np.reshape( glimpse_images_fetched, (params.num_glimpses, params.batch_size, params.num_points_per_glimpse, params.loc_dim)) # display the entire image along with the sampled locations sampled_locs_fetched = np.transpose(sampled_locs_fetched, (1, 0, 2)) plotWholePC(images[0, :], sampled_locs_fetched) # Display the glimpses plotGlimpses(f_glimpse_images, glimpse_axes) if display: plt.ion() plt.draw() time.sleep(0.05) plt.pause(pause_time) elif save: plt.savefig('%s_%s_%s.png' % (str(i), pred_text, true_text)) plt.close() else:
size_w = integer, number of grid points in cake state space w_grid = vector, size_w x 1 vector of cake grid points ------------------------------------------------------------------------ ''' lb = 10 ub = 15 size_c = 50 size_z = 50 c_grid = np.linspace(lb, ub, size_c) z_grid, pi = ar1.addacooper(size_z, mu, rho, sigma_z) z_grid = np.exp(z_grid) prob_m = np.transpose(pi) ''' ------------------------------------------------------------------------ Create grid of current utility values ------------------------------------------------------------------------ C = matrix, current consumption (c=w-w') U = matrix, current period utility value for all possible choices of w and w' (rows are w, columns w') ------------------------------------------------------------------------ ''' C = np.zeros((size_c, size_c, size_c)) for i in range(size_c): for j in range(size_c): for k in range(size_c): C[i, j, k] = z_grid[k] * c_grid[i]**alpha + ( 1 - delta) * c_grid[i] - c_grid[j]
sys.stdout.write("\r{0}".format(index)) sys.stdout.flush() x = X1_full[index] same_as_untreated = np.all(X0 == x, axis=1) # True if untreated is same as treated if any(same_as_untreated ): # if same as treated, assign uniform weights to these untreated untreated_id = [i for i, x in enumerate(same_as_untreated) if x] all_w[index, untreated_id] = 1 / len(untreated_id) else: in_hull_flag = in_hull(x, X0) if in_hull_flag: X0_tilde, antiranks = incremental_pure_synth(X1_full[index], X0) w = pensynth_weights(np.transpose(X0_tilde), X1_full[index], pen=0.000001) all_w[index, antiranks] = np.transpose(w) else: w = pensynth_weights(np.transpose(X0), X1_full[index], pen=0) all_w[index, ] = np.transpose(w) print( f"Temps d'exécution total : {(time.time() - start_time):.2f} secondes ---") ########## Save weights ########## np.savetxt('/Users/jeremylhour/Documents/code/puresynth_solution.csv', all_w, delimiter=',')