def progress_all(self): """ Competence progress of the overall tree. """ return self.progress_idxs(range(np.shape(self.get_data_x())[0] - self.progress_win_size, np.shape(self.get_data_x())[0]))
def sum_to_shape(X, s): """ Sum axes of the array such that the resulting shape is as given. Thus, the shape of the result will be s or an error is raised. """ # First, sum and remove axes that are not in s if np.ndim(X) > len(s): axes = tuple(range(-np.ndim(X), -len(s))) else: axes = () Y = np.sum(X, axis=axes) # Second, sum axes that are 1 in s but keep the axes axes = () for i in range(-np.ndim(Y), 0): if s[i] == 1: if np.shape(Y)[i] > 1: axes = axes + (i,) else: if np.shape(Y)[i] != s[i]: raise ValueError("Shape %s can't be summed to shape %s" % (np.shape(X), s)) Y = np.sum(Y, axis=axes, keepdims=True) return Y
def __init__(self, data, classes, tree_features, n_trees=100): self.n_features = np.shape(data)[1] n_rows = np.shape(data)[0] n_nans = np.sum(np.isnan(data), 0) data = data[:, n_nans < n_rows] self.n_features = np.shape(data)[1] n_nans = np.sum(np.isnan(data), 1) data = data[n_nans < self.n_features, :] self.n_rows = np.shape(data)[0] if (tree_features > self.n_features): tree_features = self.n_features self.col_list = np.zeros((n_trees, tree_features), dtype='int') self.n_trees = n_trees self.bags = [] for i in range(n_trees): cols = sample(range(self.n_features), tree_features) cols.sort() self.col_list[i, :] = cols data_temp = data[:, cols] n_nans = np.sum(np.isnan(data_temp), 1) data_temp = data_temp[n_nans == 0, :] classes_temp = classes[n_nans == 0] #bag = BaggingClassifier(n_estimators=1, max_features=tree_features) bag = RandomForestClassifier(n_estimators=1, max_features=tree_features) bag.fit(data_temp, classes_temp) self.bags.append(bag) print(np.shape(data_temp))
def train( self, eta = 0.25, iterations = 1000, outtype = "logistic" ): """ Train the network. Used instead of method early_stopping(). eta -- Learning rate. iterations -- Number of iterations to do. outtype -- Activation function to use: "linear", "logistic" """ self.eta = eta self.outtype = outtype shuffle = range( self.data_amount ) # Init arrays for weight updates self.update_w1 = ny.zeros( ( ny.shape( self.weights_layer1 ) ) ) self.update_w2 = ny.zeros( ( ny.shape( self.weights_layer2 ) ) ) # Start training for n in range( iterations ): self.outputs = self._forward( self.inputs ) # Compute error and update weights deltao, deltah = self._compute_errors() self._update_weights( deltao, deltah ) # Randomise order of training data ny.random.shuffle( shuffle ) self.inputs = self.inputs[shuffle,:] self.targets = self.targets[shuffle,:]
def computeNumericalGradient(J,theta): # numgrad = computeNumericalGradient(J, theta) # theta: a vector of parameters # J: a function that outputs r. # Calling y = J(theta) will return the function value at theta. # Initialize numgrad with zeros numgrad = np.zeros(np.shape(theta)) ## ---------- YOUR CODE HERE -------------------------------------- # Instructions: # Implement numerical gradient checking, and return the result in numgrad. # (See Section 2.3 of the lecture notes.) # You should write code so that numgrad(i) is (the numerical approximation to) the # partial derivative of J with respect to the i-th input argument, evaluated at theta. # I.e., numgrad(i) should be the (approximately) the partial derivative of J with # respect to theta(i). # # Hint: You will probably want to compute the elements of numgrad one at a time. for i in range(0,numgrad.shape[0]): k = np.zeros(np.shape(theta)) k[i] = 0.0001 y1 = J(theta+k) y2 = J(theta - k) numgrad[i] = (y1-y2)/0.0002 ## --------------------------------------------------------------- return numgrad
def test_2d_array_parameters_2d_array_input(self): """ When given an array input it must be broadcastable with all the parameters. """ t = TModel_1_2([[1, 2], [3, 4]], [[10, 20], [30, 40]], [[1000, 2000], [3000, 4000]]) y1, z1 = t([[100, 200], [300, 400]]) assert np.shape(y1) == np.shape(z1) == (2, 2) assert np.all(y1 == [[111, 222], [333, 444]]) assert np.all(z1 == [[1111, 2222], [3333, 4444]]) y2, z2 = t([[[[100]], [[200]]], [[[300]], [[400]]]]) assert np.shape(y2) == np.shape(z2) == (2, 2, 2, 2) assert np.all(y2 == [[[[111, 122], [133, 144]], [[211, 222], [233, 244]]], [[[311, 322], [333, 344]], [[411, 422], [433, 444]]]]) assert np.all(z2 == [[[[1111, 2122], [3133, 4144]], [[1211, 2222], [3233, 4244]]], [[[1311, 2322], [3333, 4344]], [[1411, 2422], [3433, 4444]]]]) with pytest.raises(ValueError): # Doesn't broadcast y3, z3 = t([[100, 200, 300], [400, 500, 600]])
def checkJointsNonLinearised(joints): """Check if joints are in the [[x,y],[x,y],...] or [[x,y,z],[x,y,z],...] format""" try: check = np.shape(joints)[0] > np.shape(joints)[1] return check except: return False
def test_scalar_parameters_1d_array_input(self): """ The dimension of the input should match the number of models unless model_set_axis=False is given, in which case the input is copied across all models. """ t = TModel_1_1([1, 2], [10, 20], n_models=2) with pytest.raises(ValueError): y = t(np.arange(5) * 100) y1 = t([100, 200]) assert np.shape(y1) == (2,) assert np.all(y1 == [111, 222]) y2 = t([100, 200], model_set_axis=False) # In this case the value [100, 200, 300] should be evaluated on each # model rather than evaluating the first model with 100 and the second # model with 200 assert np.shape(y2) == (2, 2) assert np.all(y2 == [[111, 211], [122, 222]]) y3 = t([100, 200, 300], model_set_axis=False) assert np.shape(y3) == (2, 3) assert np.all(y3 == [[111, 211, 311], [122, 222, 322]])
def test_1d_array_parameters_1d_array_input(self): """ When the input is an array, if model_set_axis=False then it must broadcast with the shapes of the parameters (excluding the model_set_axis). Otherwise all dimensions must be broadcastable. """ t = TModel_1_1([[1, 2, 3], [4, 5, 6]], [[10, 20, 30], [40, 50, 60]], n_models=2) with pytest.raises(ValueError): y1 = t([100, 200, 300]) y1 = t([100, 200]) assert np.shape(y1) == (2, 3) assert np.all(y1 == [[111, 122, 133], [244, 255, 266]]) with pytest.raises(ValueError): # Doesn't broadcast with the shape of the parameters, (3,) y2 = t([100, 200], model_set_axis=False) y2 = t([100, 200, 300], model_set_axis=False) assert np.shape(y2) == (2, 3) assert np.all(y2 == [[111, 222, 333], [144, 255, 366]])
def mlr_show4( clf, RMv, yEv, disp = True, graph = True): yEv_calc = clf.predict( RMv) if len( np.shape(yEv)) == 2 and len( np.shape(yEv_calc)) == 1: yEv_calc = np.mat( yEv_calc).T r_sqr, RMSE, MAE, DAE = estimate_accuracy4( yEv, yEv_calc, disp = disp) if graph: plt.figure() ms_sz = max(min( 4000 / yEv.shape[0], 8), 1) plt.plot( yEv.tolist(), yEv_calc.tolist(), '.', ms = ms_sz) ax = plt.gca() lims = [ np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes ] # now plot both limits against eachother #ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0) ax.plot(lims, lims, '-', color = 'pink') plt.xlabel('Experiment') plt.ylabel('Prediction') #plt.title( '$r^2$={0:.2e}, RMSE={1:.2e}, AAE={2:.2e}'.format( r_sqr, RMSE, aae)) plt.title( '$r^2$={0:.1e},$\sigma$={1:.1e},MAE={2:.1e},DAE={3:.1e}'.format( r_sqr, RMSE, MAE, DAE)) plt.show() return r_sqr, RMSE, MAE, DAE
def test_mixed_array_parameters_1d_array_input(self): """ When given an array input it must be broadcastable with all the parameters. """ t = TModel_1_1([[[0.01, 0.02, 0.03], [0.04, 0.05, 0.06]], [[0.07, 0.08, 0.09], [0.10, 0.11, 0.12]]], [1, 2, 3]) y1 = t([10, 20, 30]) assert np.shape(y1) == (2, 2, 3) assert_allclose(y1, [[[11.01, 22.02, 33.03], [11.04, 22.05, 33.06]], [[11.07, 22.08, 33.09], [11.10, 22.11, 33.12]]]) y2 = t([[[[10]]], [[[20]]], [[[30]]]]) assert np.shape(y2) == (3, 2, 2, 3) assert_allclose(y2, [[[[11.01, 12.02, 13.03], [11.04, 12.05, 13.06]], [[11.07, 12.08, 13.09], [11.10, 12.11, 13.12]]], [[[21.01, 22.02, 23.03], [21.04, 22.05, 23.06]], [[21.07, 22.08, 23.09], [21.10, 22.11, 23.12]]], [[[31.01, 32.02, 33.03], [31.04, 32.05, 33.06]], [[31.07, 32.08, 33.09], [31.10, 32.11, 33.12]]]])
def mm_to_galvo_approx(x, y=None): """ Given one or many points in mm space, map them to galvo space. e.g., >>> Printer.mm_to_galvo(0, 0) # -> galvo ticks for middle of build area. >>> Printer.mm_to_galvo([[0, 1, 2], [0, 0, 0]]) # -> A three-segment line along the x axis. """ xy = x if y is not None: if np.shape(x) != np.shape(y): raise TypeError('x and y shapes must match. Got x.shape: {}, y.shape: {}'.format(np.shape(x), np.shape(y))) xy = np.array([x, y]) # Allows calling with just an x and a y. # These polynomials are a fit to all Form 1/1+s. Px = np.array([ 3.27685507e+04, 4.80948842e+02, -1.22079970e-01, -2.88953161e-03, 6.08478254e-01, -8.81889894e-02, -2.20922460e-05, 4.41734858e-07, 6.76006698e-03, -1.02093319e-05, -1.43020804e-06, 2.03140758e-08, -6.71090318e-06, -4.36026159e-07, 2.62988209e-08, 8.32187652e-11]) Py = np.array([ 3.27661362e+04, 5.69452975e-01, -2.39793282e-03, 9.83778919e-06, 4.79035581e+02, -8.13031539e-02, -2.66499770e-03, -4.40219799e-07, -1.06247442e-01, 5.18419181e-05, 1.47754740e-06, -1.60049118e-09, -2.44473912e-03, -1.31398011e-06, 1.83452740e-08, 3.16943985e-10]) xy = np.asarray(xy, dtype=float) if xy.shape[0] != 2: raise TypeError('xy must be a two-vector or 2xn or 2xmxn... not shape {}.'.format(xy.shape)) shp = xy.shape[1:] # polyval2d wants vector inputs, not multidimensional. return np.array([polyval2d(P, *xy.reshape(2,-1)).reshape(shp) for P in (Px, Py)])
def cv_show( yEv, yEv_calc, disp = True, graph = True, grid_std = None): # if the output is a vector and the original is a metrix, # the output is translated to a matrix. if len( np.shape(yEv_calc)) == 1: yEv_calc = np.mat( yEv_calc).T if len( np.shape(yEv)) == 1: yEv = np.mat( yEv).T r_sqr, RMSE = jchem.estimate_accuracy( yEv, yEv_calc, disp = disp) if graph: #plt.scatter( yEv.tolist(), yEv_calc.tolist()) plt.figure() ms_sz = max(min( 4000 / yEv.shape[0], 8), 1) plt.plot( yEv.tolist(), yEv_calc.tolist(), '.', ms = ms_sz) # Change ms ax = plt.gca() lims = [ np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes ] # now plot both limits against eachother #ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0) ax.plot(lims, lims, '-', color = 'pink') plt.xlabel('Experiment') plt.ylabel('Prediction') if grid_std: plt.title( '($r^2$, std) = ({0:.2e}, {1:.2e}), RMSE = {2:.2e}'.format( r_sqr, grid_std, RMSE)) else: plt.title( '$r^2$ = {0:.2e}, RMSE = {1:.2e}'.format( r_sqr, RMSE)) plt.show() return r_sqr, RMSE
def chooseBestSplit(dataSet, leafType=regLeaf, errType=regErr, ops=(1, 4)): tolS = ops[0] tolN = ops[1] if len(set(dataSet[:, -1].T.tolist()[0])) == 1: return None, leafType(dataSet) m, n = np.shape(dataSet) S = errType(dataSet) bestS = np.inf bestIndex = 0 bestValue = 0 for featIndex in xrange(n - 1): for splitVal in set(dataSet[:, featIndex]): mat0, mat1 = binSplitData(dataSet, featIndex, splitVal) if np.shape(mat0)[0] < tolN or np.shape(mat1)[0] < tolN: continue newS = errType(mat0) + errType(mat1) if newS < bestS: bestIndex = featIndex bestValue = splitVal bestS = newS if S - bestS < tolS: return None, leafType(dataSet) mat0, mat1 = binSplitData(dataSet, bestIndex, bestValue) if np.shape(mat0)[0] < tolN or np.shape(mat1)[0] < tolN: return None, leafType(dataSet) return bestIndex, bestValue
def mm_to_galvo(self, x, y): """ Given one or many points in mm space, map them to galvo space. e.g., >>> Printer.mm_to_galvo(0, 0) # -> galvo ticks for middle of build area. >>> Printer.mm_to_galvo([[0, 1, 2], [0, 0, 0]]) # -> A three-segment line along the x axis. The returned array is 2xN, where N is the number of source points """ xshape = np.shape(x) if self._grid_table is None: grid = np.array(self.read_grid_table()) assert grid.shape == (5, 5, 2) pts_mm = np.linspace(-64, 64, 5) # Grid positions in mm # Interpolators for X and Y values (mm to galvo ticks) fit_x = scipy.interpolate.interp2d(pts_mm, pts_mm, grid[:,:,0]) fit_y = scipy.interpolate.interp2d(pts_mm, pts_mm, grid[:,:,1]) self._grid_table = (fit_x, fit_y) if np.shape(x) != np.shape(y): raise TypeError('x and y shapes must match. Got x.shape: {}, y.shape: {}'.format(np.shape(x), np.shape(y))) x = np.atleast_1d(x) y = np.atleast_1d(y) x_ = [self._grid_table[0](a, b) for a, b in zip(x, y)] y_ = [self._grid_table[1](a, b) for a, b in zip(x, y)] result = np.hstack([x_, y_]).T if xshape == (): # If it's called with scalars, return a flat result. return result.flatten() return result
def plot_checkpoint(self,b): orig_filename = "/data/batch_check_"+str(b)+"_original.png" image_A = self.X_test_A[5] image_A = np.reshape(image_A, [self.W_A_test,self.H_A_test,self.C_A_test]) print("Image_A shape: " +str(np.shape(image_A))) fake_B = self.generator_A_to_B.Generator.predict(image_A.reshape(1, self.W_A, self.H_A, self.C_A )) fake_B = np.reshape(fake_B, [self.W_A_test,self.H_A_test,self.C_A_test]) print("fake_B shape: " +str(np.shape(fake_B))) reconstructed_A = self.generator_B_to_A.Generator.predict(fake_B.reshape(1, self.W_A, self.H_A, self.C_A )) reconstructed_A = np.reshape(reconstructed_A, [self.W_A_test,self.H_A_test,self.C_A_test]) print("reconstructed_A shape: " +str(np.shape(reconstructed_A))) # from IPython import embed; embed() checkpoint_images = np.array([image_A, fake_B, reconstructed_A]) # Rescale images 0 - 1 checkpoint_images = 0.5 * checkpoint_images + 0.5 titles = ['Original', 'Translated', 'Reconstructed'] fig, axes = plt.subplots(1, 3) for i in range(3): image = checkpoint_images[i] image = np.reshape(image, [self.H_A_test,self.W_A_test,self.C_A_test]) axes[i].imshow(image) axes[i].set_title(titles[i]) axes[i].axis('off') fig.savefig("/data/batch_check_"+str(b)+".png") plt.close('all') return
def load_xvg(): #DONE global nb_rows, nb_cols global first_col global label_xaxis global label_yaxis global f_data global f_legend global f_col_legend global nb_col_tot f_data = {} f_legend = {} label_xaxis = "x axis" label_yaxis = "y axis" tmp_nb_rows_to_skip = 0 #get file content with open(args.xvgfilename) as f: lines = f.readlines() #determine legends and nb of lines to skip c_index = 0 for l_index in range(0,len(lines)): line = lines[l_index] if line[-1] == '\n': line = line[:-1] if line[0] in args.comments: tmp_nb_rows_to_skip += 1 if "legend \"" in line: try: tmp_col = int(int(line.split("@ s")[1].split(" ")[0])) tmp_name = line.split("legend \"")[1][:-1] f_legend[c_index] = tmp_name c_index += 1 except: print "\nError: unexpected data format in line " + str(l_index) + " in file " + str(filename) + "." print " -> " + str(line) sys.exit(1) if "xaxis" in line and "label " in line: label_xaxis = line.split("label ")[1] if "yaxis" in line and "label " in line: label_yaxis = line.split("label ")[1] #get all data in the file tmp_f_data = np.loadtxt(args.xvgfilename, skiprows = tmp_nb_rows_to_skip) #get nb of rows and cols nb_rows = np.shape(tmp_f_data)[0] nb_cols = np.shape(tmp_f_data)[1] -1 #get first col first_col = tmp_f_data[:,0] * float(args.xaxis) #stock data if scale_x_only: f_data = tmp_f_data[:,1:] else: f_data = tmp_f_data[:,1:] * conv_factor return
def solve_master(tree, num_node, Current_node, g_flag, thetaB_list, SUBD, coefficients, xBar, thetaOpt, lamOpt, muOpt, y, cons, iteration, pool=None): '''we solve the relaxed master problems based on thetaB_list, then select the infimum of all minimum values. Parameters: About the tree : tree, Current_node About the subproblem: SUBD, xBar, thetaOpt, lamOpt, muOpt, y About the boundary: theta_L, theta_U''' (M, N) = np.shape(y) K = np.shape(xBar[-1])[0] x_stor = None Q_stor = np.inf next_node = -1 #store all the MLBD MLBD_stor = [] #store all the MLBD if pool == None: tree.nodes[Current_node].set_parameters_qualifying_constraint(lamOpt, thetaOpt, muOpt, xBar, SUBD, g_flag, coefficients) #check whether the coefficients are already stored into the parents or not. print ('\n%d master problems are solving...' %len(thetaB_list)) for index in xrange(len(thetaB_list)): thetaB = thetaB_list[index].copy() status, objVal, xOpt, thetaB, lagrangian_coefficient= solve_master_s(tree, Current_node, coefficients, thetaOpt, xBar, lamOpt, muOpt, thetaB.copy(), y, g_flag, cons) #print objVal, xOpt if status == 2 and objVal < SUBD - np.spacing(1): node = tree.add_node(num_node, 0, 1, Current_node) node.set_parameters_thetaB(thetaB, xOpt, objVal, lagrangian_coefficient) MLBD_stor.append(objVal) if objVal < Q_stor: Q_stor = objVal next_node = num_node x_stor = xOpt num_node = num_node + 1 else: tree.nodes[Current_node].set_parameters_qualifying_constraint(lamOpt, thetaOpt, muOpt, xBar, SUBD, g_flag, coefficients) len_thetaB = len(thetaB_list) print ('\n%d master problems are solving...' %len_thetaB) results = [pool.apply_async(solve_master_s, args = (tree, Current_node, coefficients, thetaOpt, xBar, lamOpt, muOpt, thetaB.copy(), y, g_flag, cons)) for thetaB in thetaB_list] #put all the result into the tree. for p in results: #result = [status, objVal, xOpt, thetaB] result = p.get() if result[0] == 2 and result[1] < SUBD - np.spacing(1): node = tree.add_node(num_node, 0, 1, Current_node) node.set_parameters_thetaB(result[3], result[2], result[1], result[4]) #node.set_parameter(lamOpt, thetaOpt, result[3], muOpt, xBar, result[2], SUBD, result[1], g_flag, coefficients) MLBD_stor.append(result[1]) if result[1] < Q_stor: Q_stor = result[1] next_node = num_node x_stor = result[2] num_node += 1 return x_stor, Q_stor, next_node, num_node, MLBD_stor
def find_radial_centers(singleton): '''Intended to find areas where the rule-image is blue and return the center of such areas''' img=singleton.img Aufloesung=singleton.center_find_resolution quadrate=[] for x1 in range( np.shape(img)[0] //Aufloesung ): for x2 in range( np.shape(img)[1]//Aufloesung): quadrate.append([x1, x2]) quadrate= [q for q in quadrate if np.argmax(img[q[0]*Aufloesung][q[1]*Aufloesung]) == 2] #auf blaue reduzieren areas=[] while len(quadrate)>0: #Flaechen finden, also Quadrate auf Flaechen aufteilen areas.append([quadrate[0]]) quadrate.pop(0) h=0 while h<len( areas[len(areas)-1] ): q=areas[len(areas)-1][h] for nachbar in[[q[0]-1, q[1]], [q[0]+1, q[1]], [q[0], q[1]-1], [q[0], q[1]+1]]: if nachbar in quadrate: areas[len(areas)-1].append(nachbar) quadrate.remove(nachbar) h+=1 centers= [ np.array([sum( [x[0] for x in y])/len(y)*Aufloesung, sum([x[1] for x in y])/len(y)*Aufloesung]) for y in areas] return centers
def null(A, eps=1e-12): '''Compute a base of the null space of A.''' u, s, vh = np.linalg.svd(A) padding = max(0,np.shape(A)[1]-np.shape(s)[0]) null_mask = np.concatenate(((s <= eps), np.ones((padding,),dtype=bool)),axis=0) null_space = scipy.compress(null_mask, vh, axis=0) return scipy.transpose(null_space)
def count_clones_old(data): """ Count number of alive(cellcount > 0) clones at each timestep. Variables type: data - numpy array Output type: numpy 1-D array Details about data: data stores information about colonies in format time: 0 time: 1 time: 2 ... clone 1 23 23 24 clone 2 45 44 44 using numpy array >> data [[2, 1, 1, 0, 0], [0, 0, 1, 1, 0]] Usage example: >> data [[2,1,1,0,0], [0,0,1,1,0]] >> count_clones(data) [1,1,2,1,0] """ count = [] print "Shape of data array:", np.shape(data) for ind in range(np.shape(data)[1]): count.append(np.count_nonzero(data[:,ind])) return np.array(count)
def __init__(self, x, y): assert np.ndim(x)==2 and np.ndim(y)==2 and np.shape(x)==np.shape(y), \ 'x and y must be 2D arrays of the same size.' if np.any(np.isnan(x)) or np.any(np.isnan(y)): x = np.ma.masked_where( (isnan(x)) | (isnan(y)) , x) y = np.ma.masked_where( (isnan(x)) | (isnan(y)) , y) self.x_vert = x self.y_vert = y mask_shape = tuple([n-1 for n in self.x_vert.shape]) self.mask_rho = np.ones(mask_shape, dtype='d') # If maskedarray is given for verticies, modify the mask such that # non-existant grid points are masked. A cell requires all four # verticies to be defined as a water point. if isinstance(self.x_vert, np.ma.MaskedArray): mask = (self.x_vert.mask[:-1,:-1] | self.x_vert.mask[1:,:-1] | \ self.x_vert.mask[:-1,1:] | self.x_vert.mask[1:,1:]) self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d') if isinstance(self.y_vert, np.ma.MaskedArray): mask = (self.y_vert.mask[:-1,:-1] | self.y_vert.mask[1:,:-1] | \ self.y_vert.mask[:-1,1:] | self.y_vert.mask[1:,1:]) self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d') self._calculate_subgrids() self._calculate_metrics()
def get_boundaries(img, color_img_blanks): """This function calculates the upper and lower boundary of text in the document.""" img_height = np.shape(img)[0] blank_lines = [] first_non_blank = False upper_boundary = 0 lower_boundary = 0 for idx in range(img_height/2): line = img[idx,:] if len(np.where(line == 0)[0]) >= 50 and len(np.where(line == 0)[0]) < 0.1 * len(line): first_non_blank = True if len(np.where(line == 0)[0]) < 50 and first_non_blank: blank_lines.append(idx) cv2.line(color_img_blanks, (0, idx), (np.shape(img)[1], idx), (0,255,0), 1) first_non_blank = False for idx in reversed(range(img_height/2, img_height)): line = img[idx,:] if len(np.where(line == 0)[0]) >= 50 and len(np.where(line == 0)[0]) < 0.1 * len(line): first_non_blank = True if len(np.where(line == 0)[0]) < 50 and first_non_blank: blank_lines.append(idx) cv2.line(color_img_blanks, (0, idx), (np.shape(img)[1], idx), (0,255,0), 1) upper_blank_lines = [item for item in blank_lines if item < img_height/2] if len(upper_blank_lines) > 0: upper_boundary = longest_increasing_run(upper_blank_lines)[0] lower_blank_lines = [item for item in blank_lines if item >= img_height/2] if len(lower_blank_lines) > 0: lower_boundary = longest_increasing_run(lower_blank_lines)[0] return upper_boundary, lower_boundary
def equal(a, b, exact): if array_equal(a, b): return True if hasattr(a, 'dtype') and a.dtype in ['f4','f8']: nnans = isnan(a).sum() if nnans > 0: # For results containing NaNs, just check that the number # of NaNs is the same in both arrays. This check could be # made more exhaustive, but checking element by element in # python space is very expensive in general. return nnans == isnan(b).sum() ninfs = isinf(a).sum() if ninfs > 0: # Ditto for Inf's return ninfs == isinf(b).sum() if exact: return (shape(a) == shape(b)) and alltrue(ravel(a) == ravel(b), axis=0) else: if hasattr(a, 'dtype') and a.dtype == 'f4': atol = 1e-5 # Relax precission for special opcodes, like fmod else: atol = 1e-8 return (shape(a) == shape(b) and allclose(ravel(a), ravel(b), atol=atol))
def loadSamplePlanktons(numSamples=100, rotate=False, dim=28): if dim == 28: if not rotate: from pylearn2_plankton.planktonDataPylearn2 import PlanktonData ds = PlanktonData(which_set='train') designMatrix = ds.get_data()[0] # index 1 is the label print "Shape of Design Matrix", np.shape(designMatrix) designMatrix = np.reshape(designMatrix, (ds.get_num_examples(), 1, MAX_PIXEL, MAX_PIXEL) ) if numSamples != 'All': return np.array(designMatrix[:numSamples,...], dtype=np.float32) else: return np.array(designMatrix, dtype=np.float32) else: print "Loading Rotated Data" designMatrix = np.load(open(os.path.join(os.environ['PYLEARN2_DATA_PATH'] ,'planktonTrainRotatedX.p'), 'r')) return np.reshape(np.array(designMatrix[:numSamples,...], dtype=np.float32), (numSamples,1,MAX_PIXEL,MAX_PIXEL)) elif dim == 40: from pylearn2_plankton.planktonData40pixels import PlanktonData ds = PlanktonData(which_set='train') designMatrix = ds.get_data()[0] # index 1 is the label print "Shape of Design Matrix", np.shape(designMatrix) designMatrix = np.reshape(designMatrix, (ds.get_num_examples(), 1, 40, 40) ) if numSamples != 'All': return np.array(designMatrix[:numSamples,...], dtype=np.float32) else: return np.array(designMatrix, dtype=np.float32)
def test_shapes_scalarvalue_derivative(self): P = KroghInterpolator(self.xs,self.ys) n = P.n assert_array_equal(np.shape(P.derivatives(0)), (n,)) assert_array_equal(np.shape(P.derivatives(np.array(0))), (n,)) assert_array_equal(np.shape(P.derivatives([0])), (n,1)) assert_array_equal(np.shape(P.derivatives([0,1])), (n,2))
def pop_plot(data, thresh = a + L + 10., save = False, fname = '0.png'): '''Plot percentage of population in target zone.''' t = np.arange(0,np.shape(data)[0]) frac = 100*np.sum(data >= thresh, axis = 1)/np.float(np.shape(data)[1]) fig = plt.figure("Population Plot", (9,6)) ax = fig.add_subplot(111) #ax.scatter(t, frac, c = 'black', lw = 0, s = 5) ax.plot(t, frac, c = 'red', lw = 1) idx = (np.abs(frac - 75.0)).argmin() ax.hlines(75.0, t[0], t[-1], color = 'black', linestyle = 'dashed') ax.text(idx, 70.0, '$t_{75} = %d$' % idx, fontsize = 16) idx = (np.abs(frac - 50.0)).argmin() ax.hlines(50.0, t[0], t[-1], color = 'black', linestyle = 'dashed') ax.text(idx, 45.0, '$t_{50} = %d$' % idx, fontsize = 16) ax.grid(True) ax.axis([t[0],t[-1],0.0,100.0]) ax.set_ylabel("Population (%)", size = 16) ax.set_xlabel("Time (s)", size = 16) ax.set_title("Population vs. Time") # Choose whether or not to save the file. if save == False: plt.show() else: plt.savefig(fname) plt.close() return None
def test_array_sizes(): """Check we have the right sizes for the arrays""" assert(np.shape(snap.dm['pos']) == (4096, 3)) assert(np.shape(snap['vel']) == (8192, 3)) assert(np.shape(snap.gas['rho']) == (4039,)) assert(snap.gas['u'].dtype == np.float32) assert(snap.gas['iord'].dtype == np.int32)
def border_mask(img, p1, p2, device, debug, color="black"): # by using rectangle_mask to mask the edge of plotting regions you end up missing the border of the images by 1 pixel # This function fills this region in # note that p1 = (0,0) is the top left hand corner bottom right hand corner is p2 = (max-value(x), max-value(y)) # device = device number. Used to count steps in the pipeline # debug = True/False. If True; print output image if color=="black": ix, iy = np.shape(img) size = ix,iy bnk = np.zeros(size, dtype=np.uint8) cv2.rectangle(img = bnk, pt1 = p1, pt2 = p2, color = (255,255,255)) ret, bnk = cv2.threshold(bnk,127,255,0) contour,hierarchy = cv2.findContours(bnk,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE) cv2.drawContours(bnk, contour, -1 ,(255,255,255), 5) device +=1 if color=="gray": ix, iy = np.shape(img) size = ix,iy bnk = np.zeros(size, dtype=np.uint8) cv2.rectangle(img = bnk, pt1 = p1, pt2 = p2, color = (192,192,192)) ret, bnk = cv2.threshold(bnk,127,255,0) contour,hierarchy = cv2.findContours(bnk,cv2.RETR_TREE,cv2.CHAIN_APPROX_NONE) cv2.drawContours(bnk, contour, -1 ,(192,192,192), 5) device +=1 if debug: print_image(bnk, (str(device) + '_brd_mskd_' + '.png')) return device, bnk, contour, hierarchy
def mutual_information(self, x_index, y_index, log_base, debug=False): """ Calculate and return Mutual information between two random variables """ # Check if index are into the bounds assert (0 <= x_index <= self.n_rows) assert (0 <= y_index <= self.n_rows) # Variable to return MI summation = 0.0 # Get uniques values of random variables values_x = set(self.data[x_index]) values_y = set(self.data[y_index]) # Print debug info if debug: print 'MI between' print self.data[x_index] print self.data[y_index] # For each random for value_x in values_x: for value_y in values_y: px = shape(where(self.data[x_index] == value_x))[1] / self.n_cols py = shape(where(self.data[y_index] == value_y))[1] / self.n_cols pxy = len(where(in1d(where(self.data[x_index] == value_x)[0], where(self.data[y_index] == value_y)[0]) == True)[0]) / self.n_cols if pxy > 0.0: summation += pxy * math.log((pxy / (px * py)), log_base) if debug: print '(%d,%d) px:%f py:%f pxy:%f' % (value_x, value_y, px, py, pxy) return summation
def heat_map(model, char2int, unit_num=args.unit_number, song_path=args.heat_map_path, file_path='saves/heat_map', find_special=args.find_special): """ Generate the heat map and show activation functions for the LSTM/GRU hidden cells :param model: model to pass in (pytorch model) :param char2int: dictionary to tokenize characters :type char2int: dict :param unit_num: which unit to display :type unit_num: int :param song_path: path to song file :type song_path: str :param file_path: path to where to save the heat map :type file_path: str :param find_special: Boolean to indicate whether to return body and header finders :type find_special: bool :return: None """ generated_songs = utils.song_parser(song_path) num_songs = len(generated_songs) for song_ind in range(num_songs): generated_song = generated_songs[song_ind] song_length = len(generated_song) tensor_song = utils.string_to_tensor([list(generated_song)], char2int , 1, song_length) tensor_song = Variable(tensor_song) if gpu: tensor_song = tensor_song.cuda() if find_special: output = model(tensor_song[0]) hidden, cell = model.hidden hidden = hidden.data.numpy() num_units = np.shape(hidden)[2] header_correlation = -99999999 body_correlation =-999999999 header_filter = np.ones(song_length) header_filter[song_length/8:]=-1 header_filter[:song_length/8]= 6 body_filter = np.ones(song_length) body_filter[:song_length/8] = -6 for unit in range(num_units): activations = [] for index in range(song_length): output = model(tensor_song[index]) hidden, cell = model.hidden hidden = hidden.data.numpy() activations.append(hidden[0,0,unit]) activations = np.asarray(activations) correlation = np.dot(activations,header_filter) if correlation > header_correlation: header_correlation = correlation header = (activations,unit) correlation = np.dot(activations,body_filter) if correlation > body_correlation: body_correlation = correlation body = (activations,unit) activations = header[0] print("Header Detector for song "+str(song_ind)+" is Unit: "+str(header[1])) print("Body Detector for song "+str(song_ind)+" is Unit: " +str(body[1])) else: activations = [] for index in range(song_length): #model.zero_grad() output = model(tensor_song[index]) hidden, cell = model.hidden cell = cell.data.numpy() hidden = hidden.data.numpy() # print cell[0,0,0] activations.append(hidden[0, 0, unit_num]) activations = np.asarray(activations) print("Song Length: "+str(song_length)) height = int(np.sqrt(song_length)) + 1 width = int(song_length/height) + 1 print("height %d"% height) print("width %d"% width) song_activations = np.zeros(height * width) song_activations[:song_length] = activations[:] song_activations = np.reshape(song_activations, (height, width)) song_activations = [x for x in song_activations[::-1]] fig = plt.figure() heatmap = plt.pcolormesh(song_activations,cmap = 'coolwarm') countH = height-1 countW = 0 for index in range(len(generated_song)): char = generated_song[index] if char == '\n': char = 'nl' elif char == ' ': char = 'sp' plt.text(countW, countH, char) countW += 1 if countW >= width: countH -= 1 countW = 0 plt.colorbar(heatmap) plt.show() fig.savefig(file_path+str(song_ind)+'.png')
for time in times: k=k+1 if (k%packet == 0): sys.stdout.write('.') index = bisect.bisect_left(times, time) index = max(0, index) index = min(index, len(times) - 1) contact_info_source._time = times[index] # fix: should be called by contact_source? contact_info_source.method() id_t = numpy.where(pos_data[:, 0] == times[index]) if numpy.shape(spos_data)[0] > 0: set_positionv(spos_data[:, 1], spos_data[:, 2], spos_data[:, 3], spos_data[:, 4], spos_data[:, 5], spos_data[:, 6], spos_data[:, 7], spos_data[:, 8]) set_positionv( pos_data[id_t, 1], pos_data[id_t, 2], pos_data[id_t, 3], pos_data[id_t, 4], pos_data[id_t, 5], pos_data[id_t, 6], pos_data[id_t, 7], pos_data[id_t, 8]) id_tv = numpy.where(velo_data[:, 0] == times[index]) set_velocityv( velo_data[id_tv, 1],
data_test = np.hstack((data_test,aug_test)) data_test = np.hstack((data_test,feat_test)) data_test = np.hstack((data_test,azure_test)) data_test = np.hstack((data_test,azure2_test)) data_test = np.hstack((data_test,chichi_test)) data_test = np.hstack((data_test,svd_test)) #Preprocessing preprocess = StandardScaler() preprocess.fit(np.vstack((data_train,data_test))) data_train = preprocess.transform(data_train) data_test = preprocess.transform(data_test) #Adding User ID and Course ID fuck_train = np.vstack((train_users_id,train_courses_id)) #fuck_train = np.vstack((fuck_train,bycourse_train)) #fuck_train = np.vstack((fuck_train,ps_train)) data_train = np.hstack((np.transpose(fuck_train),data_train)) print np.shape(data_train) fuck_test = np.vstack((test_users_id,test_courses_id)) #fuck_test = np.vstack((fuck_test,bycourse_test)) #fuck_test = np.vstack((fuck_test,ps_test)) data_test = np.hstack((np.transpose(fuck_test),data_test)) label_train[label_train==0]=-1 np.savetxt("data_train_forrgf", data_train, delimiter=" ",fmt="%s") np.savetxt("data_test_forrgf", data_test, delimiter=" ",fmt="%s") np.savetxt("label_train_forrgf", np.c_[label_train],delimiter=" ",fmt="%s")
def Model(): epoch=1 LR=0.1 Gamma=0.9 s=0.1 F1_1=np.random.normal(0,s,[3,5,5]) F1_2=np.random.normal(0,s,[3,5,5]) F1_3=np.random.normal(0,s,[3,5,5]) F1_4=np.random.normal(0,s,[3,5,5]) F1_5=np.random.normal(0,s,[3,5,5]) F1_6=np.random.normal(0,s,[3,5,5]) F1_7=np.random.normal(0,s,[3,5,5]) F1_8=np.random.normal(0,s,[3,5,5]) F1_9=np.random.normal(0,s,[3,5,5]) F1_10=np.random.normal(0,s,[3,5,5]) F2_1=np.random.normal(0,s,[10,3,3]) F2_2=np.random.normal(0,s,[10,3,3]) F2_3=np.random.normal(0,s,[10,3,3]) F2_4=np.random.normal(0,s,[10,3,3]) F2_5=np.random.normal(0,s,[10,3,3]) F2_6=np.random.normal(0,s,[10,3,3]) F2_7=np.random.normal(0,s,[10,3,3]) F2_8=np.random.normal(0,s,[10,3,3]) F2_9=np.random.normal(0,s,[10,3,3]) F2_10=np.random.normal(0,s,[10,3,3]) F2_11=np.random.normal(0,s,[10,3,3]) F2_12=np.random.normal(0,s,[10,3,3]) F2_13=np.random.normal(0,s,[10,3,3]) F2_14=np.random.normal(0,s,[10,3,3]) F2_15=np.random.normal(0,s,[10,3,3]) W1=np.random.normal(0,s,[200,540]) W2=np.random.normal(0,s,[10,200]) Update1W=np.zeros([200,540]) Update2W=np.zeros([10,200]) Update1f=np.zeros([3,3]) Update2f=np.zeros([3,3]) Update3f=np.zeros([3,3]) Update4f=np.zeros([3,3]) Update5f=np.zeros([3,3]) Update6f=np.zeros([3,3]) Update7f=np.zeros([3,3]) Update8f=np.zeros([3,3]) Update9f=np.zeros([3,3]) Update10f=np.zeros([3,3]) Update11f=np.zeros([3,3]) Update12f=np.zeros([3,3]) Update13f=np.zeros([3,3]) Update14f=np.zeros([3,3]) Update15f=np.zeros([3,3]) Update1ff=np.zeros([5,5]) Update2ff=np.zeros([5,5]) Update3ff=np.zeros([5,5]) Update4ff=np.zeros([5,5]) Update5ff=np.zeros([5,5]) Update6ff=np.zeros([5,5]) Update7ff=np.zeros([5,5]) Update8ff=np.zeros([5,5]) Update9ff=np.zeros([5,5]) Update10ff=np.zeros([5,5]) while(epoch<=10): correct=0 for i in range(500): image=x_train[i,:,:,:]/255 R=image[:,:,0] B=image[:,:,1] G=image[:,:,2] c=np.concatenate((R,B,G)) d_img=np.reshape(c,[3,32,32]) Y=y_Train[i] target = np.zeros([10,1]) + 0.01 target[Y]= 0.99 #Input Ready #Forward Propagation First_conv=Conv_layer(d_img,F1_1,F1_2,F1_3,F1_4,F1_5,F1_6,F1_7,F1_8,F1_9,F1_10) First_convS=Relu(First_conv) pool_1,mask1=Pooling_Layer(First_convS) Second_conv=Conv_layer1(pool_1,F2_1,F2_2,F2_3,F2_4,F2_5,F2_6,F2_7,F2_8,F2_9,F2_10,F2_11,F2_12,F2_13,F2_14,F2_15) Second_convS=Relu(Second_conv) pool_2,mask2=Pooling_Layer(Second_convS) flat_layer=pool_2.flatten() # 16 nodes flat_layer=np.reshape(flat_layer,(540,1)) Z1=np.dot(W1,flat_layer) A1=Relu(Z1) Z2=np.dot(W2,A1) A2=sigmoid(Z2) output=np.reshape(A2,(10,1)) #Cost FUnction Calculation Error=target-output cost=0.5*sum((Error)**2)/10 #Backward Propagation #dw2 ???? sigmoido_grad=np.multiply(output,1-output) er_and_sigmd=np.multiply(Error,sigmoido_grad) A1_T=A1.T dW2=np.dot(er_and_sigmd,A1_T) #dW1 ?? W2_T=W2.T er_flat=np.dot(W2_T,er_and_sigmd) sigmd_der_flat=dReLU(Z1) this=np.multiply(er_flat,sigmd_der_flat) flat_layer_T=flat_layer.T dW1=np.dot(this,flat_layer_T) #error propagated at flatten and pooling layer W1_T=W1.T err_flatten=np.dot(W1_T,this) FD=np.reshape(err_flatten,(15,6,6)) repeat_2=FD.repeat(2,axis=1).repeat(2,axis=2) maxpol2_eror=repeat_2*mask2 Sig_der_fil2=dReLU(Second_conv) err_Conv2=np.multiply(Sig_der_fil2,maxpol2_eror) F=[] cc=[] for n in range(15): F1=err_Conv2[n,:,:] dF2_1=np.zeros([10,3,3]) for m in range(10): b=Conv_layerB(pool_1[m,:,:],F1) dF2_1[m,:,:]=b F=np.append(F,[dF2_1]) dF2=np.reshape(F,(15,10,3,3)) df1=dF2[0,:,:,:] df2=dF2[1,:,:,:] df3=dF2[2,:,:,:] df4=dF2[3,:,:,:] df5=dF2[4,:,:,:] df6=dF2[5,:,:,:] df7=dF2[6,:,:,:] df8=dF2[7,:,:,:] df9=dF2[8,:,:,:] df10=dF2[9,:,:,:] df11=dF2[10,:,:,:] df12=dF2[11,:,:,:] df13=dF2[12,:,:,:] df14=dF2[13,:,:,:] df15=dF2[14,:,:,:] F2_1R=np.rot90(F2_1,2, axes=(2,1)) F2_2R=np.rot90(F2_2,2, axes=(2,1)) F2_3R=np.rot90(F2_3,2, axes=(2,1)) F2_4R=np.rot90(F2_4,2, axes=(2,1)) F2_5R=np.rot90(F2_5,2, axes=(2,1)) F2_6R=np.rot90(F2_6,2, axes=(2,1)) F2_7R=np.rot90(F2_7,2, axes=(2,1)) F2_8R=np.rot90(F2_8,2, axes=(2,1)) F2_9R=np.rot90(F2_9,2, axes=(2,1)) F2_10R=np.rot90(F2_10,2, axes=(2,1)) F2_11R=np.rot90(F2_11,2, axes=(2,1)) F2_12R=np.rot90(F2_12,2, axes=(2,1)) F2_13R=np.rot90(F2_13,2, axes=(2,1)) F2_14R=np.rot90(F2_14,2, axes=(2,1)) F2_15R=np.rot90(F2_15,2, axes=(2,1)) cc=np.append(cc,[F2_1R,F2_2R,F2_3R,F2_4R,F2_5R,F2_6R,F2_7R,F2_8R,F2_9R,F2_10R,F2_11R,F2_12R,F2_13R,F2_14R,F2_15R]) A_rotated=np.reshape(cc,(15,10,3,3)) #padding f=np.shape(err_Conv2) padded_err_Conv2=np.zeros([15,16,16]) for t in range(f[0]): padded_err_Conv2[t,:,:]=np.pad(err_Conv2[t], (2, 2), 'constant', constant_values=(0,0)) Accumulated_Error=np.zeros([10,14,14]) for k in range(15): F_P=padded_err_Conv2[k,:,:] for n in range(10): F_map=A_rotated[k,n,:,:] ER=Conv_layerB(F_P,F_map) Accumulated_Error[n,:,:]=Accumulated_Error[n,:,:]+ER repeat_1=Accumulated_Error.repeat(2,axis=1).repeat(2,axis=2) maxpol1_eror=repeat_1*mask1 Sig_der_fil1=dReLU(First_conv) err_Conv1=np.multiply(Sig_der_fil1,maxpol1_eror) #df1 calculation FF=[] for n in range(10): F1=err_Conv1[n,:,:] dF2_1=np.zeros([3,5,5]) for m in range(3): b=Conv_layerB(d_img[m,:,:],F1) dF2_1[m,:,:]=b FF=np.append(FF,[dF2_1]) dF1=np.reshape(FF,(10,3,5,5)) dff1=dF1[0,:,:,:] dff2=dF1[1,:,:,:] dff3=dF1[2,:,:,:] dff4=dF1[3,:,:,:] dff5=dF1[4,:,:,:] dff6=dF1[5,:,:,:] dff7=dF1[6,:,:,:] dff8=dF1[7,:,:,:] dff9=dF1[8,:,:,:] dff10=dF1[9,:,:,:] Update1W=Gamma*Update1W+(1-Gamma)*dW1 Update2W=Gamma*Update2W+(1-Gamma)*dW2 Update1f=Gamma*Update1f+(1-Gamma)*df1 Update2f=Gamma*Update2f+(1-Gamma)*df2 Update3f=Gamma*Update3f+(1-Gamma)*df3 Update4f=Gamma*Update4f+(1-Gamma)*df4 Update5f=Gamma*Update5f+(1-Gamma)*df5 Update6f=Gamma*Update6f+(1-Gamma)*df6 Update7f=Gamma*Update7f+(1-Gamma)*df7 Update8f=Gamma*Update8f+(1-Gamma)*df8 Update9f=Gamma*Update9f+(1-Gamma)*df9 Update10f=Gamma*Update10f+(1-Gamma)*df10 Update11f=Gamma*Update11f+(1-Gamma)*df11 Update12f=Gamma*Update12f+(1-Gamma)*df12 Update13f=Gamma*Update13f+(1-Gamma)*df13 Update14f=Gamma*Update14f+(1-Gamma)*df14 Update15f=Gamma*Update15f+(1-Gamma)*df15 Update1ff=Gamma*Update1ff+(1-Gamma)*dff1 Update2ff=Gamma*Update2ff+(1-Gamma)*dff2 Update3ff=Gamma*Update3ff+(1-Gamma)*dff3 Update4ff=Gamma*Update4ff+(1-Gamma)*dff4 Update5ff=Gamma*Update5ff+(1-Gamma)*dff5 Update6ff=Gamma*Update6ff+(1-Gamma)*dff6 Update7ff=Gamma*Update7ff+(1-Gamma)*dff7 Update8ff=Gamma*Update8ff+(1-Gamma)*dff8 Update9ff=Gamma*Update9ff+(1-Gamma)*dff9 Update10ff=Gamma*Update10ff+(1-Gamma)*dff10 W2=W2+LR*Update2W W1=W1+LR*Update1W F1_1=F1_1+LR*Update1ff F1_2=F1_2+LR*Update2ff F1_3=F1_3+LR*Update3ff F1_4=F1_4+LR*Update4ff F1_5=F1_5+LR*Update5ff F1_6=F1_6+LR*Update6ff F1_7=F1_7+LR*Update7ff F1_8=F1_8+LR*Update8ff F1_9=F1_9+LR*Update9ff F1_10=F1_10+LR*Update10ff F2_1=F2_1+LR*Update1f F2_2=F2_2+LR*Update2f F2_3=F2_3+LR*Update3f F2_4=F2_4+LR*Update4f F2_5=F2_5+LR*Update5f F2_6=F2_6+LR*Update6f F2_7=F2_7+LR*Update7f F2_8=F2_8+LR*Update8f F2_9=F2_9+LR*Update9f F2_10=F2_10+LR*Update10f F2_11=F2_11+LR*Update11f F2_12=F2_12+LR*Update12f F2_13=F2_13+LR*Update13f F2_14=F2_14+LR*Update14f F2_15=F2_15+LR*Update15f #print(Z2) cost_f.append(cost) print("Cost Function After ", epoch , " Epoch : ",cost) #accuracy calculation for j in range(200): image1=x_test[j,:,:,:]/255 R=image1[:,:,0] B=image1[:,:,1] G=image1[:,:,2] c=np.concatenate((R,B,G)) d_img=np.reshape(c,[3,32,32]) y=y_Test[j] First_conv=Conv_layer(d_img,F1_1,F1_2,F1_3,F1_4,F1_5,F1_6,F1_7,F1_8,F1_9,F1_10) First_convS=Relu(First_conv) pool_1,mask1=Pooling_Layer(First_convS) Second_conv=Conv_layer1(pool_1,F2_1,F2_2,F2_3,F2_4,F2_5,F2_6,F2_7,F2_8,F2_9,F2_10,F2_11,F2_12,F2_13,F2_14,F2_15) Second_convS=Relu(Second_conv) pool_2,mask2=Pooling_Layer(Second_convS) flat_layer=pool_2.flatten() # 16 nodes flat_layer=np.reshape(flat_layer,(540,1)) Z1=np.dot(W1,flat_layer) A1=Relu(Z1) Z2=np.dot(W2,A1) A2=sigmoid(Z2) output=np.reshape(A2,(10,1)) index = np.argmax(output,axis=0) #print(index,y) #print(y,index) if (index==y): correct+=1 accuracy=(correct/200) acc.append(accuracy) print("Test Accuracy After ", epoch , " Epoch : ",accuracy) for j in range(500): image1=x_train[j,:,:,:]/255 R=image1[:,:,0] B=image1[:,:,1] G=image1[:,:,2] c=np.concatenate((R,B,G)) d_img=np.reshape(c,[3,32,32]) y=y_Train[j] First_conv=Conv_layer(d_img,F1_1,F1_2,F1_3,F1_4,F1_5,F1_6,F1_7,F1_8,F1_9,F1_10) First_convS=Relu(First_conv) pool_1,mask1=Pooling_Layer(First_convS) Second_conv=Conv_layer1(pool_1,F2_1,F2_2,F2_3,F2_4,F2_5,F2_6,F2_7,F2_8,F2_9,F2_10,F2_11,F2_12,F2_13,F2_14,F2_15) Second_convS=Relu(Second_conv) pool_2,mask2=Pooling_Layer(Second_convS) flat_layer=pool_2.flatten() # 16 nodes flat_layer=np.reshape(flat_layer,(540,1)) Z1=np.dot(W1,flat_layer) A1=Relu(Z1) Z2=np.dot(W2,A1) A2=sigmoid(Z2) output=np.reshape(A2,(10,1)) index = np.argmax(output,axis=0) #print(index,y) #print(y,index) if (index==y): correct+=1 accuracy1=(correct/500) acctr.append(accuracy1) print("Train Accuracy After ", epoch , " Epoch : ",accuracy1) epoch =epoch + 1 return output
ax.set_title("Countdown Network") ax.set_xlabel("Epoch") ax.set_ylabel("Time step") cb = fig.colorbar(im) # Now: try to inspect output of LSTM neurons at intermediate times. This is also a nice example of how to use some smart keras functionality. from tensorflow.keras import Model # get a function that represents the mapping from the # network inputs to the neuron output values of the first LSTM layer: neuron_values = Model([rnn.inputs], [firstLSTMlayer.output]) batchsize=1 test_observations,test_target=produce_batch_counting(batchsize, timesteps) print(test_observations) the_values=neuron_values.predict_on_batch([test_observations]) np.shape(the_values) fig,ax=plt.subplots(figsize=(7,5)) im=ax.imshow(the_values[0,:,:],origin='lower',interpolation='nearest',aspect='auto') ax.set_title("Neuron Value") ax.set_xlabel("1st LSTM Index") ax.set_ylabel("Time step") cb = fig.colorbar(im) ax.set_xticks([0,1])
def generate_SeqFile_SpiralDiffusion(gx, gy, tr, n_shots, mg, ms, fA, n_slices, reps, st, tPlot, tReport, b_values, n_dirs, fov, Nx): #%% --- 1 - Create new Sequence Object + Parameters seq = Sequence() # ========= # Parameters # ========= i_raster_time = 100000 assert 1 / i_raster_time == seq.grad_raster_time, "Manualy inputed inverse raster time does not match the actual value." # ========= # Code parameters # ========= fatsat_enable = 0 # Fat saturation kplot = 0 # ========= # Acquisition Parameters # ========= TR = tr # Spin-Echo parameters - TR in [s] n_TR = math.ceil( TR * i_raster_time) # Spin-Echo parameters - number of points TR bvalue = b_values # b-value [s/mm2] nbvals = np.shape(bvalue)[0] # b-value parameters ndirs = n_dirs # b-value parameters Ny = Nx slice_thickness = st # Acquisition Parameters in [m] Nshots = n_shots # ========= # Gradient Scaling # ========= gscl = np.zeros(nbvals + 1) gscl[1:] = np.sqrt(bvalue / np.max(bvalue)) gdir, nb0s = difunc.get_dirs(ndirs) # ========= # Create system # ========= system = Opts(max_grad=mg, grad_unit='mT/m', max_slew=ms, slew_unit='T/m/s', rf_ringdown_time=20e-6, rf_dead_time=100e-6, adc_dead_time=10e-6) #%% --- 2 - Fat saturation if fatsat_enable: fatsat_str = "_fatsat" b0 = 1.494 sat_ppm = -3.45 sat_freq = sat_ppm * 1e-6 * b0 * system.gamma rf_fs, _, _ = make_gauss_pulse(flip_angle=110 * math.pi / 180, system=system, duration=8e-3, bandwidth=abs(sat_freq), freq_offset=sat_freq) gz_fs = make_trapezoid(channel='z', system=system, delay=calc_duration(rf_fs), area=1 / 1e-4) else: fatsat_str = "" #%% --- 3 - Slice Selection # ========= # Create 90 degree slice selection pulse and gradient # ========= flip90 = fA * pi / 180 rf, gz, _ = make_sinc_pulse(flip_angle=flip90, system=system, duration=3e-3, slice_thickness=slice_thickness, apodization=0.5, time_bw_product=4) # ========= # Refocusing pulse with spoiling gradients # ========= rf180, gz180, _ = make_sinc_pulse(flip_angle=math.pi, system=system, duration=5e-3, slice_thickness=slice_thickness, apodization=0.5, time_bw_product=4) rf180.phase_offset = math.pi / 2 gz_spoil = make_trapezoid(channel='z', system=system, area=6 / slice_thickness, duration=3e-3) #%% --- 4 - Gradients # ========= # Spiral trajectory # ========= G = gx + 1J * gy #%% --- 5 - ADCs / Readouts delta_k = 1 / fov adc_samples = math.floor( len(G) / 4 ) * 4 - 2 # Apparently, on Siemens the number of samples needs to be divisible by 4... adc = make_adc(num_samples=adc_samples, system=system, duration=adc_samples / i_raster_time) # ========= # Pre-phasing gradients # ========= pre_time = 1e-3 n_pre_time = math.ceil(pre_time * i_raster_time) gz_reph = make_trapezoid(channel='z', system=system, area=-gz.area / 2, duration=pre_time) #%% --- 6 - Obtain TE and diffusion-weighting gradient waveform # For S&T monopolar waveforms # From an initial TE, check we satisfy all constraints -> otherwise increase TE. # Once all constraints are okay -> check b-value, if it is lower than the target one -> increase TE # Looks time-inefficient but it is fast enough to make it user-friendly. # TODO: Re-scale the waveform to the exact b-value because increasing the TE might produce slightly higher ones. # Calculate some times constant throughout the process # We need to compute the exact time sequence. For the normal SE-MONO-EPI sequence micro second differences # are not important, however, if we wanna import external gradients the allocated time for them needs to # be the same, and thus exact timing is mandatory. With this in mind, we establish the following rounding rules: # Duration of RFs + spoiling, and EPI time to the center of the k-space is always math.ceil(). # The time(gy) refers to the number of blips, thus we substract 0.5 since the number of lines is always even. # The time(gx) refers to the time needed to read each line of the k-space. Thus, if Ny is even, it would take half of the lines plus another half. n_duration_center = 0 # The spiral starts right in 0 -- or ADC_dead_time?? rf_center_with_delay = rf.delay + calc_rf_center(rf)[0] n_rf90r = math.ceil((calc_duration(gz) - rf_center_with_delay + pre_time) / seq.grad_raster_time) n_rf180r = math.ceil((calc_duration(rf180) + 2 * calc_duration(gz_spoil)) / 2 / seq.grad_raster_time) n_rf180l = math.floor( (calc_duration(rf180) + 2 * calc_duration(gz_spoil)) / 2 / seq.grad_raster_time) # ========= # Find minimum TE considering the readout times. # ========= n_TE = math.ceil(20e-3 / seq.grad_raster_time) n_delay_te1 = -1 while n_delay_te1 <= 0: n_TE = n_TE + 2 n_tINV = math.floor(n_TE / 2) n_delay_te1 = n_tINV - n_rf90r - n_rf180l # ========= # Find minimum TE for the target b-value # ========= bvalue_tmp = 0 while bvalue_tmp < np.max(bvalue): n_TE = n_TE + 2 n_tINV = math.floor(n_TE / 2) n_delay_te1 = n_tINV - n_rf90r - n_rf180l delay_te1 = n_delay_te1 / i_raster_time n_delay_te2 = n_tINV - n_rf180r - n_duration_center delay_te2 = n_delay_te2 / i_raster_time # Waveform Ramp time n_gdiff_rt = math.ceil(system.max_grad / system.max_slew / seq.grad_raster_time) # Select the shortest available time n_gdiff_delta = min(n_delay_te1, n_delay_te2) n_gdiff_Delta = n_delay_te1 + 2 * math.ceil( calc_duration(gz_spoil) / seq.grad_raster_time) + math.ceil( calc_duration(gz180) / seq.grad_raster_time) gdiff = make_trapezoid(channel='x', system=system, amplitude=system.max_grad, duration=n_gdiff_delta / i_raster_time) # delta only corresponds to the rectangle. n_gdiff_delta = n_gdiff_delta - 2 * n_gdiff_rt bv = difunc.calc_bval(system.max_grad, n_gdiff_delta / i_raster_time, n_gdiff_Delta / i_raster_time, n_gdiff_rt / i_raster_time) bvalue_tmp = bv * 1e-6 # ========= # Show final TE and b-values: # ========= print("TE:", round(n_TE / i_raster_time * 1e3, 2), "ms") for bv in range(1, nbvals + 1): print( round( difunc.calc_bval(system.max_grad * gscl[bv], n_gdiff_delta / i_raster_time, n_gdiff_Delta / i_raster_time, n_gdiff_rt / i_raster_time) * 1e-6, 2), "s/mm2") TE = n_TE / i_raster_time TR = n_TR / i_raster_time #%% --- 7 - Crusher gradients gx_crush = make_trapezoid(channel='x', area=2 * Nx * delta_k, system=system) gz_crush = make_trapezoid(channel='z', area=4 / slice_thickness, system=system) # TR delay - Takes everything into account # Distance between the center of the RF90s must be TR # The n_pre_time here is the time used to drive the Gx, and Gy spiral gradients to zero. n_spiral_time = adc_samples n_tr_per_slice = math.ceil(TR / n_slices * i_raster_time) if fatsat_enable: n_tr_delay = n_tr_per_slice - (n_TE - n_duration_center + n_spiral_time) \ - math.ceil(rf_center_with_delay * i_raster_time) \ - n_pre_time \ - math.ceil(calc_duration(gx_crush, gz_crush) * i_raster_time) \ - math.ceil(calc_duration(rf_fs, gz_fs) * i_raster_time) else: n_tr_delay = n_tr_per_slice - (n_TE - n_duration_center + n_spiral_time) \ - math.ceil(rf_center_with_delay * i_raster_time) \ - n_pre_time \ - math.ceil(calc_duration(gx_crush, gz_crush) * i_raster_time) tr_delay = n_tr_delay / i_raster_time #%% --- 8 - Checks # ========= # Check TR delay time # ========= assert n_tr_delay > 0, "Such parameter configuration needs longer TR." # ========= # Delay time, # ========= # Time between the gradient and the RF180. This time might be zero some times, although it is not normal. if n_delay_te1 > n_delay_te2: n_gap_te1 = n_delay_te1 - n_delay_te2 gap_te1 = n_gap_te1 / i_raster_time gap_te2 = 0 else: n_gap_te2 = n_delay_te2 - n_delay_te1 gap_te2 = n_gap_te2 / i_raster_time gap_te1 = 0 #%% --- 9 - b-zero acquisition for r in range(reps): for d in range(nb0s): for nshot in range(Nshots): for s in range(n_slices): # Fat saturation if fatsat_enable: seq.add_block(rf_fs, gz_fs) # RF90 rf.freq_offset = gz.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf, gz) seq.add_block(gz_reph) # Delay for RF180 seq.add_block(make_delay(delay_te1)) # RF180 seq.add_block(gz_spoil) rf180.freq_offset = gz180.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf180, gz180) seq.add_block(gz_spoil) # Delay for spiral seq.add_block(make_delay(delay_te2)) # Read k-space # Imaging Gradient waveforms gx = make_arbitrary_grad(channel='x', waveform=np.squeeze( G[:, nshot].real), system=system) gy = make_arbitrary_grad(channel='y', waveform=np.squeeze( G[:, nshot].imag), system=system) seq.add_block(gx, gy, adc) # Make the spiral finish in zero - I use pre_time because I know for sure it's long enough. # Furthermore, this is after readout and TR is supposed to be long. amp_x = [G[:, nshot].real[-1], 0] amp_y = [G[:, nshot].imag[-1], 0] gx_to_zero = make_extended_trapezoid(channel='x', amplitudes=amp_x, times=[0, pre_time], system=system) gy_to_zero = make_extended_trapezoid(channel='y', amplitudes=amp_y, times=[0, pre_time], system=system) seq.add_block(gx_to_zero, gy_to_zero) seq.add_block(gx_crush, gz_crush) # Wait TR if tr_delay > 0: seq.add_block(make_delay(tr_delay)) #%% --- 9 - DWI acquisition for r in range(reps): for bv in range(1, nbvals + 1): for d in range(ndirs): for nshot in range(Nshots): for s in range(n_slices): # Fat saturation if fatsat_enable: seq.add_block(rf_fs, gz_fs) # RF90 rf.freq_offset = gz.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf, gz) seq.add_block(gz_reph) # Diffusion-weighting gradient gdiffx = make_trapezoid(channel='x', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 0], duration=calc_duration(gdiff)) gdiffy = make_trapezoid(channel='y', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 1], duration=calc_duration(gdiff)) gdiffz = make_trapezoid(channel='z', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 2], duration=calc_duration(gdiff)) seq.add_block(gdiffx, gdiffy, gdiffz) # Delay for RF180 seq.add_block(make_delay(gap_te1)) # RF180 seq.add_block(gz_spoil) rf180.freq_offset = gz180.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf180, gz180) seq.add_block(gz_spoil) # Diffusion-weighting gradient seq.add_block(gdiffx, gdiffy, gdiffz) # Delay for spiral seq.add_block(make_delay(gap_te2)) # Read k-space # Imaging Gradient waveforms gx = make_arbitrary_grad(channel='x', waveform=np.squeeze( G[:, nshot].real), system=system) gy = make_arbitrary_grad(channel='y', waveform=np.squeeze( G[:, nshot].imag), system=system) seq.add_block(gx, gy, adc) # Make the spiral finish in zero - I use pre_time because I know for sure it's long enough. # Furthermore, this is after readout and TR is supposed to be long. amp_x = [G[:, nshot].real[-1], 0] amp_y = [G[:, nshot].imag[-1], 0] gx_to_zero = make_extended_trapezoid( channel='x', amplitudes=amp_x, times=[0, pre_time], system=system) gy_to_zero = make_extended_trapezoid( channel='y', amplitudes=amp_y, times=[0, pre_time], system=system) seq.add_block(gx_to_zero, gy_to_zero) seq.add_block(gx_crush, gz_crush) # Wait TR if tr_delay > 0: seq.add_block(make_delay(tr_delay)) if tPlot: seq.plot() if tReport: print(seq.test_report()) seq.check_timing() return seq, TE, TR, fatsat_str
def generate_SeqFile_EPIDiffusion(FOV, nx, ny, ns, mg, ms, reps, st, tr, fA, b_values, n_dirs, partialFourier, tPlot, tReport): #%% --- 1 - Create new Sequence Object + Parameters seq = Sequence() # ========= # Parameters # ========= i_raster_time = 100000 assert 1 / i_raster_time == seq.grad_raster_time, "Manualy inputed inverse raster time does not match the actual value." # ========= # Code parameters # ========= fatsat_enable = 0 # Fat saturation kplot = 0 # ========= # Acquisition Parameters # ========= fov = FOV Nx = nx Ny = ny n_slices = ns TR = tr # Spin-Echo parameters - TR in [s] n_TR = math.ceil( TR * i_raster_time) # Spin-Echo parameters - number of points TR bvalue = b_values # b-value [s/mm2] nbvals = np.shape(bvalue)[0] # b-value parameters ndirs = n_dirs # b-value parameters slice_thickness = st # Acquisition Parameters in [m] # ========= # Partial Fourier # ========= pF = partialFourier Nyeff = int(pF * Ny) # Number of Ny samples acquired if pF is not 1: pF_str = "_" + str(pF) + "pF" else: pF_str = "" # ========= # Gradient Scaling # ========= gscl = np.zeros(nbvals + 1) gscl[1:] = np.sqrt(bvalue / np.max(bvalue)) gdir, nb0s = difunc.get_dirs(ndirs) # ========= # Create system # ========= system = Opts(max_grad=mg, grad_unit='mT/m', max_slew=ms, slew_unit='T/m/s', rf_ringdown_time=20e-6, rf_dead_time=100e-6, adc_dead_time=10e-6) #%% --- 2 - Fat saturation if fatsat_enable: fatsat_str = "_fatsat" b0 = 1.494 sat_ppm = -3.45 sat_freq = sat_ppm * 1e-6 * b0 * system.gamma rf_fs, _, _ = make_gauss_pulse(flip_angle=110 * math.pi / 180, system=system, duration=8e-3, bandwidth=abs(sat_freq), freq_offset=sat_freq) gz_fs = make_trapezoid(channel='z', system=system, delay=calc_duration(rf_fs), area=1 / 1e-4) else: fatsat_str = "" #%% --- 3 - Slice Selection # ========= # Create 90 degree slice selection pulse and gradient # ========= flip90 = fA * pi / 180 rf, gz, _ = make_sinc_pulse(flip_angle=flip90, system=system, duration=3e-3, slice_thickness=slice_thickness, apodization=0.5, time_bw_product=4) # ========= # Refocusing pulse with spoiling gradients # ========= rf180, gz180, _ = make_sinc_pulse(flip_angle=math.pi, system=system, duration=5e-3, slice_thickness=slice_thickness, apodization=0.5, time_bw_product=4) rf180.phase_offset = math.pi / 2 gz_spoil = make_trapezoid(channel='z', system=system, area=6 / slice_thickness, duration=3e-3) #%% --- 4 - Define other gradients and ADC events delta_k = 1 / fov k_width = Nx * delta_k dwell_time = seq.grad_raster_time # Full receiver bandwith readout_time = Nx * dwell_time # T_acq (acquisition time) flat_time = math.ceil( readout_time / seq.grad_raster_time) * seq.grad_raster_time gx = make_trapezoid(channel='x', system=system, amplitude=k_width / readout_time, flat_time=flat_time) adc = make_adc(num_samples=Nx, duration=readout_time, delay=gx.rise_time + flat_time / 2 - (readout_time - dwell_time) / 2) # ========= # Pre-phasing gradients # ========= pre_time = 1e-3 gx_pre = make_trapezoid(channel='x', system=system, area=-gx.area / 2, duration=pre_time) gz_reph = make_trapezoid(channel='z', system=system, area=-gz.area / 2, duration=pre_time) gy_pre = make_trapezoid( channel='y', system=system, area=-(Ny / 2 - 0.5 - (Ny - Nyeff)) * delta_k, duration=pre_time ) # Es -0.5 y no +0.5 porque hay que pensar en areas, no en rayas! # ========= # Phase blip in shortest possible time # ========= gy = make_trapezoid(channel='y', system=system, area=delta_k) dur = math.ceil( calc_duration(gy) / seq.grad_raster_time) * seq.grad_raster_time #%% --- 5 - Obtain TE and diffusion-weighting gradient waveform # ========= # Calculate some times constant throughout the process # ========= duration_center = math.ceil( (calc_duration(gx) * (Ny / 2 + 0.5 - (Ny - Nyeff)) + calc_duration(gy) * (Ny / 2 - 0.5 - (Ny - Nyeff))) / seq.grad_raster_time) * seq.grad_raster_time rf_center_with_delay = rf.delay + calc_rf_center(rf)[0] rf180_center_with_delay = rf180.delay + calc_rf_center(rf180)[0] # ========= # Find minimum TE considering the readout times. # ========= TE = 40e-3 # [s] delay_te2 = -1 while delay_te2 <= 0: TE = TE + 0.02e-3 # [ms] delay_te2 = math.ceil((TE / 2 - calc_duration(rf180) + rf180_center_with_delay - calc_duration(gz_spoil) - \ calc_duration(gx_pre, gy_pre) - duration_center) / seq.grad_raster_time) * seq.grad_raster_time # ========= # Find minimum TE for the target b-value # ========= bvalue_tmp = 0 while bvalue_tmp < np.max(bvalue): TE = TE + 2 * seq.grad_raster_time # [ms] delay_te1 = math.ceil((TE / 2 - calc_duration(gz) + rf_center_with_delay - pre_time - calc_duration(gz_spoil) - \ rf180_center_with_delay) / seq.grad_raster_time) * seq.grad_raster_time delay_te2 = math.ceil((TE / 2 - calc_duration(rf180) + rf180_center_with_delay - calc_duration(gz_spoil) - \ calc_duration(gx_pre, gy_pre) - duration_center) / seq.grad_raster_time) * seq.grad_raster_time # Waveform Ramp time gdiff_rt = math.ceil(system.max_grad / system.max_slew / seq.grad_raster_time) * seq.grad_raster_time # Select the shortest available time gdiff_delta = min(delay_te1, delay_te2) gdiff_Delta = math.ceil( (delay_te1 + 2 * calc_duration(gz_spoil) + calc_duration(gz180)) / seq.grad_raster_time) * seq.grad_raster_time gdiff = make_trapezoid(channel='x', system=system, amplitude=system.max_grad, duration=gdiff_delta) # delta only corresponds to the rectangle. gdiff_delta = math.ceil((gdiff_delta - 2 * gdiff_rt) / seq.grad_raster_time) * seq.grad_raster_time bv = difunc.calc_bval(system.max_grad, gdiff_delta, gdiff_Delta, gdiff_rt) bvalue_tmp = bv * 1e-6 # ========= # Show final TE and b-values: # ========= print("TE:", round(TE * 1e3, 2), "ms") for bv in range(1, nbvals + 1): print( round( difunc.calc_bval(system.max_grad * gscl[bv], gdiff_delta, gdiff_Delta, gdiff_rt) * 1e-6, 2), "s/mm2") # ========= # Crusher gradients # ========= gx_crush = make_trapezoid(channel='x', area=2 * Nx * delta_k, system=system) gz_crush = make_trapezoid(channel='z', area=4 / slice_thickness, system=system) #%% --- 6 - Delays # ========= # TR delay - Takes everything into account # EPI reading time: # Distance between the center of the RF90s must be TR # ========= EPI_time = calc_duration(gx) * Nyeff + calc_duration(gy) * (Nyeff - 1) if fatsat_enable: tr_delay = math.floor( (TR - (TE - duration_center + EPI_time) - rf_center_with_delay - calc_duration(gx_crush, gz_crush) \ - calc_duration(rf_fs, gz_fs)) \ / seq.grad_raster_time) * seq.grad_raster_time else: tr_delay = math.floor( (TR - (TE - duration_center + EPI_time) - rf_center_with_delay - calc_duration(gx_crush, gz_crush)) \ / seq.grad_raster_time) * seq.grad_raster_time # ========= # Check TR delay time # ========= assert tr_delay > 0, "Such parameter configuration needs longer TR." # ========= # Delay time # ========= # ========= # Time between the gradient and the RF180. This time might be zero some times, although it is not normal. # ========= gap_te1 = math.ceil((delay_te1 - calc_duration(gdiff)) / seq.grad_raster_time) * seq.grad_raster_time # ========= # Time between the gradient and the locate k-space gradients. # ========= gap_te2 = math.ceil((delay_te2 - calc_duration(gdiff)) / seq.grad_raster_time) * seq.grad_raster_time #%% --- 9 - b-zero acquisition for d in range(nb0s): for s in range(n_slices): # Fat saturation if fatsat_enable: seq.add_block(rf_fs, gz_fs) # RF90 rf.freq_offset = gz.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf, gz) seq.add_block(gz_reph) # Delay for RF180 seq.add_block(make_delay(delay_te1)) # RF180 seq.add_block(gz_spoil) rf180.freq_offset = gz180.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf180, gz180) seq.add_block(gz_spoil) # Delay for EPI seq.add_block(make_delay(delay_te2)) # Locate k-space seq.add_block(gx_pre, gy_pre) for i in range(Nyeff): seq.add_block(gx, adc) # Read one line of k-space if i is not Nyeff - 1: seq.add_block(gy) # Phase blip gx.amplitude = -gx.amplitude # Reverse polarity of read gradient seq.add_block(gx_crush, gz_crush) # Wait TR if tr_delay > 0: seq.add_block(make_delay(tr_delay)) #%% --- 10 - DWI acquisition for bv in range(1, nbvals + 1): for d in range(ndirs): for s in range(n_slices): # Fat saturation if fatsat_enable: seq.add_block(rf_fs, gz_fs) # RF90 rf.freq_offset = gz.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf, gz) seq.add_block(gz_reph) # Diffusion-weighting gradient gdiffx = make_trapezoid(channel='x', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 0], duration=calc_duration(gdiff)) gdiffy = make_trapezoid(channel='y', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 1], duration=calc_duration(gdiff)) gdiffz = make_trapezoid(channel='z', system=system, amplitude=system.max_grad * gscl[bv] * gdir[d, 2], duration=calc_duration(gdiff)) seq.add_block(gdiffx, gdiffy, gdiffz) # Delay for RF180 seq.add_block(make_delay(gap_te1)) # RF180 seq.add_block(gz_spoil) rf180.freq_offset = gz180.amplitude * slice_thickness * ( s - (n_slices - 1) / 2) seq.add_block(rf180, gz180) seq.add_block(gz_spoil) # Diffusion-weighting gradient seq.add_block(gdiffx, gdiffy, gdiffz) # Delay for EPI seq.add_block(make_delay(gap_te2)) # Locate k-space seq.add_block(gx_pre, gy_pre) for i in range(Nyeff): seq.add_block(gx, adc) # Read one line of k-space if i is not Nyeff - 1: seq.add_block(gy) # Phase blip gx.amplitude = -gx.amplitude # Reverse polarity of read gradient seq.add_block(gx_crush, gz_crush) # Wait TR if tr_delay > 0: seq.add_block(make_delay(tr_delay)) if tPlot: seq.plot() if tReport: print(seq.test_report()) seq.check_timing() return seq, TE, TR, fatsat_str
def predict(self, X): prob_matrix = np.zeros((np.shape(X)[0], self.num_classes)) for i in range(self.num_classes): prob_matrix[:, i] = self._softmax_score(X, i) print(prob_matrix) return np.max(X, axis=1)
def __init__(self, data, thr=0.2): self.dtype = data.dtype self.shape = np.shape(data) self.data = data.astype(np.float) self.thr = thr
batch_x_val = batch_x_val.cpu().data.numpy() batch_y_val = batch_y_val.cpu().data.numpy() output_val = output_val.cpu().data.numpy() output_val = np.moveaxis(output_val, 1, -1) seg_val = np.argmax(output_val[0], axis=-1) input_3D = batch_x_val[0][0] seed_3D = batch_x_val[0][1] truth_3D = batch_y_val[0] seg_3D = seg_val intersect = truth_3D + seg_3D combined = np.zeros(np.shape(seg_3D)) combined[truth_3D > 0] = 1 combined[seg_3D > 0] = 2 combined[intersect > 1] = 3 # plt.figure(); # ma = np.amax(combined, axis=0) # plt.imshow(ma, cmap='magma') """ Get sklearn metric """ from sklearn.metrics import jaccard_score #jacc_new = jaccard_score(truth_3D.flatten(), seg_3D.flatten())
def main(): # MAIN -- TRADES + EMCEE # READ COMMAND LINE ARGUMENTS cli = get_args() # STARTING TIME start = time.time() # RENAME working_path = cli.full_path nthreads=cli.nthreads np.random.RandomState(cli.seed) # INITIALISE TRADES WITH SUBROUTINE WITHIN TRADES_LIB -> PARAMETER NAMES, MINMAX, INTEGRATION ARGS, READ DATA ... pytrades_lib.pytrades.initialize_trades(working_path, cli.sub_folder, nthreads) # RETRIEVE DATA AND VARIABLES FROM TRADES_LIB MODULE #global n_bodies, n_planets, ndata, npar, nfit, dof, inv_dof n_bodies = pytrades_lib.pytrades.n_bodies # NUMBER OF TOTAL BODIES OF THE SYSTEM n_planets = n_bodies - 1 # NUMBER OF PLANETS IN THE SYSTEM ndata = pytrades_lib.pytrades.ndata # TOTAL NUMBER OF DATA AVAILABLE npar = pytrades_lib.pytrades.npar # NUMBER OF TOTAL PARAMATERS ~n_planets X 6 nfit = pytrades_lib.pytrades.nfit # NUMBER OF PARAMETERS TO FIT nfree = pytrades_lib.pytrades.nfree # NUMBER OF FREE PARAMETERS (ie nrvset) dof = pytrades_lib.pytrades.dof # NUMBER OF DEGREES OF FREEDOM = NDATA - NFIT global inv_dof #inv_dof = np.float64(1.0 / dof) inv_dof = pytrades_lib.pytrades.inv_dof # READ THE NAMES OF THE PARAMETERS FROM THE TRADES_LIB AND CONVERT IT TO PYTHON STRINGS #reshaped_names = pytrades_lib.pytrades.parameter_names.reshape((10,nfit), order='F').T #parameter_names = [''.join(reshaped_names[i,:]).strip() for i in range(0,nfit)] #parameter_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names, nfit, str_len=10) #trades_names = anc.convert_fortran2python_strarray(pytrades_lib.pytrades.parameter_names, #nfit, str_len=10 #) ##parameter_names = anc.trades_names_to_emcee(trades_names) str_len = pytrades_lib.pytrades.str_len temp_names = pytrades_lib.pytrades.get_parameter_names(nfit,str_len) trades_names = anc.convert_fortran_charray2python_strararray(temp_names) parameter_names = trades_names if(cli.trades_previous is not None): temp_names, trades_parameters = anc.read_fitted_file(cli.trades_previous) if(nfit != np.shape(trades_parameters)[0]): anc.print_both(' NUMBER OF PARAMETERS (%d) IN TRADES-PREVIOUS FILE DOES NOT' \ 'MATCH THE CURRENT CONFIGURATION nfit=%d\nSTOP' \ %(np.shape(trades_parameters)[0], nfit) ) sys.exit() del temp_names else: # INITIAL PARAMETER SET (NEEDED ONLY TO HAVE THE PROPER ARRAY/VECTOR) #fitting_parameters = pytrades_lib.pytrades.fitting_parameters trades_parameters = pytrades_lib.pytrades.fitting_parameters # save initial_fitting parameters into array original_fit_parameters = trades_parameters.copy() #fitting_parameters = anc.e_to_sqrte_fitting(trades_parameters, trades_names) fitting_parameters = trades_parameters trades_minmax = pytrades_lib.pytrades.parameters_minmax # PARAMETER BOUNDARIES parameters_minmax = trades_minmax.copy() #parameters_minmax[:,0] = anc.e_to_sqrte_fitting(parameters_minmax[:,0], trades_names) #parameters_minmax[:,1] = anc.e_to_sqrte_fitting(parameters_minmax[:,1], trades_names) # RADIAL VELOCITIES SET n_rv = pytrades_lib.pytrades.nrv n_set_rv = pytrades_lib.pytrades.nrvset # TRANSITS SET n_t0 = pytrades_lib.pytrades.nt0 #n_t0_sum = np.sum(n_t0) n_t0_sum = pytrades_lib.pytrades.ntts n_set_t0 = 0 for i in range(0, n_bodies): #if (np.sum(n_t0[i]) > 0): n_set_t0 += 1 if (n_t0[i] > 0): n_set_t0 += 1 # compute global constant for the loglhd global ln_err_const #try: ## fortran variable RV in python will be rv!!! #e_RVo = np.array(pytrades_lib.pytrades.ervobs[:], dtype=np.float64) #except: #e_RVo = np.array([0.], dtype=np.float64) #try: #e_T0o = np.array(pytrades_lib.pytrades.et0obs[:,:], dtype=np.float64).reshape((-1)) #except: #e_T0o = np.array([0.], dtype=np.float64) #ln_err_const = anc.compute_ln_err_const(dof, e_RVo, e_T0o, cli.ln_flag) ln_err_const = pytrades_lib.pytrades.ln_err_const # SET EMCEE PARAMETERS: nwalkers, nruns, nsave, npost = get_emcee_arguments(cli,nfit) # INITIALISE SCRIPT FOLDER/LOG FILE working_folder, run_log, of_run = init_folder(working_path, cli.sub_folder) anc.print_both('',of_run) anc.print_both(' ======== ',of_run) anc.print_both(' pyTRADES' ,of_run) anc.print_both(' ======== ',of_run) anc.print_both('',of_run) anc.print_both(' WORKING PATH = %s' %(working_path),of_run) anc.print_both(' NUMBER OF THREADS = %d' %(nthreads),of_run) anc.print_both(' dof = ndata(%d) - nfit(%d) - nfree(%d) = %d' %(ndata, nfit, nfree, dof),of_run) anc.print_both(' Total N_RV = %d for %d set(s)' %(n_rv, n_set_rv),of_run) anc.print_both(' Total N_T0 = %d for %d out of %d planet(s)' %(n_t0_sum, n_set_t0, n_planets),of_run) anc.print_both(' %s = %.7f' %('log constant error = ', ln_err_const),of_run) anc.print_both(' %s = %.7f' %('IN FORTRAN log constant error = ', pytrades_lib.pytrades.ln_err_const),of_run) anc.print_both(' seed = %s' %(str(cli.seed)), of_run) if(cli.trades_previous is not None): anc.print_both('\n ******\n INITIAL FITTING PARAMETERS FROM PREVIOUS' \ ' TRADES-EMCEE SIM IN FILE:\n %s\n ******\n' %(cli.trades_previous), of_run ) anc.print_both(' ORIGINAL PARAMETER VALUES -> 0000', of_run) fitness_0000, lgllhd_0000, check_0000 = pytrades_lib.pytrades.write_summary_files(0, original_fit_parameters) anc.print_both(' ', of_run) #anc.print_both(' TESTING LNPROB_SQ ...', of_run) lgllhd_zero = lnprob(trades_parameters) #lgllhd_sq_zero = lnprob(fitting_parameters, parameter_names) anc.print_both(' ', of_run) anc.print_both(' %15s %23s %23s' %('trades_names', 'original_trades', 'trades_par'), of_run) for ifit in range(0, nfit): anc.print_both(' %15s %23.16e %23.16e' %(trades_names[ifit], original_fit_parameters[ifit], trades_parameters[ifit]), of_run) anc.print_both(' ', of_run) anc.print_both(' %15s %23.16e %23.16e' %('lnprob', lgllhd_0000,lgllhd_zero), of_run) anc.print_both(' ', of_run) # INITIALISES THE WALKERS if(cli.emcee_previous is not None): anc.print_both(' Use a previous emcee simulation: %s' %(cli.emcee_previous), of_run) last_p0, old_nwalkers, last_done = anc.get_last_emcee_iteration(cli.emcee_previous, nwalkers) if(not last_done): anc.print_both('**STOP: USING A DIFFERENT NUMBER OF WALKERS (%d) W.R.T. PREVIOUS EMCEE SIMULATION (%d).' %(nwalkers, old_nwalkers), of_run) sys.exit() p0 = last_p0 else: p0 = compute_initial_walkers(nfit, nwalkers, fitting_parameters, parameters_minmax, parameter_names, cli.delta_sigma, of_run) anc.print_both(' emcee chain: nwalkers = %d nruns = %d' %(nwalkers, nruns), of_run) anc.print_both(' sampler ... ',of_run) # old version with threads #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, threads=nthreads) #sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob_sq, threads=nthreads, args=[parameter_names]) threads_pool = emcee.interruptible_pool.InterruptiblePool(nthreads) sampler = emcee.EnsembleSampler(nwalkers, nfit, lnprob, pool=threads_pool) anc.print_both(' ready to go', of_run) anc.print_both(' with nsave = %s' %(str(nsave)), of_run) sys.stdout.flush() #sys.exit() if (nsave != False): # save temporary sampling during emcee every nruns*10% #if(os.path.exists(os.path.join(working_folder, 'emcee_temp.hdf5')) and os.path.isfile(os.path.join(working_folder, 'emcee_temp.hdf5'))): #os.remove(os.path.join(working_folder, 'emcee_temp.hdf5')) if(os.path.exists(os.path.join(working_folder, 'emcee_summary.hdf5')) and os.path.isfile(os.path.join(working_folder, 'emcee_summary.hdf5'))): os.remove(os.path.join(working_folder, 'emcee_summary.hdf5')) f_hdf5 = h5py.File(os.path.join(working_folder, 'emcee_summary.hdf5'), 'a') f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) temp_dset = f_hdf5.create_dataset('chains', (nwalkers, nruns, nfit), dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree temp_lnprob = f_hdf5.create_dataset('lnprobability', (nwalkers, nruns), dtype=np.float64) temp_lnprob.attrs['ln_err_const'] = ln_err_const temp_acceptance = f_hdf5.create_dataset('acceptance_fraction', data=np.zeros((nfit)), dtype=np.float64) temp_acor = f_hdf5.create_dataset('autocor_time', data=np.zeros((nfit)), dtype=np.float64) f_hdf5.close() pos = p0 nchains = int(nruns/nsave) state=None anc.print_both(' Running emcee with temporary saving', of_run) sys.stdout.flush() for i in range(0, nchains): anc.print_both('', of_run) anc.print_both(' iter: %6d ' %(i+1), of_run) aaa = i*nsave bbb = aaa+nsave pos, prob, state = sampler.run_mcmc(pos, N=nsave, rstate0=state) anc.print_both('completed %d steps of %d' %(bbb, nruns), of_run) f_hdf5 = h5py.File(os.path.join(working_folder, 'emcee_summary.hdf5'), 'a') temp_dset = f_hdf5['chains'] #[:,:,:] temp_dset[:,aaa:bbb,:] = sampler.chain[:, aaa:bbb, :] temp_dset.attrs['completed_steps'] = bbb temp_lnprob = f_hdf5['lnprobability'] #[:,:] temp_lnprob[:, aaa:bbb] = sampler.lnprobability[:, aaa:bbb] shape_lnprob = sampler.lnprobability.shape acceptance_fraction = sampler.acceptance_fraction temp_acceptance = f_hdf5['acceptance_fraction'] temp_acceptance = acceptance_fraction #f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) mean_acceptance_fraction = np.mean(acceptance_fraction) #temp_chains_T = np.zeros((bbb, nwalkers, nfit)) #for ifit in range(0,nfit): #temp_chains_T[:,:,ifit] = sampler.chain[:, :bbb, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler, steps_done=bbb) temp_acor = f_hdf5['autocor_time'] temp_acor[...] = acor_time #f_hdf5.create_dataset('autocor_time', data=np.array(acor_temp, dtype=np.float64), dtype=np.float64) #f_hdf5.create_dataset('autocor_time', data=np.array(sampler.acor, dtype=np.float64), dtype=np.float64) # not working #print 'aaa = %6d bbb = %6d -> sampler.lnprobability.shape = (%6d , %6d)' %(aaa, bbb, shape_lnprob[0], shape_lnprob[1]) f_hdf5.close() sys.stdout.flush() anc.print_both('', of_run) anc.print_both('...done with saving temporary total shape = %s' %(str(np.shape(sampler.chain))), of_run) anc.print_both('', of_run) sys.stdout.flush() # RUN EMCEE AND RESET AFTER REMOVE BURN-IN #pos, prob, state = sampler.run_mcmc(p0, npost) #sampler.reset() #sampler.run_mcmc(pos, nruns, rstate0=state) else: # GOOD COMPLETE SINGLE RUNNING OF EMCEE, WITHOUT REMOVING THE BURN-IN anc.print_both(' Running full emcee ...', of_run) sys.stdout.flush() sampler.run_mcmc(p0, nruns) anc.print_both('done', of_run) anc.print_both('', of_run) sys.stdout.flush() flatchains = sampler.chain[:, :, :].reshape((nwalkers*nruns, nfit)) # full chain values acceptance_fraction = sampler.acceptance_fraction mean_acceptance_fraction = np.mean(acceptance_fraction) #autocor_time = sampler.acor temp_chains_T = np.zeros((bbb, nwalkers, nfit)) for ifit in range(0,nfit): temp_chains_T[:,:,ifit] = sampler.chain[:, :, ifit].T #acor_time = anc.compute_autocor_time(temp_chains_T, walkers_transposed=True) acor_time = anc.compute_acor_time(sampler) lnprobability = sampler.lnprobability # save chains with original shape as hdf5 file f_hdf5 = h5py.File(os.path.join(working_folder, 'emcee_summary.hdf5'), 'w') f_hdf5.create_dataset('chains', data=sampler.chain, dtype=np.float64) f_hdf5['chains'].attrs['nwalkers'] = nwalkers f_hdf5['chains'].attrs['nruns'] = nruns f_hdf5['chains'].attrs['nfit'] = nfit f_hdf5['chains'].attrs['nfree'] = nfree f_hdf5['chains'].attrs['completed_steps'] = nruns f_hdf5.create_dataset('parameter_names', data=parameter_names, dtype='S10') f_hdf5.create_dataset('boundaries', data=parameters_minmax, dtype=np.float64) f_hdf5.create_dataset('acceptance_fraction', data=acceptance_fraction, dtype=np.float64) f_hdf5.create_dataset('autocor_time', data=acor_time, dtype=np.float64) f_hdf5.create_dataset('lnprobability', data=lnprobability, dtype=np.float64) f_hdf5['lnprobability'].attrs['ln_err_const'] = ln_err_const f_hdf5.close() anc.print_both(" Mean_acceptance_fraction should be between [0.25-0.5] = %.6f" %(mean_acceptance_fraction), of_run) anc.print_both('', of_run) # close the pool of threads threads_pool.close() threads_pool.terminate() threads_pool.join() anc.print_both('COMPLETED EMCEE', of_run) elapsed = time.time() - start elapsed_d, elapsed_h, elapsed_m, elapsed_s = anc.computation_time(elapsed) anc.print_both('', of_run) anc.print_both(' pyTRADES: EMCEE FINISHED in %2d day %02d hour %02d min %.2f sec - bye bye' %(int(elapsed_d), int(elapsed_h), int(elapsed_m), elapsed_s), of_run) anc.print_both('', of_run) of_run.close() pytrades_lib.pytrades.deallocate_variables() return
def execute(vec, dx, dy, quan=1.0): """Calculate the divergence of Vector from dx and dy.\n\ for each output cell, \n u = Vector[0] \n v = Vector[1] \n diff_u = u[i+1,j]*q[i+1,j] - u[i-1,j]*q[i-1,j] \n diff_v = u[i,j+1]*q[i,j+1] - u[i,j-1]*q[i,j-1] \n dudx = diff_u/dx[i,j] \n dvdy = diff_v/dy[i,j] \n diverg[i,j] = (dudx + dvdy)/2 \n """ vec_U, vec_V = vec vshape = shape(vec_U) if len(vshape) < 2: raise TypeError, "Divergence: Vector must be at least 2-D." if vshape[0] < 3: raise ValueError, "Divergence: Vector's first dimension must be at least 3." if vshape[1] < 3: raise ValueError, "Divergence: Vector's second dimension must be at least 3." rslt = empty(vshape, dtype=vec_U.dtype) rslt[0, :] = NaN rslt[-1, :] = NaN rslt[1:-1, 0] = NaN rslt[1:-1, -1] = NaN if shape(quan) == (): quan_rl = quan quan_tb = quan else: quan_rl = quan[1:-1] quan_tb = quan[:, 1:-1] QU = quan_rl * vec_U[1:-1, :] QV = quan_tb * vec_V[:, 1:-1] diff_U = QU[:, 2:] - QU[:, 0:-2] diff_V = QV[0:-2, :] - QV[2:, :] # if dx is a constant or one-element array, just divide by it. # otherwise, divide each cell in diff_U by the corresponding cell in dx. dxshape = shape(dx) ldxs = len(dxshape) if ldxs == 0 or sum(dxshape) == ldxs: dudx = diff_U / dx elif dxshape == vshape: dudx = diff_U / dx[1:-1, 1:-1] else: raise TypeError, "Divergence: dx must be a scalar or the same shape as Vector." # if dy is a constant or one-element array, just divide by it. # otherwise, divide each cell in diff_V by the corresponding cell in dy. dyshape = shape(dy) ldys = len(dyshape) if ldys == 0 or sum(dyshape) == ldys: dvdy = diff_V / dy elif dyshape == vshape: dvdy = diff_V / dy[1:-1, 1:-1] else: raise TypeError, "Divergence: dy must be a scalar or the same shape as Vector." rslt[1:-1, 1:-1] = (dudx + dvdy) / 2 return rslt
23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, 36.13037109, -3.3407526 , -10.48023796], [ -19.74851799, 166.00675964, 66.80673981, 56.04732895, 0.34365317, 11.8905611 , 56.04732895, 0.34365317, 11.8905611 , 56.04732895, 0.34365317, 11.8905611 , 56.04732895, 0.34365317, 11.8905611 , 56.04732895, 0.34365317, 11.8905611 , -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, -29.4766407 , -43.37115479, 23.08412361, 35.89316177, -4.19855595, -9.6547451 ], [ -17.18582535, 166.63139343, 66.30813599, 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, 35.89316177, -4.19855595, -9.6547451 ], [ -17.18582535, 166.63139343, 66.30813599, 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , 55.81494522, 0.57037091, 12.9295435 , -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, -29.48789024, -43.1723938 , 23.43960762, 36.07330322, -4.09444809, -9.29074287] ], "float32") print np.shape(baseball)
file_path="/Users/thomasaref/Dropbox/Current stuff/Logbook/TA210715A46_cooldown1/Data_1008/TA46_refll_fluxpowswp_4p2GHz4pGHz.hdf5" with File(file_path, 'r') as f: print f["Traces"].keys() print f.attrs["comment"] print f["Instrument config"].keys() #ctl_frq=f["Instrument config"]['PXI Aeroflex 302x Signal Generator - GPIB: PXI3::13::INSTR, PXI SigGen at localhost'].attrs["Frequency"] #["Power"] probe_frq=f["Instrument config"]['Rohde&Schwarz Network Analyzer - IP: 169.254.107.192, at localhost'].attrs["Start frequency"] probe_pwr=f["Instrument config"]['Rohde&Schwarz Network Analyzer - IP: 169.254.107.192, at localhost'].attrs["Output power"] #print probe_frq, probe_pwr, ctl_frq print f["Data"]["Channel names"][:] Magvec=f["Traces"]["Rohde&Schwarz Network Analyzer - S12"]#[:] data=f["Data"]["Data"] #pwr2=data[:,0,:].astype(float64) print shape(data) yoko=data[:,0,0].astype(float64) pwr=data[0,1,:].astype(float64) print pwr fstart=f["Traces"]['Rohde&Schwarz Network Analyzer - S12_t0dt'][0][0] fstep=f["Traces"]['Rohde&Schwarz Network Analyzer - S12_t0dt'][0][1] print shape(Magvec) sm=shape(Magvec)[0] sy=shape(data) s=(sm, sy[0], sy[2]) print s Magcom=Magvec[:,0,:]+1j*Magvec[:,1,:] Magcom=reshape(Magcom, s, order="F") freq=linspace(fstart, fstart+fstep*(sm-1), sm) #
def err_band_sz1(self, orth=False, svar=False, repl=1000, signif=0.05, seed=None, burn=100, component=None): """ IRF Sims-Zha error band method 1. Assumes symmetric error bands around mean. Parameters ---------- orth : bool, default False Compute orthogonalized impulse responses repl : int, default 1000 Number of MC replications signif : float (0 < signif < 1) Significance level for error bars, defaults to 95% CI seed : int, default None np.random seed burn : int, default 100 Number of initial simulated obs to discard component : neqs x neqs array, default to largest for each Index of column of eigenvector/value to use for each error band Note: period of impulse (t=0) is not included when computing principle component References ---------- Sims, Christopher A., and Tao Zha. 1999. "Error Bands for Impulse Response". Econometrica 67: 1113-1155. """ model = self.model periods = self.periods irfs = self._choose_irfs(orth, svar) neqs = self.neqs irf_resim = model.irf_resim(orth=orth, repl=repl, steps=periods, seed=seed, burn=burn) q = util.norm_signif_level(signif) W, eigva, k = self._eigval_decomp_SZ(irf_resim) if component is not None: if np.shape(component) != (neqs, neqs): raise ValueError("Component array must be " + str(neqs) + " x " + str(neqs)) if np.argmax(component) >= neqs * periods: raise ValueError( "Atleast one of the components does not exist") else: k = component # here take the kth column of W, which we determine by finding the largest eigenvalue of the covaraince matrix lower = np.copy(irfs) upper = np.copy(irfs) for i in range(neqs): for j in range(neqs): lower[1:, i, j] = irfs[1:, i, j] + W[i, j, :, k[i, j]] * q * np.sqrt( eigva[i, j, k[i, j]]) upper[1:, i, j] = irfs[1:, i, j] - W[i, j, :, k[i, j]] * q * np.sqrt( eigva[i, j, k[i, j]]) return lower, upper
def average_y_over_kxky(y_vs_kxky, vec_time, t_range, interpolate, vec_kx, vec_ky): from stellapy.decorators.verbose_wrapper import indent # phi2_vs_kx_ky has dimensions (ntime, nakx, naky) # Here, we remove all values of phi2_vs_kx_ky of times out of the selected interval. y_to_avrg = y_vs_kxky[(vec_time > t_range[0]) & (vec_time < t_range[1]), :, :] # Each tile is the average of |phi(kx,ky)^2| over the time range y_avrg = np.nanmean(y_to_avrg, axis=0) # Remove broken tiles print( indent, 'Warning: all tiles with phi2<E-30 or phi2>100 are removed and shown in black in phi' + str(np.shape(y_avrg)) + '.') print(indent, ' Tiles lower than E-30 are at:', np.argwhere(y_avrg[:, :] < 1.E-30)) print(indent, ' Their values are:', y_avrg[y_avrg[:, :] < 1.E-30]) print(indent, ' Tiles bigger than 100 are at: ', np.argwhere(y_avrg[:, :] > 100)) print(indent, ' Their values are:', y_avrg[y_avrg[:, :] > 100]) filter_low = y_avrg[:, :] < 1.E-30 # kx,ky = 0,0 is 2.58485853e-32 filter_high = y_avrg[:, :] > 100 # kx,ky = left right,0 is 28 # Interpolate the data if interpolate: y_avrg[filter_low] = np.nan y_avrg[filter_high] = np.nan for index in np.argwhere( np.isnan(y_avrg)): # At ky,kx=0,0 the values are nan's if (index[0] <= 27 or index[0] == len(y_avrg[:, 1]) - 1) and (not index[0] == 0): y_avrg[index[0], index[1]] = y_avrg[index[0] - 1, index[1]] if (index[0] >= 28 or index[0] == 0) and (not index[0] == len(y_avrg[:, 1]) - 1): y_avrg[index[0], index[1]] = y_avrg[index[0] + 1, index[1]] function = interp2d(vec_ky, vec_kx, y_avrg, kind='linear') xnew = np.linspace(vec_ky[0], vec_ky[-1], int(len(vec_ky)) * interpolate) ynew = np.linspace(vec_kx[0], vec_kx[-1], int(len(vec_kx)) * interpolate) y_avrg_interp = function(xnew, ynew) vec_ky_interp, vec_kx_interp = np.meshgrid(xnew, ynew) vec_ky_interp = vec_ky_interp[0, :] vec_kx_interp = vec_kx_interp[:, 0] # Remove broken tiles if interpolate: y_avrg_interp[np.repeat(np.repeat(filter_low, interpolate, axis=1), interpolate, axis=0)] = np.nan y_avrg_interp[np.repeat(np.repeat(filter_high, interpolate, axis=1), interpolate, axis=0)] = np.nan if not interpolate: y_avrg[filter_low] = np.nan y_avrg[filter_high] = np.nan if not interpolate: return vec_kx, vec_ky, y_avrg if interpolate: return vec_kx_interp, vec_ky_interp, y_avrg_interp
def main(): """ Read input video and process it, the output video will be exported output_video path which can be set by input arguments. Example: python inference_video.py --config configs/config.json --input_video_path data/video/sample.mov --output_video data/videos/output.avi """ argparse = ArgumentParser() argparse.add_argument('--config', type=str, help='json config file path') argparse.add_argument('--input_video_path', type=str, help='the path of input video', default='') argparse.add_argument('--output_video', type=str, help='the name of output video file', default='face_mask_output.avi') args = argparse.parse_args() config_path = args.config cfg = Config(path=config_path) if args.input_video_path == '': input_path = cfg.APP_VIDEO_PATH else: input_path = args.input_video_path print("INFO: Input video is: ", input_path) output_path = args.output_video file_name_size = len(output_path.split('/')[-1]) output_dir = output_path[:-file_name_size] if not os.path.exists(output_dir): os.makedirs(output_dir) print("INFO: The output video will be exported at: ", output_path) detector_input_size = (cfg.DETECTOR_INPUT_SIZE[0], cfg.DETECTOR_INPUT_SIZE[1], 3) classifier_img_size = (cfg.CLASSIFIER_INPUT_SIZE, cfg.CLASSIFIER_INPUT_SIZE, 3) device = cfg.DEVICE detector = None classifier = None output_vidwriter = None if device == "x86": from libs.detectors.x86.detector import Detector from libs.classifiers.x86.classifier import Classifier elif device == "EdgeTPU": from libs.detectors.edgetpu.detector import Detector from libs.classifiers.edgetpu.classifier import Classifier elif device == "Jetson": from libs.detectors.jetson.detector import Detector from libs.classifiers.jetson.classifier import Classifier else: raise ValueError('Not supported device named: ', device) detector = Detector(cfg) classifier_model = Classifier(cfg) input_cap = cv.VideoCapture(input_path) print("INFO: Start inferencing") frame_id = 0 while (input_cap.isOpened()): ret, raw_img = input_cap.read() if output_vidwriter is None: output_vidwriter = cv.VideoWriter( output_path, cv.VideoWriter_fourcc('M', 'J', 'P', 'G'), 24, (raw_img.shape[1], raw_img.shape[0])) height, width = raw_img.shape[:2] if ret == False: break _, cv_image = input_cap.read() if np.shape(cv_image) != (): resized_image = cv.resize(cv_image, tuple(detector_input_size[:2])) rgb_resized_image = cv.cvtColor(resized_image, cv.COLOR_BGR2RGB) objects_list = detector.inference(rgb_resized_image) faces = [] cordinates = [] cordinates_head = [] for obj in objects_list: if 'bbox' in obj.keys(): face_bbox = obj['bbox'] # [ymin, xmin, ymax, xmax] xmin, xmax = np.multiply([face_bbox[1], face_bbox[3]], width) ymin, ymax = np.multiply([face_bbox[0], face_bbox[2]], height) croped_face = cv_image[int(ymin):int(ymin) + (int(ymax) - int(ymin)), int(xmin):int(xmin) + (int(xmax) - int(xmin))] # Resizing input image croped_face = cv.resize(croped_face, tuple(classifier_img_size[:2])) croped_face = cv.cvtColor(croped_face, cv.COLOR_BGR2RGB) # Normalizing input image to [0.0-1.0] croped_face = croped_face / 255.0 faces.append(croped_face) cordinates.append( [int(xmin), int(ymin), int(xmax), int(ymax)]) if 'bbox_head' in obj.keys(): head_bbox = obj['bbox_head'] # [ymin, xmin, ymax, xmax] xmin, xmax = np.multiply([head_bbox[1], head_bbox[3]], width) ymin, ymax = np.multiply([head_bbox[0], head_bbox[2]], height) cordinates_head.append( [int(xmin), int(ymin), int(xmax), int(ymax)]) faces = np.array(faces) face_mask_results, scores = classifier_model.inference(faces) for i, cor in enumerate(cordinates): if face_mask_results[i] == 1: color = (0, 0, 255) elif face_mask_results[i] == 0: color = (0, 255, 0) else: color = (0, 0, 0) cv.rectangle(raw_img, (cor[0], cor[1]), (cor[2], cor[3]), color, 2) for cor in cordinates_head: cv.rectangle(raw_img, (cor[0], cor[1]), (cor[2], cor[3]), (200, 200, 200), 2) output_vidwriter.write(raw_img) print('{} Frames number are processed. {} fps'.format( frame_id, detector.fps)) frame_id = frame_id + 1 else: continue input_cap.release() output_vidwriter.release() print('INFO: Finish:) Output video is exported at: ', output_path)
def err_band_sz2(self, orth=False, svar=False, repl=1000, signif=0.05, seed=None, burn=100, component=None): """ IRF Sims-Zha error band method 2. This method Does not assume symmetric error bands around mean. Parameters ---------- orth : bool, default False Compute orthogonalized impulse responses repl : int, default 1000 Number of MC replications signif : float (0 < signif < 1) Significance level for error bars, defaults to 95% CI seed : int, default None np.random seed burn : int, default 100 Number of initial simulated obs to discard component : neqs x neqs array, default to largest for each Index of column of eigenvector/value to use for each error band Note: period of impulse (t=0) is not included when computing principle component References ---------- Sims, Christopher A., and Tao Zha. 1999. "Error Bands for Impulse Response". Econometrica 67: 1113-1155. """ model = self.model periods = self.periods irfs = self._choose_irfs(orth, svar) neqs = self.neqs irf_resim = model.irf_resim(orth=orth, repl=repl, T=periods, seed=seed, burn=100) W, eigva, k = self._eigval_decomp_SZ(irf_resim) if component is not None: if np.shape(component) != (neqs, neqs): raise ValueError("Component array must be " + str(neqs) + " x " + str(neqs)) if np.argmax(component) >= neqs * periods: raise ValueError( "Atleast one of the components does not exist") else: k = component gamma = np.zeros((repl, periods + 1, neqs, neqs)) for p in range(repl): for i in range(neqs): for j in range(neqs): gamma[p, 1:, i, j] = W[i, j, k[i, j], :] * irf_resim[p, 1:, i, j] gamma_sort = np.sort(gamma, axis=0) #sort to get quantiles indx = round(signif / 2 * repl) - 1, round((1 - signif / 2) * repl) - 1 lower = np.copy(irfs) upper = np.copy(irfs) for i in range(neqs): for j in range(neqs): lower[:, i, j] = irfs[:, i, j] + gamma_sort[indx[0], :, i, j] upper[:, i, j] = irfs[:, i, j] + gamma_sort[indx[1], :, i, j] return lower, upper
def histogram_enhancement(im, etype='linear2', target=None, maxCount=255): """ Function to run histogram enhancement and histogram matching for a given image. The function returns an image after the histogram matching or enhancement has been performed. Args: im (array): image to be modified based on the etype or target etype (optional[string]): type of enhancement to perform Can be 'linear2', 'linear3', etc..., to do a linear enhancement based on the CDF. Can be 'equalize' to do enhancement based on a flat histogram. Can be 'match', to do an image or histogram (PDF) based enhancement. target (optional[image, histogram]): image or histogram to match against Can be None, to do enhancement based on other etypes. maxCount (optional[int]): the maximum value for a digital count. Returns: the enhanced image Raises: TypeError: if image is not a numpy ndarray TypeError: if target is not a numpy ndarray when performing a match ValueError: if etype did not contain a digit: e.g. 'linear3' ValueError: if etype is not 'linear', 'match', or 'equalize' """ outputImage = np.zeros(np.shape(im)) # get int type for the image (to return to that later) dtype = im.dtype # type checking, look at the above Raises section if (not isinstance(im, np.ndarray)): raise TypeError("image is not a numpy ndarray; use openCV's imread") # use the etype to determine type of transform if (not isinstance(etype, str)): raise TypeError("etype is not a string value") if (etype.find("linear") == 0): # linear was at index 0, so we're doing a linear transformation linearValue = etype.split("linear")[1] if (not linearValue.isdigit()): # check if "linear" was not followed by a digit raise ValueError("linear etype should contain a digit") # linear percentage to enhance by linPct = int(linearValue) # perform linear enhancement if (len(np.shape(im)) == 3): # color image, build special lookup table lut = build_color_linear_lookup_table(im, linPct, maxCount + 1) outputImage[:, :, 0] = lut[0][im[:, :, 0]] outputImage[:, :, 1] = lut[1][im[:, :, 1]] outputImage[:, :, 2] = lut[2][im[:, :, 2]] else: lut = build_linear_lookup_table(im, linPct, maxCount + 1) outputImage = lut[im] elif (etype == "match"): if (not isinstance(target, np.ndarray)): raise TypeError("target is not a numpy ndarray") else: # perform match enhancement if (len(np.shape(im)) == 3): # color image, build special lookup table lut = build_color_match_lookup_table(im, target, maxCount) outputImage[:, :, 0] = lut[0][im[:, :, 0]] outputImage[:, :, 1] = lut[1][im[:, :, 1]] outputImage[:, :, 2] = lut[2][im[:, :, 2]] else: lut = build_match_lookup_table(im, target, maxCount) outputImage = lut[im] elif (etype == "equalize"): # build pdf to match against equalizePDF = np.zeros(maxCount) equalizePDF.fill(1 / maxCount) # perform match enhancement if (len(np.shape(im)) == 3): # color image, build special lookup table lut = build_color_match_lookup_table(im, equalizePDF, maxCount) outputImage[:, :, 0] = lut[0][im[:, :, 0]] outputImage[:, :, 1] = lut[1][im[:, :, 1]] outputImage[:, :, 2] = lut[2][im[:, :, 2]] else: lut = build_match_lookup_table(im, equalizePDF, maxCount) outputImage = lut[im] else: raise (ValueError, "etype must be 'linear', 'match', or 'equalize'") outputImage = np.array(outputImage, dtype) """ # Compare Histograms ======================================================# print("plot original image") plotImgHist(im) print("plot output image") plotImgHist(outputImage) """ return outputImage
import numpy as np import csv import tensorflow as tf test_abs_move = False data = list(csv.reader(open("/home/lilach/MarketStudies/TestConverter_scaled.csv"))) data = np.array(data) rowSize = np.shape(data)[0] colSize = np.shape(data)[1] print('RowSize:', rowSize, 'ColSize:', colSize) lastRow = 50001 X = (data[1:lastRow,0:colSize-1]) Y = (data[1:lastRow,colSize-1:colSize]) X_TEST = (data[lastRow:lastRow+1000,0:colSize-1]) Y_TEST = (data[lastRow:lastRow+1000,colSize-1:colSize]) X = X.astype(np.float) Y = (Y.astype(np.float)) if test_abs_move: Y = np.abs(Y) #print(X) #print(Y) tf_x = tf.constant(dtype=tf.float32,value=X) y_true = tf.constant(dtype=tf.float32,value=Y) #Build layers l1 = tf.layers.Dense(units=20, activation=tf.sigmoid) y1 = l1(tf_x)
for i in range(0, epochs): for j in range(0, len(training_images), batch_size): print(j) if args.use_ref: fc1, fc1b, fc2, fc2b, fc3, fc3b, c1, c1b, c2, c2b, c3, c3b, c4, c4b, c5, c5b, pred, _total_correct, g, e, _ = sess.run([get_fc1_weights, get_fc1_bias, \ get_fc2_weights, get_fc2_bias, \ get_fc3_weights, get_fc3_bias, \ get_conv1_weights, get_conv1_bias, \ get_conv2_weights, get_conv2_bias, \ get_conv3_weights, get_conv3_bias, \ get_conv4_weights, get_conv4_bias, \ get_conv5_weights, get_conv5_bias, \ predict, total_correct, grads, error, optimizer]) print("fc1: ") print(np.shape(fc1), np.average(fc1), np.std(fc1)) print("fc2: ") print(np.shape(fc2), np.average(fc2), np.std(fc2)) print("fc3: ") print(np.shape(fc3), np.average(fc3), np.std(fc3)) # print ("E: ") # print (e[0] / 128.) # pretty sure the 2nd dim here is the bias # print (np.shape(g[7][0]), np.shape(g[7][1])) # print ("G: ") # print (g[7][0][0])
A fast way to calculate binomial coefficients by Andrew Dalke (contrib). """ if 0 <= k <= n: ntok = 1 ktok = 1 for t in range(1, min(k, n - k) + 1): ntok *= n ktok *= t n -= 1 return ntok // ktok else: return 0 #======================================================================= infile=sys.argv[1] dataframe=pandas.read_csv(infile) (a,b)= numpy.shape(dataframe) print (a) print (b) feature = dataframe.values[:,0:b-1] target = dataframe.values[:,b-1] data=feature n=b-1 omega=3 storedval=numpy.full((n,n),-2.0) #using the fact that normalized_mutual_info_score lies b/w 0 and 1 for i in range(n): print(i) var1=data[:,i] for j in range(n): #this mutual_info_score : we can use any other criteria here
def setLibor(self, libor): self.libor = libor / libor.loc[self.referencedate] #self.ntimes = np.shape(self.datelist)[0] self.ntrajectories = np.shape(self.libor)[1] self.ones = np.ones(shape=[self.ntrajectories])
def _get_masks(self,output, utt_info): '''estimate the masks Args: output: the output of a single utterance of the neural network tensor of dimension [Txfeature_dimension*emb_dim] Returns: the estimated masks''' embeddings = output['bin_emb'] noise_filter = output['noise_filter'] #only the non-silence bins will be used for the clustering mix_to_mask, _ = self.mix_to_mask_reader(self.pos) [T,F] = np.shape(mix_to_mask) emb_dim = np.shape(embeddings)[1]/F N = T*F if np.shape(embeddings)[0] != T: raise 'Number of frames in usedbins does not match the sequence length' if np.shape(noise_filter)[0] != T: raise 'Number of frames in usedbins does not match the sequence length' if np.shape(noise_filter)[1] != F: raise 'Number of noise filter outputs does not match number of frequency bins' #reshape the outputs emb_vec = embeddings[:T,:] emb_vec_resh = np.reshape(emb_vec,[T*F,emb_dim]) X_hat_clean = np.multiply(mix_to_mask,noise_filter[:T,:]) maxbin = np.max(X_hat_clean) floor=maxbin/self.usedbin_threshold #apply floor to get the used bins usedbins=np.greater(X_hat_clean,floor) noise_filter_reshape = np.reshape(noise_filter[:T,:],[T*F,1]) usedbins_resh = np.reshape(usedbins, T*F) #Only keep the active bins (above threshold) for clustering output_speech_resh = emb_vec_resh[usedbins_resh] # dim:N' x embdim (N' is number of bins that are used N'<N) if np.shape(output_speech_resh)[0] < 2: print 'insufficient bins with energie' return np.zeros([self.nrS,T,F]) #apply kmeans clustering and assign each bin to a clustering kmeans_model=KMeans(n_clusters=self.nrS, init='k-means++', n_init=10, max_iter=100, n_jobs=-1) for _ in range(5): # Sometime it fails due to some indexerror and I'm not sure why. Just retry then. max 5 times try: kmeans_model.fit(output_speech_resh) except IndexError: continue break A = kmeans_model.cluster_centers_ # dim: nrS x embdim prod_1 = np.matmul(A,emb_vec_resh.T) # dim: nrS x N numerator = np.exp(prod_1-np.max(prod_1,axis=0)) denominator = np.sum(numerator,axis=0) M = numerator/denominator M_final = np.multiply(M,np.transpose(noise_filter_reshape)) #reconstruct the masks from the cluster labels masks = np.reshape(M_final,[self.nrS,T,F]) np.save(os.path.join(self.center_store_dir,utt_info['utt_name']),kmeans_model.cluster_centers_) return masks
def train_net(model): f = open('data.pickle', 'wb') ''' VISUALIZE ''' desiredLayers = [0] desiredOutputs = [model.layers[i].output for i in desiredLayers] newModel = Model(model.inputs, desiredOutputs) normalise = 32 train_frames = 10000 t = 0 dataset = load_dataset() while t < train_frames: t += 1 # time.sleep(1) batch = random.sample(dataset,batch_size) inputs = [] labels = [] for i in range(batch_size): inputs.append(batch[i][0]) # inputs.append( process_img_cnn(batch[i][0] )) labels.append(batch[i][1]) inputs = np.asarray(inputs) # print(dataset[0]) # plt.imshow(inputs[0]) # plt.show() # print(inputs,file = f) pickle.dump(inputs,f) # f.close() # np.savetxt('text.txt',inputs) labels = np.asarray(labels) # print(np.shape(inputs[1])) # for layer in model.layers: # print(layer.get_weights()) # s # plt.savefig('Images/'+str(1)+'.png') # state = inputs[0] # misc.imshow(state) # state = np.expand_dims(state, axis=0) # arr = newModel.predict(state) # print(np.shape(arr)) # # print(np.shape(arr)) # # # print('count = ',np.count_nonzero(arr)) # for filter_ in range(arr.shape[3]): # # Get the 5x5x1 filter: # extracted_filter = arr[:, :, :, filter_] # print(np.shape(extracted_filter)) # # Get rid of the last dimension (hence get 5x5): # extracted_filter = np.squeeze(extracted_filter) # # # # display the filter (might be very small - you can resize the window) # misc.imshow(extracted_filter) # # # plt.imshow(arr[0,:,:,1]) # # # plt.savefig('Images/'+str(1)+'.png') print(np.shape(inputs)) print("Predicted = ",model.predict( inputs, batch_size=batch_size, verbose=0)) print("Labels = ",labels) model.fit( inputs,labels, batch_size=batch_size, nb_epoch=1, verbose=1 ) # plt.savefig('Images/'+str(game_state.num_steps)+'.png') # Save the model every 25,000 frames. if t % 500== 0: model.save_weights('saved-models/' + str(t) + '.h5', overwrite=True) print("Saving model %d" % (t))
def evaluate(self, X, *args, return_values_of="auto", return_as_dictionary=False, **kwargs): ''' Evaluate the given problem. The function values set as defined in the function. The constraint values are meant to be positive if infeasible. A higher positive values means "more" infeasible". If they are 0 or negative, they will be considered as feasible what ever their value is. Parameters ---------- X : np.array A two dimensional matrix where each row is a point to evaluate and each column a variable. return_as_dictionary : bool, default=False If this is true than only one object, a dictionary, is returned. This contains all the results that are defined by return_values_of. Otherwise, by default a tuple as defined is returned. return_values_of : list of strings, default='auto' You can provide a list of strings which defines the values that are returned. By default it is set to "auto" which means depending on the problem the function values or additional the constraint violation (if the problem has constraints) are returned. Otherwise, you can provide a list of values to be returned.\n Allowed is ["F", "CV", "G", "dF", "dG", "dCV", "feasible"] where the d stands for derivative and h stands for hessian matrix. Returns ------- A dictionary, if return_as_dictionary enabled, or a list of values as defined in return_values_of. ''' assert self.surrogate_model is not None, 'surrogate model must be set first before evaluation' # call the callback of the problem if self.callback is not None: self.callback(X) # make the array at least 2-d - even if only one row should be evaluated only_single_value = len(np.shape(X)) == 1 X = np.atleast_2d(X) # check the dimensionality of the problem and the given input if X.shape[1] != self.n_var: raise Exception('Input dimension %s are not equal to n_var %s!' % (X.shape[1], self.n_var)) # automatic return the function values and CV if it has constraints if not defined otherwise if type(return_values_of) == str and return_values_of == "auto": return_values_of = ["F"] if self.n_constr > 0: return_values_of.append("CV") # all values that are set in the evaluation function values_not_set = [val for val in return_values_of if val not in self.evaluation_of] # have a look if gradients are not set and try to use autograd and calculate grading if implemented using it gradients_not_set = [val for val in values_not_set if val.startswith("d")] # whether gradient calculation is necessary or not calc_gradient = (len(gradients_not_set) > 0) # handle hF (hessian) computation, which is not supported by Pymoo calc_hessian = (type(return_values_of) == list and 'hF' in return_values_of) # set in the dictionary if the output should be calculated - can be used for the gradient out = {} for val in return_values_of: out[val] = None # calculate the output array - either elementwise or not. also consider the gradient self._evaluate(X, out, *args, calc_gradient=calc_gradient, calc_hessian=calc_hessian, **kwargs) at_least2d(out) calc_gradient_of = [key for key, val in out.items() if "d" + key in return_values_of and out.get("d" + key) is None and (type(val) == autograd.numpy.numpy_boxes.ArrayBox)] if len(calc_gradient_of) > 0: deriv = self._calc_gradient(out, calc_gradient_of) out = {**out, **deriv} # convert back to conventional numpy arrays - no array box as return type for key in out.keys(): if type(out[key]) == autograd.numpy.numpy_boxes.ArrayBox: out[key] = out[key]._value # if constraint violation should be returned as well if self.n_constr == 0: CV = np.zeros([X.shape[0], 1]) else: CV = Problem.calc_constraint_violation(out["G"]) if "CV" in return_values_of: out["CV"] = CV # if an additional boolean flag for feasibility should be returned if "feasible" in return_values_of: out["feasible"] = (CV <= 0) # if asked for a value but not set in the evaluation set to None for val in return_values_of: if val not in out: out[val] = None # remove the first dimension of the output - in case input was a 1d- vector if only_single_value: for key in out.keys(): if out[key] is not None: out[key] = out[key][0, :] if return_as_dictionary: return out else: # if just a single value do not return a tuple if len(return_values_of) == 1: return out[return_values_of[0]] else: return tuple([out[val] for val in return_values_of])
def timeIntegration(params): """Sets up the parameters for time integration :param params: Parameter dictionary of the model :type params: dict :return: Integrated activity variables of the model :rtype: (numpy.ndarray,) """ dt = params["dt"] # Time step for the Euler intergration (ms) duration = params["duration"] # imulation duration (ms) RNGseed = params["seed"] # seed for RNG # ------------------------------------------------------------------------ # local parameters # See Papadopoulos et al., Relations between large-scale brain connectivity and effects of regional stimulation # depend on collective dynamical state, arXiv, 2020 tau_exc = params["tau_exc"] # tau_inh = params["tau_inh"] # c_excexc = params["c_excexc"] # c_excinh = params["c_excinh"] # c_inhexc = params["c_inhexc"] # c_inhinh = params["c_inhinh"] # a_exc = params["a_exc"] # a_inh = params["a_inh"] # mu_exc = params["mu_exc"] # mu_inh = params["mu_inh"] # # external input parameters: # Parameter of the Ornstein-Uhlenbeck process for the external input(ms) tau_ou = params["tau_ou"] # Parameter of the Ornstein-Uhlenbeck (OU) process for the external input ( mV/ms/sqrt(ms) ) sigma_ou = params["sigma_ou"] # Mean external excitatory input (OU process) (mV/ms) exc_ou_mean = params["exc_ou_mean"] # Mean external inhibitory input (OU process) (mV/ms) inh_ou_mean = params["inh_ou_mean"] # ------------------------------------------------------------------------ # global coupling parameters # Connectivity matrix # Interareal relative coupling strengths (values between 0 and 1), Cmat(i,j) connection from jth to ith Cmat = params["Cmat"] N = len(Cmat) # Number of nodes K_gl = params["K_gl"] # global coupling strength # Interareal connection delay lengthMat = params["lengthMat"] signalV = params["signalV"] if N == 1: Dmat = np.zeros((N, N)) else: # Interareal connection delays, Dmat(i,j) Connnection from jth node to ith (ms) Dmat = dp.computeDelayMatrix(lengthMat, signalV) Dmat[np.eye(len(Dmat)) == 1] = np.zeros(len(Dmat)) Dmat_ndt = np.around(Dmat / dt).astype( int) # delay matrix in multiples of dt params["Dmat_ndt"] = Dmat_ndt # ------------------------------------------------------------------------ # Initialization # Floating point issue in np.arange() workaraound: use integers in np.arange() t = np.arange(1, round(duration, 6) / dt + 1) * dt # Time variable (ms) sqrt_dt = np.sqrt(dt) max_global_delay = np.max(Dmat_ndt) startind = int(max_global_delay + 1) # timestep to start integration at exc_ou = params["exc_ou"] inh_ou = params["inh_ou"] exc_ext = params["exc_ext"] inh_ext = params["inh_ext"] # state variable arrays, have length of t + startind # they store initial conditions AND simulated data excs = np.zeros((N, startind + len(t))) inhs = np.zeros((N, startind + len(t))) # ------------------------------------------------------------------------ # Set initial values # if initial values are just a Nx1 array if np.shape(params["exc_init"])[1] == 1: exc_init = np.dot(params["exc_init"], np.ones((1, startind))) inh_init = np.dot(params["inh_init"], np.ones((1, startind))) # if initial values are a Nxt array else: exc_init = params["exc_init"][:, -startind:] inh_init = params["inh_init"][:, -startind:] # xsd = np.zeros((N,N)) # delayed activity exc_input_d = np.zeros(N) # delayed input to x inh_input_d = np.zeros(N) # delayed input to y if RNGseed: np.random.seed(RNGseed) # Save the noise in the activity array to save memory excs[:, startind:] = np.random.standard_normal((N, len(t))) inhs[:, startind:] = np.random.standard_normal((N, len(t))) excs[:, :startind] = exc_init inhs[:, :startind] = inh_init noise_exc = np.zeros((N, )) noise_inh = np.zeros((N, )) # ------------------------------------------------------------------------ return timeIntegration_njit_elementwise( startind, t, dt, sqrt_dt, N, Cmat, K_gl, Dmat_ndt, excs, inhs, exc_input_d, inh_input_d, exc_ext, inh_ext, tau_exc, tau_inh, a_exc, a_inh, mu_exc, mu_inh, c_excexc, c_excinh, c_inhexc, c_inhinh, noise_exc, noise_inh, exc_ou, inh_ou, exc_ou_mean, inh_ou_mean, tau_ou, sigma_ou, )
def visualize_2D_AE(model, training_df, validation_df, example_data, num_examples, batch_size, num_gpus, dims, iter_, n_cols=4, std_to_plot=2.5, summary_density=50, save_loc=False, n_samps_per_dim=8): """ Visualization of AE as it trains in 2D space """ # choose a color palette current_palette = sns.color_palette() # summarize training bins = [0] + np.unique( np.logspace(0, np.log2(np.max(training_df.batch + [100])), num=summary_density, base=2).astype('int')) training_df['batch_bin'] = pd.cut(training_df.batch + 1, bins, labels=bins[:-1]) training_summary = training_df.groupby(['batch_bin']).describe() validation_df['batch_bin'] = pd.cut(validation_df.batch + 1, bins, labels=bins[:-1]) validation_summary = validation_df.groupby(['batch_bin']).describe() validation_df[:3] # get reconstructions of example data example_recon, z = model.sess.run((model.x_tilde, model.z_x), {model.x_input: example_data}) # get Z representations of data z = np.array( generate_manifold(model, dims, iter_, num_examples, batch_size, num_gpus)) if np.shape(z)[1] == 2: # generate volumetric data x_bounds = [ -inv_z_score(std_to_plot, z[:, 0]), inv_z_score(std_to_plot, z[:, 0]) ] y_bounds = [ -inv_z_score(std_to_plot, z[:, 1]), inv_z_score(std_to_plot, z[:, 1]) ] maxx, maxy, hx, hy, pts = make_grid(x_bounds, y_bounds, maxx=int(n_samps_per_dim), maxy=int(n_samps_per_dim)) dets = metric_and_volume(model, maxx, maxy, hx, hy, pts, dims, batch_size) fig = plt.figure(figsize=(10, 10)) outer = gridspec.GridSpec(2, 2, wspace=0.2, hspace=0.2) scatter_ax = plt.Subplot(fig, outer[0]) scatter_ax.scatter(z_score(z[:, 0]), z_score(z[:, 1]), alpha=.1, s=3, color='k') scatter_ax.axis('off') scatter_ax.set_xlim([-std_to_plot, std_to_plot]) scatter_ax.set_ylim([-std_to_plot, std_to_plot]) fig.add_subplot(scatter_ax) if np.shape(z)[1] == 2: volume_ax = plt.Subplot(fig, outer[1]) volume_ax.axis('off') volume_ax.matshow(np.log2(dets), cmap=plt.cm.viridis) fig.add_subplot(volume_ax) recon_ax = gridspec.GridSpecFromSubplotSpec(int(n_cols), int(n_cols / 2), subplot_spec=outer[2], wspace=0.1, hspace=0.1) for axi in range(int(n_cols) * int(n_cols / 2)): recon_sub_ax = gridspec.GridSpecFromSubplotSpec( 1, 2, subplot_spec=recon_ax[axi], wspace=0.1, hspace=0.1) orig_ax = plt.Subplot(fig, recon_sub_ax[0]) orig_ax.matshow(np.squeeze(example_data[axi].reshape(dims)), origin='lower') orig_ax.axis('off') rec_ax = plt.Subplot(fig, recon_sub_ax[1]) rec_ax.matshow(np.squeeze(example_recon[axi].reshape(dims)), origin='lower') rec_ax.axis('off') fig.add_subplot(orig_ax) fig.add_subplot(rec_ax) error_ax = plt.Subplot(fig, outer[3]) #error_ax.plot(training_df.batch, training_df.recon_loss) training_plt, = error_ax.plot( training_summary.recon_loss['mean'].index.astype('int').values, training_summary.recon_loss['mean'].values, alpha=1, color=current_palette[0], label='training') error_ax.fill_between( training_summary.recon_loss['mean'].index.astype('int').values, training_summary.recon_loss['mean'].values - training_summary.recon_loss['std'].values, training_summary.recon_loss['mean'].values + training_summary.recon_loss['std'].values, alpha=.25, color=current_palette[0]) error_ax.fill_between( validation_summary.recon_loss['mean'].index.astype('int').values, validation_summary.recon_loss['mean'].values - validation_summary.recon_loss['std'].values, validation_summary.recon_loss['mean'].values + validation_summary.recon_loss['std'].values, alpha=.25, color=current_palette[1]) validation_plt, = error_ax.plot( validation_summary.recon_loss['mean'].index.astype('int').values, validation_summary.recon_loss['mean'].values - validation_summary.recon_loss['std'].values, alpha=1, color=current_palette[1], label='validation') error_ax.legend(handles=[validation_plt, training_plt], loc=1) error_ax.set_yscale("log") error_ax.set_xscale("log") fig.add_subplot(error_ax) if save_loc != False: if not os.path.exists('/'.join(save_loc.split('/')[:-1])): os.makedirs('/'.join(save_loc.split('/')[:-1])) plt.savefig(save_loc) plt.show()
def _array_like(x, x0): """Return ndarray `x` as same array subclass and shape as `x0`""" x = np.reshape(x, np.shape(x0)) wrap = getattr(x0, '__array_wrap__', x.__array_wrap__) return wrap(x)
# Create the model model = Sequential() # INPUT LAYER model.add(Dropout(0.0, input_shape=(inp_dim,))) model.add(Dense(inp_dim,activation='selu')) # HIDDEN 1 model.add(Dense(encoding_dim, activation='selu')) # HIDDEN 2 model.add(Dense(256, activation='selu')) # HIDDEN 3 model.add(Dense(encoding_dim, activation='selu')) model.add(Dense(out_dim, activation='selu')) # Compile the model sgd = SGD(lr=0.1, momentum=0.2, decay=1e-6, nesterov=False) model.compile(optimizer=sgd, loss='mse') model.summary() model.fit(train_input,train_output,epochs=30, batch_size=32, shuffle=True,callbacks=[LoggingCallback(logging.info)]) print("input shape is ------------------------------------------------------------------------------>",numpy.shape(train_input)) train_model() valid_model()