def MeshSmooth(NC, neighbList, Iter): print "Two-stage Taubin Smoothing" N1 = NC.shape[0] NPP1 = N1 / LIM for i in range(Iter): print " Iteration ", i + 1 print " Umbrella-Operator Step" Umb = -np.zeros((NC.shape)) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(MeshUmbrella)) for j in range(0, LIM): calc(np.array(range(0, NPP1)) + j * NPP1, NC, neighbList, Umb) for j in range(0, LIM): Umb[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] Umb[range(LIM * NPP1, N1), :] = MeshUmbrella(range(LIM * NPP1, N1), NC, neighbList, Umb) print " Smoothing Step" results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(MeshTaubin)) for j in range(0, LIM): calc(np.array(range(0, NPP1)) + j * NPP1, NC, neighbList, Umb) for j in range(0, LIM): NC[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] NC[range(LIM * NPP1, N1), :] = MeshTaubin(range(LIM * NPP1, N1), NC, neighbList, Umb) return NC
def MeshSmooth(nodes, NC, neighbList, Iter, listPos=0): # If listPos ==0: use neighbourlist position same as nodal position in nodes. i.e len(neighbList)==nodes.size # If listPos <>0: use neighbList[ndnr] where nodes[i]=ndnr print "Two-stage Taubin Smoothing" print "ListPos: ", listPos N1 = nodes.size NPP1 = N1 / LIM for i in range(Iter): print " Iteration ", i + 1 print " Umbrella-Operator Step" Umb = -np.zeros((NC.shape)) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(MeshUmbrella)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, nodes, NC, neighbList, Umb, listPos) for j in range(0, LIM): Umb[nodes[np.array(range(0, NPP1)) + j * NPP1], :] = results[j] Umb[nodes[range(LIM * NPP1, N1)], :] = MeshUmbrella( range(LIM * NPP1, N1), nodes, NC, neighbList, Umb, listPos) print " Smoothing Step" results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(MeshTaubin)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, nodes, NC, neighbList, Umb, listPos) for j in range(0, LIM): NC[nodes[np.array(range(0, NPP1)) + j * NPP1], :] = results[j] NC[nodes[range(LIM * NPP1, N1)], :] = MeshTaubin( range(LIM * NPP1, N1), nodes, NC, neighbList, Umb, listPos) return NC
def elemQual_mu(elem, NC, Tet, disp=1, deltPresc=0): if disp == 1: print 'Determine element Quality [mu]' gamma = 0.0000001 delta = 0 T1 = elem.shape[0] TPP1 = T1 / LIM Sn2 = np.zeros((elem.shape[0], )) Sig = np.zeros((elem.shape[0], )) if elem.size > LIM: results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(Sn2Sig)) for j in range(0, LIM): calc(elem[np.array(range(0, TPP1))] + j * TPP1, NC, Tet) for j in range(0, LIM): Sn2[j * TPP1:(1 + j) * TPP1], Sig[j * TPP1:(1 + j) * TPP1] = results[j] if np.array(range(LIM * TPP1, T1)).size > 0: Sn2[LIM * TPP1:T1], Sig[LIM * TPP1:T1] = Sn2Sig( elem[np.array(range(LIM * TPP1, T1))], NC, Tet) if np.min(Sig) < gamma: if disp == 1: print ' minnimum Sig = ', np.min(Sig) delta = np.sqrt(gamma * (gamma - np.min(Sig))) if delta < deltPresc: delta = deltPresc if disp == 1: print ' delta = ', delta h_sig = (Sig + np.sqrt(Sig * Sig + 4 * delta * delta)) / 2 return 3 * np.power(h_sig, 2. / 3) / Sn2, delta, Sn2, Sig
def Neigenvalues(NC,Tri,layer,ls): KDTNC = KDTree(NC,ls) VNORM = vertexnormal(NC,Tri) Neig = np.zeros(NC.shape) N1 = Neig.size/3; NPP1 = N1/LIM results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(NodeEigVal)) for j in range(0,LIM): calc(np.array(range(0,NPP1))+j*NPP1,NC,Tri,KDTNC,ls,VNORM,layer,Neig) for j in range(0,LIM): Neig[np.array(range(0,NPP1))+j*NPP1,] = results[j] Neig[range(LIM*NPP1,N1),]=NodeEigVal(range(LIM*NPP1,N1),NC,Tri,KDTNC,ls,VNORM,layer,Neig) #queue=pprocess.Queue(limit=LIM) #results = [] #Nneig = queue.manage(pprocess.MakeParallel(NodeEigVal)) #for i in range(0,N): #Nneig(i,NC,Tri,KDTNC,ls,VNORM,2) #for i,NE in queue: #Neig[i,] = NE rows=np.where(Neig[:,0]<Neig[:,1])[0] Neig[rows,:]=np.c_[Neig[rows,1],Neig[rows,0],Neig[rows,2]] rows=np.where(Neig[:,0]<Neig[:,2])[0] Neig[rows,:]=np.c_[Neig[rows,2],Neig[rows,1],Neig[rows,0]] rows=np.where(Neig[:,1]<Neig[:,2])[0] Neig[rows,:]=np.c_[Neig[rows,0],Neig[rows,2],Neig[rows,1]] return Neig
def filter_regions(multi_level_mask, thresholds_list, parameter_dict, one_label): one_threshold = thresholds_list[one_label] mask = (multi_level_mask > one_label) min_size = parameter_dict['small_vol_threshold'] label_image, bounding_box_slices = lyBWareaopen(mask, min_size) region_result = [False] '''single thread''' # for label_number in range(1, np.max(label_image) + 1): # single_result = region_task(label_number, label_image, bounding_box_slices, one_threshold, parameter_dict) # region_result.append(single_result) '''multi thread''' results = pprocess.Map(limit=pprocess.get_number_of_cores()) calc = results.manage(pprocess.MakeParallel(region_task)) for label_number in range(1, np.max(label_image) + 1): calc(label_number, label_image, bounding_box_slices, one_threshold, parameter_dict) for i, result in enumerate(results): region_result.append(result) region_result = np.array(region_result, np.bool) tuild_result = np.logical_not(region_result) label_image[tuild_result[label_image]] = 0 return (label_image > 0)
def main(): lidc_dict = get_series_uid_path_dict(LIDCPath) list_of_args = range(ProcessorNum) group_num = len(csvLines) // ProcessorNum cutPoint = np.empty([ProcessorNum, 2], dtype=int) for row in range(ProcessorNum): # start point cutPoint[row, 0] = row * group_num if row == ProcessorNum - 1: # stop point cutPoint[row, 1] = len(csvLines) else: # stop point cutPoint[row, 1] = row * group_num + group_num - 1 + 1 # starting parallel reading st = time.time() results = pprocess.Map() parallel_function = results.manage(pprocess.MakeParallel(write_csv)) for args in list_of_args: parallel_function(args, cutPoint[args, 0], cutPoint[args, 1], lidc_dict) print('\nStarting Parallel time {:.2f} seconds...'.format(time.time() - st)) st = time.time() results[:] # parallel_results = results[:] print('\nParallel costs {:.2f} seconds...'.format(time.time() - st))
def ensemble_predictions(members, testX, nproc): if nproc>6: nproc=6 def para_predict(modname): from keras.models import load_model model = load_model(modname) prediction = model.predict(testX) return prediction queue = pprocess.Queue(limit=nproc) calc = queue.manage(pprocess.MakeParallel(para_predict)) yhats = [] for i in members: calc(i) #yhats.append(para_predict(i,testX)) for preds in queue: yhats.append(preds) # make predictions #yhats = [model.predict(testX) for model in members] yhats = numpy.array(yhats) # weighted sum across ensemble members #summed = numpy.tensordot(yhats, weights, axes=((0),(0))) summed = numpy.sum(yhats, axis=0) # argmax across classes result = numpy.argmax(summed, axis=1) return result
def TetCellToVertex(NC, Connect, ScalarValue): N1 = NC.shape[0] NPP1 = N1 / LIM ConnectNC = np.c_[ np.sum( np.c_[NC[Connect[:, 0], 0], NC[Connect[:, 1], 0], NC[Connect[:, 2], 0], NC[Connect[:, 3], 0]], 1) / 4, np.sum( np.c_[NC[Connect[:, 0], 1], NC[Connect[:, 1], 1], NC[Connect[:, 2], 1], NC[Connect[:, 3], 1]], 1) / 4, np.sum( np.c_[NC[Connect[:, 0], 2], NC[Connect[:, 1], 2], NC[Connect[:, 2], 2], NC[Connect[:, 3], 2]], 1) / 4] Scalars = np.zeros(NC.shape[0]) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(weightSCVAL)) for j in range(LIM): calc( np.array(range(0, NPP1)) + j * NPP1, NC, Connect, ConnectNC, ScalarValue) for j in range(LIM): Scalars[np.array(range(0, NPP1)) + j * NPP1] = results[j] #for j in range(LIM): if N1 > LIM * NPP1: Scalars[range(LIM * NPP1, N1)] = weightSCVAL(np.array(range(LIM * NPP1, N1)), NC, Connect, ConnectNC, ScalarValue) return Scalars
def cast_unsuitable_regions_by_label_MT(multi_level_mask, thresholds_list, parameter_dict): '''cast small region as noise and big region as vessel''' nodule_mask = np.zeros_like(multi_level_mask, np.int8) multi_image_labels = range(int(np.max(multi_level_mask))) multi_image_labels.reverse() loop_times = len(multi_image_labels) # TODO map style parallezision shared_array_s = mp.Array(ctypes.c_int8, loop_times * np.size(nodule_mask)) shared_array = np.frombuffer(shared_array_s.get_obj(), dtype=np.int8).reshape((loop_times,) + nodule_mask.shape) num_of_proc = pprocess.get_number_of_cores() results = pprocess.Map(limit=num_of_proc / 2) para_func = results.manage(pprocess.MakeParallel(put_result_into_shared_memory)) for i in range(loop_times): one_label = multi_image_labels[i] para_func(shared_array, multi_level_mask, thresholds_list, parameter_dict, one_label) results.finish() for num_of_loop in range(shared_array.shape[0]): nodule_mask = np.logical_or(nodule_mask, shared_array[num_of_loop, ...]) datastate = shared_array_s.get_obj()._wrapper._state arenaobj = datastate[0][0] arenaobj.buffer.close() mp.heap.BufferWrapper._heap = mp.heap.Heap() return nodule_mask
def _sl_call(self, dataset, roi_ids, nproc): """Classical generic searchlight implementation """ assert (self.results_backend in ('native', 'hdf5')) # compute if nproc is not None and nproc > 1: # split all target ROIs centers into `nproc` equally sized blocks nproc_needed = min(len(roi_ids), nproc) nblocks = nproc_needed \ if self.nblocks is None else self.nblocks roi_blocks = np.array_split(roi_ids, nblocks) # the next block sets up the infrastructure for parallel computing # this can easily be changed into a ParallelPython loop, if we # decide to have a PP job server in PyMVPA import pprocess p_results = pprocess.Map(limit=nproc_needed) if __debug__: debug( 'SLC', "Starting off %s child processes for nblocks=%i" % (nproc_needed, nblocks)) compute = p_results.manage(pprocess.MakeParallel(self._proc_block)) for iblock, block in enumerate(roi_blocks): # should we maybe deepcopy the measure to have a unique and # independent one per process? seed = mvpa2.get_random_seed() compute(block, dataset, copy.copy(self.__datameasure), seed=seed, iblock=iblock) else: # otherwise collect the results in an 1-item list p_results = [ self._proc_block(roi_ids, dataset, self.__datameasure) ] # Finally collect and possibly process results # p_results here is either a generator from pprocess.Map or a list. # In case of a generator it allows to process results as they become # available result_ds = self.results_fx( sl=self, dataset=dataset, roi_ids=roi_ids, results=self.__handle_all_results(p_results)) # Assure having a dataset (for paranoid ones) if not is_datasetlike(result_ds): try: result_a = np.atleast_1d(result_ds) except ValueError, e: if 'setting an array element with a sequence' in str(e): # try forcing object array. Happens with # test_custom_results_fx_logic on numpy 1.4.1 on Debian # squeeze result_a = np.array(result_ds, dtype=object) else: raise result_ds = Dataset(result_a)
def pp_wavelength_loop(cube, head, wavels, out_cube, AO, psfvars, adrvals, newsize, outspax, adr_switch='ON', usecpus=mp.cpu_count() - 1): '''Function to take input datacube and process it iteratively through each wavelength channel as follows: - Generate PSF array for given channel - Add effect of ADR to channel - Convolve cube channel with PSF - Frebin up to chosen output spaxel scale Inputs: cube: Datacube head: Datacube header wavels: Wavelength array out_cube: Empty output cube AO: AO mode [LTAO, SCAO, Gaussian] psdvars: list containing [psfparams, psfspax, psfsize, [D,eps], res_jitter, seeing] adrvals: array of ADR values newsize: tuple containing (x_newsize, y_newsize) array sizes outspax: tuple containing (x, y) output spaxel scales adr_switch: On or OFF. Turns ADR effect on or off usecpus: no. of CPUs to use Output: cube: Processed cube head: Updated header inspax: Input spaxel scale (mas, mas) outspax: Output spaxel scale (mas, mas) ''' import pprocess as pp print 'Using ', usecpus, ' CPUs' queue = pp.Queue(limit=usecpus) waveloop = queue.manage(pp.MakeParallel(pp_wavelength_channel)) print "Initialising..." for lam in xrange(len(wavels)): waveloop(cube[lam, :, :], head, wavels[lam], lam, AO, psfvars, adrvals[lam], newsize, outspax, adr_switch) print "Processing..." for chan, it in queue: update_progress(n.round(it / float(len(wavels)), 2)) out_cube[it, :, :] = chan return out_cube, head
def pp_wavelength_loop(cube, head, wavels, out_cube, newsize, outspax, usecpus=mp.cpu_count() - 1): '''Function to take input datacube and process it iteratively through each wavelength channel as follows: - Generate PSF array for given channel - Add effect of ADR to channel - Convolve cube channel with PSF - Frebin up to chosen output spaxel scale Inputs: cube: Datacube head: Datacube header wavels: Wavelength array out_cube: Empty output cube newsize: tuple containing (x_newsize, y_newsize) array sizes outspax: tuple containing (x, y) output spaxel scales usecpus: no. of CPUs to use Output: cube: Processed cube head: Updated header inspax: Input spaxel scale (mas, mas) outspax: Output spaxel scale (mas, mas) ''' import pprocess as pp print 'Using ', usecpus, ' CPUs' queue = pp.Queue(limit=usecpus) waveloop = queue.manage(pp.MakeParallel(pp_wavelength_channel)) print "Initialising..." for lam in xrange(len(wavels)): waveloop(cube[lam, :, :], head, wavels[lam], lam, newsize, outspax) print "Processing..." for chan, it in queue: update_progress(n.round(it / float(len(wavels)), 2)) out_cube[it, :, :] = chan return out_cube, head
def TetListScalars(NodeTetList, ScalarVal, AoM=0): # if AoM ==0, use average, if AoM<>0, use the minimum associated with that point N1 = len(NodeTetList) NPP1 = N1 / LIM NodeScal = np.zeros((N1, )) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(TetListScalarsSec)) for j in range(LIM): calc(np.array(range(0, NPP1)) + j * NPP1, NodeTetList, ScalarVal, AoM) for j in range(LIM): NodeScal[np.array(range(0, NPP1)) + j * NPP1] = results[j] #for j in range(LIM): if N1 > LIM * NPP1: NodeScal[range(LIM * NPP1, N1)] = TetListScalarsSec( np.array(range(LIM * NPP1, N1)), NodeTetList, ScalarVal, AoM) return NodeScal
def __call__(self, callable, sequence, *args, **kw): "Wrap and invoke 'callable' for each element in the 'sequence'." if not isinstance(callable, pprocess.MakeParallel): wrapped = pprocess.MakeParallel(callable) else: wrapped = callable self.init() # Start processes for each element in the sequence. for i in sequence: self.start(wrapped, i, *args, **kw) # Access to the results occurs through this object. self.finish() return self
def ShapeHist(NC, Tri, FeatPoints, radius, thetaB=12, phiB=12, rhoB=6): # Construct Shape Context histogram for Feature points N1 = FeatPoints.size NPP1 = N1 / LIM print 'Get Triangle and Vertex normals' VNORM = vertexnormal(NC, Tri) KDTnc = KDTree(NC, 5) print 'Set up Polar Histogram for given points' PolarHist = np.zeros((N1, 12, 12, 6)) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(HistP)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, FeatPoints, NC, VNORM, KDTnc, radius, wd, thetaB, phiB, rhoB) for j in range(0, LIM): PolarHist[np.array(range(0, NPP1)) + j * NPP1, ] = results[j] PolarHist[np.array(range(LIM * NPP1, N1)), ] = HistP( np.array(range(LIM * NPP1, N1)), FeatPoints, NC, VNORM, KDTnc, radius, wd, thetaB, phiB, rhoB) return PolarHist
def LaplacMesh(nodes, NC, neighbList, Iter, listPos=0): # If listPos ==0: use neighbourlist position same as nodal position in nodes. i.e len(neighbList)==nodes.size # If listPos <>0: use neighbList[ndnr] where nodes[i]=ndnr print "Laplacian Smooth" N1 = nodes.size NPP1 = N1 / LIM for i in range(Iter): print " Iteration ", i + 1 Umb = np.zeros((NC.shape)) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(MeshUmbrella)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, nodes, NC, neighbList, Umb, listPos) for j in range(0, LIM): Umb[nodes[np.array(range(0, NPP1)) + j * NPP1], :] = results[j] Umb[nodes[range(LIM * NPP1, N1)], :] = MeshUmbrella( range(LIM * NPP1, N1), nodes, NC, neighbList, Umb, listPos) NC = NC + Umb return NC
def Get3DCube_multiprocess(csvpath, augpath, datapath): PPNum = 6 augLines = ReadCSV(augpath)[1:] csvLines = ReadCSV(csvpath)[1:] LIDCPath = '/home/yanghan/data/LIDC-IDRI' KAGGLEPath = '/home/yanghan/data/stage1' lidc_dict = GetAbsolutePath(LIDCPath, KAGGLEPath, csvLines) list_pp = range(PPNum) group_num = len(csvLines) // PPNum cutPoint = np.empty([PPNum, 2], dtype=int) for row in range(PPNum): # start point cutPoint[row, 0] = row * group_num if row == PPNum - 1: # stop point cutPoint[row, 1] = len(csvLines) else: # stop point cutPoint[row, 1] = row * group_num + group_num - 1 + 1 # starting parallel reading st = time.time() results = pprocess.Map() parallel_function = results.manage( pprocess.MakeParallel( write_csv(0, 0, len(csvLines), lidc_dict, csvLines, augLines, datapath, 1))) for args in list_pp: parallel_function(args, cutPoint[args, 0], cutPoint[args, 1], lidc_dict) print('\nStarting Parallel time {:.2f} seconds...'.format(time.time() - st)) st = time.time() results[:] # parallel_results = results[:] print('\nParallel costs {:.2f} seconds...'.format(time.time() - st))
def FeatPoints(Kmax, Kmin, NC, Tri, radius, alpha=0.5, beta=0.5): # Get feature points from computed maximum and minnimum feature curvature print 'Get Triangle and Vertex normals' VNORM = vertexnormal(NC, Tri) print 'Use Shape Index for Feature Point extraction' SI = 0.5 - np.arctan((Kmax + Kmin) / (Kmax - Kmin)) / np.pi KDTnc = KDTree(NC, 5) N1 = NC.shape[0] NPP1 = N1 / LIM FeatPoints = np.array([np.nan] * NC.shape[0]) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(IsFeat)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, NC, VNORM, KDTnc, radius, SI, alpha, beta, FeatPoints) for j in range(0, LIM): FeatPoints[np.array(range(0, NPP1)) + j * NPP1] = results[j] FeatPoints[np.array(range(LIM * NPP1, N1))] = IsFeat(np.array(range(LIM * NPP1, N1)), NC, VNORM, KDTnc, radius, SI, alpha, beta, FeatPoints) FeatPoints = FeatPoints[FeatPoints >= 0] return np.array(FeatPoints, int)
err_list = [] for x in freqs: flux_list.append(data["int_flux_"+x][brightsrcs]) # Error propagation: error on log(x) = err_x/x fitting_error = data["err_int_flux_"+x][brightsrcs]/data["int_flux_"+x][brightsrcs] err_list.append(np.sqrt(fitting_error**2 + calibration_error**2)) flux_array = np.transpose(np.ma.vstack(flux_list)).astype("float32") flux_array = np.ma.log(flux_array) flux_errors = np.transpose(np.ma.vstack(err_list)).astype("float32") #names = data["Name"][brightsrcs] #weights = 1/(flux_errors*flux_errors) results = pprocess.Map(limit=cores) calc = results.manage(pprocess.MakeParallel(fit_spectrum)) for i in range(0,len(brightsrcs)): calc(freq_array,flux_array[i],flux_errors[i]) # ,options.plot) # Unpack results alpha, err_alpha, amp, err_amp, chi2red = map(list, zip(*results)) # Convert to numpy arrays alpha = np.array(alpha, dtype="float32") err_alpha = np.array(err_alpha, dtype="float32") amp = np.array(amp, dtype="float32") err_amp = np.array(err_amp, dtype="float32") chi2red = np.array(chi2red, dtype="float32") # Exclude any sources which came out with NaN alphas or amps
def OptINT(inner, outer, NC, Tet, neighbListTet, iterations, boundPenalty=10): N1 = NC.shape[0] NPP1 = N1 / LIM NCcur = np.r_[NC] T1 = Tet.shape[0] TPP1 = T1 / LIM print 'Submesh Optimization on ', inner.size, ' of ', NC.shape[0], ' nodes' for i in range(iterations): print ' Iteration ', i + 1 NCprev = np.r_[NCcur] EQ, delt, Sn2, Sig = elemQual_mu(np.array(range(Tet.shape[0])), NCprev, Tet) np.ma.dump(EQ, 'ElQual' + str(i + 1)) #EQ = np.ma.load('ElQual1') print ' Max ', np.max(EQ), '; Min ', np.min( EQ), '; Average ', np.average(EQ) print ' number of degenerate elements : ', np.where(Sig < 0)[0].size print ' build optimization matrix G' global Gmat Gmat = ps.spmatrix.ll_mat(NC.shape[0] + outer.size, NC.shape[0]) Gmat.put(1, np.array(range(NC.shape[0])), np.array(range(NC.shape[0]))) rows = np.array(range(outer.size)) + NC.shape[0] Gmat.put(1, rows, outer) print ' number of initial non-zeros: ', Gmat.nnz EQinv = 1 / EQ GMparts = [[]] * LIM results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(US_Gmat)) for j in range(LIM): calc( np.array(range(0, NPP1)) + j * NPP1, NCprev, Tet, neighbListTet, EQinv, Sig, j) for j in range(LIM): vals, rows, cols = results[j] #for j in range(LIM): Gmat.put(vals, rows, cols) if N1 > LIM * NPP1: vals, rows, cols = US_Gmat(np.array(range(LIM * NPP1, N1)), NCprev, Tet, neighbListTet, EQinv, Sig, 1) Gmat.put(vals, rows, cols) print ' final number of non-zeros: ', Gmat.nnz Gmat.export_mtx('Gmat' + str(i + 1)) print ' do optimization using conjugate gradient' GTG = ps.spmatrix.dot(Gmat, Gmat) gVec = ps.spmatrix.ll_mat(NC.shape[0] + outer.size, 3) for j in outer: rows = np.array([j, j, j]) + NC.shape[0] cols = np.array([0, 1, 2]) vals = NC[j, ] gVec.put(vals, rows, cols) GTg = ps.spmatrix.dot(Gmat, gVec) GTgNUMPY = np.zeros(NC.shape) GTgNUMPY[GTg.keys()] = GTg.values() mu = GTg.norm('inf') mu = mu * mu if mu < 1: mu = 1 GTG.update_add_at(mu * np.ones((NC.shape[0], )), np.array(range(NC.shape[0])), np.array(range(NC.shape[0]))) RHS = GTgNUMPY + mu * NCprev X, Y, Z = np.zeros((NC.shape[0], )), np.zeros( (NC.shape[0], )), np.zeros((NC.shape[0], )) print 'X' infx, itex, resx = ps.itsolvers.cgs(GTG, RHS[:, 0], X, 0.00000000001, 100) print 'Y' infy, itey, resy = ps.itsolvers.cgs(GTG, RHS[:, 1], Y, 0.00000000001, 100) print 'Z' infz, itez, resz = ps.itsolvers.cgs(GTG, RHS[:, 2], Z, 0.00000000001, 100) NCcur = np.c_[X, Y, Z] np.ma.dump(NCcur, 'NCopt_temp' + str(i + 1)) return NCcur
def elasticsurf(NCB, ConnectB, LandmB, LandmB_NC, AllowableBI, NCT, ConnectT, AllowableT, UseN_B, UseN_T, k_max, USENORMALS, gamm=2, sigm0=10, f=1.0715): # Elastic surface registration: #inputs: # NCB,NCT: nodal coordinates of base and target surfaces # ConnectB;ConnectT: Base&target connectivity # LandmB,LandmB_NC landmarks that have to have a 1 to 1 correspondence (input 0 no landmarks are present) # UseN_B & AllowableB: Feature dependant nodes on Base-mesh (indices in NCB) and allowable triangles to match. # UseN_T & AllowableT: Selective Feature preserving nodes and triangles (indices in NCT and ConnectT) on target mesh. # k_max: maximum number of iterations ######## ADDITIONAL SETTINGS REQUIRED ARE SET INTERNAL TO CODE ######### print print "SELECTIVE MESH MORPHING ALGORITHM USING ELASTIC SURFACE REGISTRATION" print " -G.J.J.v.Rensburg - 22/04/2010-" t_start = time.clock() ConnectB = np.array(ConnectB, int) ConnectT = np.array(ConnectT, int) #LandmB = np.array(LandmB[:,0],int) # do -1 later to be consistent with python indexing, first need to do other "temporary landmarks"& check that they dont fall on actual landmark positions! # Settings for elastic surface registration: m = 20 # nearest neighbour parameter alph = 0.5 # normilization factor #gamm=2 # smoothing parameter1 #sigm0=10 # smoothing parameter2 #f=1.0715 # smoothing parameter3 Tol = 0.0001 # stopping criteria # determine N1,N2,T1 and T2: N1 = NCB.shape[0] N2 = NCT.shape[0] T1 = ConnectB.shape[0] T2 = ConnectT.shape[0] NL = LandmB.shape[0] # For parallel programming devide Nr of computations by number of parallel processes (LIM) NPP1 = N1 / LIM NPP2 = N2 / LIM ################################ INITIALIZE & NODES OF CONCERN: ################################# ######################################################################################################## print print print "Set up 1-ring neighbor list for all points on the generic mesh" #neighbList = [[0]]*N1 #results = pprocess.Map(limit=LIM) #calc = results.manage(pprocess.MakeParallel(Get1neigh)) #for j in range(0,LIM): #calc(np.array(range(0,NPP1))+j*NPP1,NCB,ConnectB) #for j in range(0,LIM): #neighbList[j*NPP1:(1+j)*NPP1] = results[j] #neighbList[LIM*NPP1:N1]=Get1neigh(np.array(range(LIM*NPP1,N1)),NCB,ConnectB) #np.ma.dump(neighbList,'SkullSurf_neighbList') neighbList = np.ma.load('SkullSurf_neighbList') print print "INITIALIZE SURFACE DEFORMATION" CONV = [] print " enquire nodes where required displacement is checked" ###remove Landmarks from FDNB and SFPNT: #for i in range(0,NL): #if find_repeats(np.r_[UseN_B,LandmB[i,]])[0].size>0: #r=np.where(UseN_B==LandmB[i,])[0] #UseN_B = np.r_[UseN_B[0:r,],UseN_B[r+1:UseN_B.size,]] SamplingB = UseN_B.size SamplingT = UseN_T.size ## Full list of nodes used in Surface registration: LMB = np.r_[ UseN_B] #,LandmB] # Last NL entries are reserved for Landmarks that HAVE TO FIT points on the target mesh LMT = np.r_[UseN_T] # For parallel programming devide Nr of computations by number of parallel processes (LIM) SBPP = SamplingB / LIM STPP = SamplingT / LIM FMorph = 0 print print "COARSE SURFACE REGISTRATION" #print " Compute known displacement for Base_Landmarks " #knownC = NCB[LandmB,] #knownD = LandmB_NC-knownC ####print " using landmark displacements to deform using RBF" ####W_km1 = RBFmorph(NCB,knownC,knownD) ####tic = time.clock() ####W_km1 = MeshSmooth(W_km1,neighbList,10) ####print " Smoothing done in ",time.clock()-tic," seconds" ####np.ma.dump(W_km1,'TempElasNodes_Iter'+str(k-1)+'_Time'+time.ctime()) #print 'Smooth Gaussian Weight deformation to align Landmarks to target positions' #k=0 #Err = 2 #W_km1 = np.r_[NCB] #while (k<100)|(Err>Tol): #k=k+1 #print 'Iteration : ',k #DS = np.zeros((N1,3)) #knownC = W_km1[LandmB,] #knownD = LandmB_NC-knownC #knownD[np.isnan(knownD)]=0 ## Deform mesh using Gaussian smoothing as suggested in paper by R.Bryan et al. #sigma_k2 = np.power(np.power(f,-k)*20,2) #results = pprocess.Map(limit=LIM) #calc = results.manage(pprocess.MakeParallel(GaussianSmooth)) #for j in range(0,LIM): #calc(np.array(range(0,NPP1))+j*NPP1,W_km1,knownC,knownD,sigma_k2,gamm) #for j in range(0,LIM): #DS[np.array(range(0,NPP1))+j*NPP1,:] = results[j] #DS[range(LIM*NPP1,N1),:]=GaussianSmooth(np.array(range(LIM*NPP1,N1)),W_km1,knownC,knownD,sigma_k2,gamm) #DS[np.isnan(DS)]=0 #W_km1 = W_km1+DS #Err = np.sum(np.sqrt(np.sum(DS*DS,1)),0)/N1 #W_km1 = MeshSmooth(W_km1,neighbList,10) #np.ma.dump(W_km1,'TempElasNodes_Iter0_TimeWedMar14_2011_20') ###np.ma.dump(W_km1,'TempElasNodes_Iter0_Time'+time.ctime()) W_km1 = NCB ################################ MAIN MESH DEFORMATION ALGORITHM: ################################# ######################################################################################################## k = 1 print print "ELASTIC SURFACE REGISTRATION" print "determine vertex normals of target surface" #Compute target-mesh triangle centroids: print "determining centroids of target surface triangles" S_2_centr = np.c_[np.sum( np.c_[NCT[ConnectT[:, 0], 0], NCT[ConnectT[:, 1], 0], NCT[ConnectT[:, 2], 0]], 1) / 3, np.sum( np.c_[NCT[ConnectT[:, 0], 1], NCT[ConnectT[:, 1], 1], NCT[ConnectT[:, 2], 1]], 1) / 3, np.sum( np.c_[NCT[ConnectT[:, 0], 2], NCT[ConnectT[:, 1], 2], NCT[ConnectT[:, 2], 2]], 1) / 3] print "determine triangle and vertex normals of target surface" TNORMT = np.cross(NCT[ConnectT[:, 1], :] - NCT[ConnectT[:, 0], :], NCT[ConnectT[:, 2], :] - NCT[ConnectT[:, 0], :]) TNORMT = (TNORMT.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([TNORMT * TNORMT]), 2)))).T VNORMT = vrtxnormal(NCT, ConnectT, S_2_centr, TNORMT) print "determining kd-trees of target surface centroids and nodal coordinates" KDT_TC = KDTree(S_2_centr, m) KDT_TN = KDTree(NCT, m) print 'initialize absolute Gaussian weight for final displacement to preserve element quality' GW = np.ones((SamplingB + SamplingT, 1)) while k <= k_max: D1 = np.zeros((SamplingB, 3)) D2 = np.zeros((SamplingT, 3)) DS = np.zeros((N1, 3)) AllowableB = np.r_[AllowableBI] print print "MESH DEFORMATION ITERATION", k print " determining known displacement of landmarks" if NL > 0: knownD = LandmB_NC - W_km1[LandmB, ] print " determining centroids of deforming mesh" W_km1_centr = np.c_[ np.sum( np.c_[W_km1[ConnectB[:, 0], 0], W_km1[ConnectB[:, 1], 0], W_km1[ConnectB[:, 2], 0]], 1) / 3, np.sum( np.c_[W_km1[ConnectB[:, 0], 1], W_km1[ConnectB[:, 1], 1], W_km1[ConnectB[:, 2], 1]], 1) / 3, np.sum( np.c_[W_km1[ConnectB[:, 0], 2], W_km1[ConnectB[:, 1], 2], W_km1[ConnectB[:, 2], 2]], 1) / 3] print " determine triangle and vertex normals of deforming surface" TNORMB = np.cross(W_km1[ConnectB[:, 1], :] - W_km1[ConnectB[:, 0], :], W_km1[ConnectB[:, 2], :] - W_km1[ConnectB[:, 0], :]) TNORMB = (TNORMB.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([TNORMB * TNORMB]), 2)))).T VNORMB = vrtxnormal(W_km1, ConnectB, W_km1_centr, TNORMB) print " determining kd-tree of current deforming surface centroids and nodal coordinates" KDT_KC = KDTree(W_km1_centr, m) KDT_KN = KDTree(W_km1, m) #if find_repeats(np.r_[USENORMALS,k])[0].size>0: #print " ### Use triangle and vertex normals in setting up point correspondence" print " setting up D1(i,d)" tic = time.clock() results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(DsetupNorm)) for j in range(0, LIM): calc( np.array(range(0, SBPP)) + j * SBPP, W_km1, VNORMB, NCT, TNORMT, VNORMT, ConnectT, S_2_centr, AllowableT, LMB, D1) for j in range(0, LIM): D1[np.array(range(0, SBPP)) + j * SBPP, :] = results[j] D1[range(LIM * SBPP, SamplingB), :] = DsetupNorm( range(LIM * SBPP, SamplingB), W_km1, VNORMB, NCT, TNORMT, VNORMT, ConnectT, S_2_centr, AllowableT, LMB, D1) #D1=np.r_[D1,knownD] print " ", time.clock() - tic, " seconds" print " update allowable triangles on generic mesh:" remP = D1[:, 0] + D1[:, 1] + D1[:, 2] == 0 removeP = LMB[remP] print " unregistered points on generic mesh: ", removeP.size print " number of original generic triangles allowed: ", AllowableB.shape[ 0] for rp in removeP: rowsNo = np.where(AllowableB == rp)[0] rowsNo.sort for rr in rowsNo[::-1]: AllowableB = AllowableB[np.where( range(AllowableB.shape[0]) <> rr)[0], ] print " number of generic triangles allowed for current iteration: ", AllowableB.shape[ 0] if find_repeats(np.r_[USENORMALS, k])[0].size > 0: print " ### Use triangle and vertex normals in setting up point correspondence" print " setting up D2(j,c)" tic = time.clock() results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(DsetupNorm)) for j in range(0, LIM): calc( np.array(range(0, STPP)) + j * STPP, NCT, VNORMT, W_km1, TNORMB, VNORMB, ConnectB, W_km1_centr, AllowableB, LMT, D2) for j in range(0, LIM): D2[np.array(range(0, STPP)) + j * STPP, :] = results[j] D2[range(LIM * STPP, SamplingT), :] = DsetupNorm( range(LIM * STPP, SamplingT), NCT, VNORMT, W_km1, TNORMB, VNORMB, ConnectB, W_km1_centr, AllowableB, LMT, D2) print " ", time.clock() - tic, " seconds" else: print " Simple closest point search iteration " #print " setting up D1(i,d)" #tic = time.clock() #results = pprocess.Map(limit=LIM) #calc = results.manage(pprocess.MakeParallel(Dsetup)) #for j in range(0,LIM): #calc(np.array(range(0,SBPP))+j*SBPP,W_km1,NCT,ConnectT,S_2_centr,AllowableT,LMB,D1,KDT_TC,KDT_TN) #for j in range(0,LIM): #D1[np.array(range(0,SBPP))+j*SBPP,:] = results[j] #D1[range(LIM*SBPP,SamplingB),:]=Dsetup(range(LIM*SBPP,SamplingB),W_km1,NCT,ConnectT,S_2_centr,AllowableT,LMB,D1,KDT_TC,KDT_TN) ##D1=np.r_[D1,knownD] #print " ",time.clock()-tic," seconds" #remP = D1[:,0]+D1[:,1]+D1[:,2]==0 #removeP = LMB[remP] #print " unregistered points on generic mesh: ",removeP.size #print " number of original generic triangles allowed: ",AllowableB.shape[0] #for rp in removeP: #rowsNo = np.where(AllowableB==rp)[0] #rowsNo.sort #for rr in rowsNo[::-1]: #AllowableB = AllowableB[np.where(range(AllowableB.shape[0])<>rr)[0],] #print " number of generic triangles allowed for current iteration: ",AllowableB.shape[0] print " setting up D2(j,c)" tic = time.clock() results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(Dsetup)) for j in range(0, LIM): calc( np.array(range(0, STPP)) + j * STPP, NCT, W_km1, ConnectB, W_km1_centr, AllowableB, LMT, D2, KDT_KC, KDT_KN) for j in range(0, LIM): D2[np.array(range(0, STPP)) + j * STPP, :] = results[j] D2[range(LIM * STPP, SamplingT), :] = Dsetup( range(LIM * STPP, SamplingT), NCT, W_km1, ConnectB, W_km1_centr, AllowableB, LMT, D2, KDT_KC, KDT_KN) print " ", time.clock() - tic, " seconds" # Compute displacement update for each node using suggested Gaussian radial basis function: print " determining smoothed displacement field" tic = time.clock() NCp = np.r_[W_km1[LMB, :], NCT[LMT, :] + D2] DD = np.r_[D1, -D2] # Mask Nan and Inf values if any: DD[np.isnan(DD)] = 0 DD[np.isinf(DD)] = 0 #keepP = DD[:,0]+DD[:,1]+DD[:,2]<>0 #print keepP #NCp,DD = NCp[keepP,:],DD[keepP,:] #KDTp = KDTree(NCp,5) # Deform mesh using Gaussian smoothing as suggested in paper by R.Bryan et al. sigma_k2 = np.power(np.power(f, -k) * sigm0, 2) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(GaussianSmooth)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, W_km1, NCp, DD, sigma_k2, gamm) for j in range(0, LIM): DS[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] DS[range(LIM * NPP1, N1), :] = GaussianSmooth( np.array(range(LIM * NPP1, N1)), W_km1, NCp, DD, sigma_k2, gamm) print " ", time.clock() - tic, " seconds" # Mask Nan and Inf if any: DS[np.isnan(DS)] = 0 DS[np.isinf(DS)] = 0 #print 'Check if current iteration reduces element quality to below allowable and stiffen mesh accordingly' print print print 'Convergence History' print CONV print print # Determine Jacobian of all elements and if unsattisfied apply stiffening (Decrease GW <1) untill this doesn't happen # determine wheter convergence is acheived #TotalMorph = np.sum(np.sqrt(np.sum(DS*DS,1)),0)/NCB.shape[0] TotalMorph = np.sum(np.sqrt(np.sum(DD * DD, 1))) / (DD.size / 3) CONV = CONV + [TotalMorph] FMorph = (k == 1) * TotalMorph + FMorph print " average nodal displacement for current deformation iteration:" print TotalMorph if (TotalMorph < Tol): print print "CONVERGED SOLUTION OBTAINED" #CONV = CONV + [TotalMorph] k = k_max * 10 + 1 W_km1 = W_km1 + DS elif (k < 10) | (TotalMorph < 10 * FMorph): print "problem not yet converged at iteration", k #CONV = CONV + [TotalMorph] k = k + 1 # Deform mesh: print " deforming mesh (update of W_{k-1})" W_km1 = W_km1 + DS #np.ma.dump(W_km1,'Femur2NC_'+str(k)) else: print "PROBLEM DIVERGING" k = k_max * 10 - 1 #np.ma.dump(W_km1,'TempElasNodes_Iter'+str(k-1)+'_Time'+time.ctime()) if (k > 2) & (np.mod(k - 1, 5) == 0): print #np.ma.dump(W_km1,'TempElasNodes_Iter'+str(k-1)+'_Time'+time.ctime()) #W_km1 = RBFmorph(W_km1,W_km1[LandmB,],LandmB_NC-W_km1[LandmB,]) tic = time.clock() W_km1 = MeshSmooth(W_km1, neighbList, 10) np.ma.dump( W_km1, 'SkullUnique2_gamm' + str(gamm) + '_sigN' + str(sigm0) + '_iter' + str(k - 1)) print " Smoothing done in ", time.clock() - tic, " seconds" #print "COARSE SURFACE REGISTRATION" #print " using landmark displacements to deform using RBF" #W_km1 = RBFmorph(W_km1,W_km1[LandmB,],LandmB_NC-W_km1[LandmB,]) print if k == k_max + 1: print print "SOLUTION TERMINATED: maximum iterations,(", k_max, ") reached" print print "TOTAL TIME FOR ELASTIC SURFACE REGISTRATION : ", time.clock( ) - t_start, "seconds" CONV = np.array(CONV) return W_km1, CONV
def voxel_selection(vol_surf_mapping, radius, source_surf=None, source_surf_nodes=None, distance_metric='dijkstra', eta_step=10, nproc=None, outside_node_margin=None, results_backend=None, tmp_prefix='tmpvoxsel'): """ Voxel selection for multiple center nodes on the surface Parameters ---------- vol_surf_mapping: volsurf.VolSurfMapping Contains gray and white matter surface, and volume geometry radius: int or float Size of searchlight. If an integer, then it indicates the number of voxels. If a float, then it indicates the radius of the disc source_surf: surf.Surface or None Surface used to compute distance between nodes. If omitted, it is the average of the gray and white surfaces. source_surf_nodes: list of int or numpy array or None Indices of nodes in source_surf that serve as searchlight center. By default every node serves as a searchlight center. distance_metric: str Distance metric between nodes. 'euclidean' or 'dijksta' (default) eta_step: int Report progress every eta_step (default: 10). nproc: int or None Number of parallel threads. None means as many threads as the system supports. The pprocess is required for parallel threads; if it cannot be used, then a single thread is used. outside_node_margin: float or True or None (default) By default nodes outside the volume are skipped; using this parameter allows for a marign. If this value is a float (possibly np.inf), then all nodes within outside_node_margin Dijkstra distance from any node within the volume are still assigned associated voxels. If outside_node_margin is True, then a node is always assigned voxels regardless of its position in the volume. results_backend : 'native' or 'hdf5' or None (default). Specifies the way results are provided back from a processing block in case of nproc > 1. 'native' is pickling/unpickling of results by pprocess, while 'hdf5' would use h5save/h5load functionality. 'hdf5' might be more time and memory efficient in some cases. If None, then 'hdf5' if used if available, else 'native'. tmp_prefix : str, optional If specified -- serves as a prefix for temporary files storage if results_backend == 'hdf5'. Thus can specify the directory to use (trailing file path separator is not added automagically). Returns ------- sel: volume_mask_dict.VolumeMaskDictionary Voxel selection results, that associates, which each node, the indices of the surrounding voxels. """ # construct the intermediate surface, which is used # to measure distances intermediate_surf = (vol_surf_mapping.pial_surface * .5) + \ (vol_surf_mapping.white_surface * .5) if source_surf is None: source_surf = intermediate_surf else: source_surf = surf.from_any(source_surf) if _debug(): debug( 'SVS', "Generated high-res intermediate surface: " "%d nodes, %d faces" % (intermediate_surf.nvertices, intermediate_surf.nfaces)) debug( 'SVS', "Mapping source to high-res surface:" " %d nodes, %d faces" % (source_surf.nvertices, source_surf.nfaces)) if distance_metric[0].lower() == 'e' and outside_node_margin: # euclidean distance: identity mapping # this is *slow* n = source_surf.nvertices xyz = source_surf.vertices src2intermediate = dict((i, tuple(xyz[i])) for i in xrange(n)) else: # find a mapping from nodes in source_surf to those in # intermediate surface src2intermediate = source_surf.map_to_high_resolution_surf(\ intermediate_surf) # if no sources are given, then visit all ndoes if source_surf_nodes is None: source_surf_nodes = np.arange(source_surf.nvertices) n = len(source_surf_nodes) if _debug(): debug('SVS', "Performing surface-based voxel selection" " for %d centers" % n) # visit in random order, for for better ETA estimate visitorder = list(np.random.permutation(len(source_surf_nodes))) # construct mapping from nodes to enclosing voxels n2v = vol_surf_mapping.get_node2voxels_mapping() if __debug__: debug('SVS', "Generated mapping from nodes" " to intersecting voxels") # build voxel selector voxel_selector = VoxelSelector(radius, intermediate_surf, n2v, distance_metric, outside_node_margin=outside_node_margin) if _debug(): debug('SVS', "Instantiated voxel selector (radius %r)" % radius) # structure to keep output data. Initialize with None, then # make a sparse_attributes instance when we know what the attributes are node2volume_attributes = None attribute_mapper = voxel_selector.disc_voxel_indices_and_attributes srcs_order = [source_surf_nodes[node] for node in visitorder] src_trg_nodes = [(src, src2intermediate[src]) for src in srcs_order] if nproc is not None and nproc > 1 and not externals.exists('pprocess'): raise RuntimeError("The 'pprocess' module is required for " "multiprocess searchlights. Please either " "install python-pprocess, or reduce `nproc` " "to 1 (got nproc=%i) or set to default None" % nproc) if nproc is None: if externals.exists('pprocess'): try: import pprocess nproc = pprocess.get_number_of_cores() or 1 if _debug(): debug("SVS", 'Using pprocess with %d cores' % nproc) except: if _debug(): debug("SVS", 'pprocess not available') if nproc is None: # importing pprocess failed - so use a single core nproc = 1 debug("SVS", 'Using %d cores - pprocess not available' % nproc) # get the the voxel selection parameters parameter_dict = vol_surf_mapping.get_parameter_dict() parameter_dict.update(dict(radius=radius, outside_node_margin=outside_node_margin, distance_metric=distance_metric), source_nvertices=source_surf.nvertices) init_output = lambda: volume_mask_dict.VolumeMaskDictionary( vol_surf_mapping.volgeom, intermediate_surf, meta=parameter_dict) if nproc > 1: if results_backend == 'hdf5': externals.exists('h5py', raise_=True) elif results_backend is None: if externals.exists( 'h5py') and externals.versions['hdf5'] >= '1.8.7': results_backend = 'hdf5' else: results_backend = 'native' if _debug(): debug('SVS', "Using '%s' backend" % (results_backend, )) if not results_backend in ('native', 'hdf5'): raise ValueError('Illegal results backend %r' % results_backend) import pprocess n_srcs = len(src_trg_nodes) blocks = np.array_split(np.arange(n_srcs), nproc) results = pprocess.Map(limit=nproc) reducer = results.manage(pprocess.MakeParallel(_reduce_mapper)) if __debug__: debug('SVS', "Starting %d child processes", (len(blocks), )) for i, block in enumerate(blocks): empty_dict = init_output() src_trg = [] for idx in block: src_trg.append(src_trg_nodes[idx]) if _debug(): debug('SVS', " starting block %d/%d: %d centers" % (i + 1, nproc, len(src_trg)), cr=True) reducer(empty_dict, attribute_mapper, src_trg, eta_step=eta_step, proc_id='%d' % (i + 1, ), results_backend=results_backend, tmp_prefix=tmp_prefix) if _debug(): debug('SVS', '') debug('SVS', 'Started all %d child processes' % (len(blocks))) tstart = time.time() node2volume_attributes = None for i, result in enumerate(results): if result is None: continue if results_backend == 'hdf5': result_fn = result result = h5load(result_fn) os.remove(result_fn) if node2volume_attributes is None: # first time we have actual results. # Use as a starting point node2volume_attributes = result if _debug(): debug('SVS', '') debug( 'SVS', "Merging results from %d child " "processes using '%s' backend" % (len(blocks), results_backend)) else: # merge new with current data node2volume_attributes.merge(result) if _debug(): debug('SVS', " merged result block %d/%d" % (i + 1, nproc), cr=True) if _debug(): telapsed = time.time() - tstart debug('SVS', "") debug( 'SVS', 'Merged results from %d child processed - ' 'took %s' % (len(blocks), seconds2prettystring(telapsed))) else: empty_dict = init_output() node2volume_attributes = _reduce_mapper(empty_dict, attribute_mapper, src_trg_nodes, eta_step=eta_step) debug('SVS', "") if _debug(): if node2volume_attributes is None: msgs = [ "Voxel selection completed: none of %d nodes have " "voxels associated" % len(visitorder) ] else: nvox_selected = np.sum(node2volume_attributes.get_mask() != 0) vg = vol_surf_mapping.volgeom msgs = [ "Voxel selection completed: %d / %d nodes have " "voxels associated" % (len(node2volume_attributes.keys()), len(visitorder)), "Selected %d / %d voxels (%.0f%%) in the mask at least once" % (nvox_selected, vg.nvoxels_mask, 100. * nvox_selected / vg.nvoxels_mask) ] for msg in msgs: debug("SVS", msg) if node2volume_attributes is None: warning('No voxels associated with any of %d nodes' % len(visitorder)) return node2volume_attributes
def lineRST(NCB, RlinesB, VlinesB, NCT, RlinesT, VlinesT, UseFeat=0, UseScale=0, Use1Scale=1): # Use lines of curvature on two models to determine rigid body transformation for best fit # Takes as input the nodal coordinates of the two surface meshes as well as the ridge and valley lines of the two. # Target mesh is then rotated, scaled and translated to best fit the corresponding lines of curvature on the Base mesh #global RsegmB,VsegmB,RsegmT,VsegmT #global RnodesB,VnodesB,RnodesT,VnodesT if UseFeat == 1: [RsegmB, VsegmB, RsegmT, VsegmT] = [ np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)) ] [RnodesB, VnodesB, RnodesT, VnodesT] = [ np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)) ] # Transform Lines into linesegments with i'th segment allocated as [NC_point1,NC_point2,Line_nr] # and list of nodes allocated as [Nd_nr, Line_nr] for i in range(1, RlinesB[0] + 1): Lsize = RlinesB[i].size RnodesB = np.array( np.r_[RnodesB, np.c_[RlinesB[i], np.ones((Lsize, 1)) * i]], int) RsegmB = np.array( np.r_[RsegmB, np.c_[RlinesB[i][0:Lsize - 1], RlinesB[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, VlinesB[0] + 1): Lsize = VlinesB[i].size VnodesB = np.array( np.r_[VnodesB, np.c_[VlinesB[i], np.ones((Lsize, 1)) * i]], int) VsegmB = np.array( np.r_[VsegmB, np.c_[VlinesB[i][0:Lsize - 1], VlinesB[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, RlinesT[0] + 1): Lsize = RlinesT[i].size RnodesT = np.array( np.r_[RnodesT, np.c_[RlinesT[i], np.ones((Lsize, 1)) * i]], int) RsegmT = np.array( np.r_[RsegmT, np.c_[RlinesT[i][0:Lsize - 1], RlinesT[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, VlinesT[0] + 1): Lsize = VlinesT[i].size VnodesT = np.array( np.r_[VnodesT, np.c_[VlinesT[i], np.ones((Lsize, 1)) * i]], int) VsegmT = np.array( np.r_[VsegmT, np.c_[VlinesT[i][0:Lsize - 1], VlinesT[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) # find average nodal coordinate of base linesegments RsegBNC, VsegBNC = (NCB[RsegmB[:, 0], ] + NCB[RsegmB[:, 1], ]) / 2, ( NCB[VsegmB[:, 0], ] + NCB[VsegmB[:, 1], ]) / 2 # set up k-d tree of nodes and segments in Base model lines kdt_RBS, kdt_VBS = KDTree(RsegBNC, 5), KDTree(VsegBNC, 5) else: N1, N2 = NCB.shape[0], NCT.shape[0] NPP1 = N1 / LIM NPP2 = N2 / LIM kdt_Base = KDTree(NCB, 20) diff = 2 k = 0 Conv = np.zeros((100, )) while (diff > 0.000001) & (k < 100): k = k + 1 print ' ITERATION ', k if UseFeat == 1: RsegTNC, VsegTNC = (NCT[RsegmT[:, 0], ] + NCT[RsegmT[:, 1], ] ) / 2, (NCT[VsegmT[:, 0], ] + NCT[VsegmT[:, 1], ]) / 2 kdt_RTS, kdt_VTS = KDTree(RsegTNC, 5), KDTree(VsegTNC, 5) R12R = LineICP(RnodesB, NCB, RnodesT, NCT, kdt_RTS, RsegmT) #find registered mesh 1 to 2 Ridges R12V = LineICP(VnodesB, NCB, VnodesT, NCT, kdt_VTS, VsegmT) #find registered mesh 1 to 2 Valleys R21R = LineICP(RnodesT, NCT, RnodesB, NCB, kdt_RBS, RsegmB) #find registered mesh 2 to 1 Ridges R21V = LineICP(VnodesT, NCT, VnodesB, NCB, kdt_VBS, VsegmB) #find registered mesh 2 to 1 Valleys # Determine translation required for best fit. Known that minnimum is at this position [D. Du et al] B12R, T12R = NCB[np.array(R12R[:, 0], int), ], R12R[:, 1:4] B12V, T12V = NCB[np.array(R12V[:, 0], int), ], R12V[:, 1:4] B21R, T21R = R21R[:, 1:4], NCT[np.array(R21R[:, 0], int), ] B21V, T21V = R21V[:, 1:4], NCT[np.array(R21V[:, 0], int), ] Base_i = np.r_[B12R, B12V, B21R, B21V] Target_i = np.r_[T12R, T12V, T21R, T21V] else: kdt_Targ = KDTree(NCT, 20) print 'k-d trees and closest point search' B2T, T2B = np.zeros((NCB.shape[0], 1)), np.zeros((NCT.shape[0], 1)) print ' base' results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(FullICP)) for j in range(0, LIM): calc(np.array(range(0, NPP1)) + j * NPP1, NCB, kdt_Targ, B2T) for j in range(0, LIM): B2T[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] B2T[range(LIM * NPP1, N1), :] = FullICP(range(LIM * NPP1, N1), NCB, kdt_Targ, B2T) #for i in range(0,NCB.shape[0]): #B2T = B2T + [kdt_Targ.query(NCB[i,])[1]] print ' target' results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(FullICP)) for j in range(0, LIM): calc(np.array(range(0, NPP2)) + j * NPP2, NCT, kdt_Base, T2B) for j in range(0, LIM): T2B[np.array(range(0, NPP2)) + j * NPP2, :] = results[j] T2B[range(LIM * NPP2, N2), :] = FullICP(range(LIM * NPP2, N2), NCT, kdt_Base, T2B) #for i in range(0,NCT.shape[0]): #T2B = T2B + [kdt_Base.query(NCT[i,])[1]] B2T, T2B = np.array(B2T, int).reshape( (NCB.shape[0], )), np.array(T2B, int).reshape((NCT.shape[0], )) Base_i = np.r_[NCB, NCB[T2B, ]] Target_i = np.r_[NCT[B2T, ], NCT] print 'translate' Distance = Base_i - Target_i Translate = np.sum(Distance, 0) / Distance.shape[0] # Apply translation to Target nodal coordinates and Base to Target registered pairs NCTT = np.c_[NCT[:, 0] + Translate[0], NCT[:, 1] + Translate[1], NCT[:, 2] + Translate[2]] Target_i = np.c_[Target_i[:, 0] + Translate[0], Target_i[:, 1] + Translate[1], Target_i[:, 2] + Translate[2]] print ' translation = ', Translate # Determine Current Quadratic approximation: E1, E2, E3 = np.matrix([[0, -1, 0], [1, 0, 0], [0, 0, 0]]), np.matrix([[0, 0, 0], [0, 0, -1], [0, 1, 0]]), np.matrix([[0, 0, 1], [0, 0, 0], [-1, 0, 0]]) D1, D2, D3 = np.matrix([[1, 0, 0], [0, 0, 0], [0, 0, 0]]), np.matrix([[0, 0, 0], [0, 1, 0], [0, 0, 0]]), np.matrix([[0, 0, 0], [0, 0, 0], [0, 0, 1]]) IMat = np.matrix(np.eye(3)) #Ukm1,Skm1,Rkm1 = np.linalg.svd(AO) Ukm1, Skm1, Rkm1 = IMat, IMat, IMat #*1.3 #k2,diff2=0,2 #while (diff2>0.1)&(k2<100): #k2=k2+1 #Bi1,Bi2,Bi3 = np.array(Ukm1*E1*Skm1*Rkm1*Target_i.T).T,np.array(Ukm1*E2*Skm1*Rkm1*Target_i.T).T,np.array(Ukm1*E3*Skm1*Rkm1*Target_i.T).T #Bi4,Bi5,Bi6 = np.array(Ukm1*Skm1*D1*Rkm1*Target_i.T).T,np.array(Ukm1*Skm1*D2*Rkm1*Target_i.T).T,np.array(Ukm1*Skm1*D3*Rkm1*Target_i.T).T #Bi7,Bi8,Bi9 = np.array(Ukm1*Skm1*Rkm1*E1*Target_i.T).T,np.array(Ukm1*Skm1*Rkm1*E2*Target_i.T).T,np.array(Ukm1*Skm1*Rkm1*E3*Target_i.T).T #Distance = np.array(Ukm1*Skm1*Rkm1*Target_i.T-Base_i.T).T ## Set up Hessian matrix: #Hes = np.matrix([[np.sum(Bi1*Bi1),np.sum(Bi1*Bi2),np.sum(Bi1*Bi3),np.sum(Bi1*Bi4),np.sum(Bi1*Bi5),np.sum(Bi1*Bi6),np.sum(Bi1*Bi7),np.sum(Bi1*Bi8),np.sum(Bi1*Bi9)], #[np.sum(Bi2*Bi1),np.sum(Bi2*Bi2),np.sum(Bi2*Bi3),np.sum(Bi2*Bi4),np.sum(Bi2*Bi5),np.sum(Bi2*Bi6),np.sum(Bi2*Bi7),np.sum(Bi2*Bi8),np.sum(Bi2*Bi9)], #[np.sum(Bi3*Bi1),np.sum(Bi3*Bi2),np.sum(Bi3*Bi3),np.sum(Bi3*Bi4),np.sum(Bi3*Bi5),np.sum(Bi3*Bi6),np.sum(Bi3*Bi7),np.sum(Bi3*Bi8),np.sum(Bi3*Bi9)], #[np.sum(Bi4*Bi1),np.sum(Bi4*Bi2),np.sum(Bi4*Bi3),np.sum(Bi4*Bi4),np.sum(Bi4*Bi5),np.sum(Bi4*Bi6),np.sum(Bi4*Bi7),np.sum(Bi4*Bi8),np.sum(Bi4*Bi9)], #[np.sum(Bi5*Bi1),np.sum(Bi5*Bi2),np.sum(Bi5*Bi3),np.sum(Bi5*Bi4),np.sum(Bi5*Bi5),np.sum(Bi5*Bi6),np.sum(Bi5*Bi7),np.sum(Bi5*Bi8),np.sum(Bi5*Bi9)], #[np.sum(Bi6*Bi1),np.sum(Bi6*Bi2),np.sum(Bi6*Bi3),np.sum(Bi6*Bi4),np.sum(Bi6*Bi5),np.sum(Bi6*Bi6),np.sum(Bi6*Bi7),np.sum(Bi6*Bi8),np.sum(Bi6*Bi9)], #[np.sum(Bi7*Bi1),np.sum(Bi7*Bi2),np.sum(Bi7*Bi3),np.sum(Bi7*Bi4),np.sum(Bi7*Bi5),np.sum(Bi7*Bi6),np.sum(Bi7*Bi7),np.sum(Bi7*Bi8),np.sum(Bi7*Bi9)], #[np.sum(Bi8*Bi1),np.sum(Bi8*Bi2),np.sum(Bi8*Bi3),np.sum(Bi8*Bi4),np.sum(Bi8*Bi5),np.sum(Bi8*Bi6),np.sum(Bi8*Bi7),np.sum(Bi8*Bi8),np.sum(Bi8*Bi9)], #[np.sum(Bi9*Bi1),np.sum(Bi9*Bi2),np.sum(Bi9*Bi3),np.sum(Bi9*Bi4),np.sum(Bi9*Bi5),np.sum(Bi9*Bi6),np.sum(Bi9*Bi7),np.sum(Bi9*Bi8),np.sum(Bi9*Bi9)]]) ## set up fj #Fj = np.matrix([[np.sum(Bi1*Distance),np.sum(Bi2*Distance),np.sum(Bi3*Distance),np.sum(Bi4*Distance),np.sum(Bi5*Distance), #np.sum(Bi6*Distance),np.sum(Bi7*Distance),np.sum(Bi8*Distance),np.sum(Bi9*Distance)]]).T print 'find rotation and scale' Xf = fmin_powell(costF, np.array([0, 0, 0, 1, 1, 1, 0, 0, 0]), args=(Base_i, Target_i, Ukm1, Skm1, Rkm1, UseScale, Use1Scale)) Xf = np.array(Xf).reshape((9, )) print 'u1..3,s1..3,r1..3 = ', Xf FvP = np.array(Ukm1 * Skm1 * Rkm1 * Target_i.T - Base_i.T).T FvP = np.sum(FvP * FvP) Conv[k] = FvP Ukm1 = Ukm1 + Ukm1 * np.matrix(E1 * Xf[0] + E2 * Xf[1] + E3 * Xf[2]) if UseScale == 1: Skm1 = Skm1 + Skm1 * np.matrix(D1 * Xf[3] + D2 * Xf[4] + D3 * Xf[5]) if Use1Scale == 1: Skm1 = Skm1 + Skm1 * np.matrix(D1 * Xf[3] + D2 * Xf[3] + D3 * Xf[3]) print print Skm1 print Rkm1 = Rkm1 + Rkm1 * np.matrix(E1 * Xf[6] + E2 * Xf[7] + E3 * Xf[8]) NCTT = np.array(Ukm1 * Skm1 * Rkm1 * NCTT.T).T diff = np.sum((NCT - NCTT) * (NCT - NCTT)) / NCT.shape[0] print ' Average difference between current and previous nodal coordinates: ', diff np.ma.dump(NCTT, 'Femur1NC_' + str(k)) NCT = NCTT return NCT, Conv
def __call__(self, datasets): """Estimate mappers for each dataset using searchlight-based hyperalignment. Parameters ---------- datasets : list or tuple of datasets Returns ------- A list of trained StaticProjectionMappers of the same length as datasets """ # Perform some checks first before modifying internal state params = self.params ndatasets = len(datasets) if len(datasets) <= 1: raise ValueError("SearchlightHyperalignment needs > 1 dataset to " "operate on. Got: %d" % self.ndatasets) if params.ref_ds in params.exclude_from_model: raise ValueError("Requested reference dataset %i is also " "in the exclude list." % params.ref_ds) if params.ref_ds >= ndatasets: raise ValueError("Requested reference dataset %i is out of " "bounds. We have only %i datasets provided" % (params.ref_ds, self.ndatasets)) # The rest of the checks are just warnings self.ndatasets = ndatasets _shpaldebug("SearchlightHyperalignment %s for %i datasets" % (self, self.ndatasets)) selected = [ _ for _ in range(ndatasets) if _ not in params.exclude_from_model ] ref_ds_train = selected.index(params.ref_ds) params.hyperalignment.params.ref_ds = ref_ds_train warning('Using %dth dataset as the reference dataset (%dth after ' 'excluding datasets)' % (params.ref_ds, ref_ds_train)) if len(params.exclude_from_model) > 0: warning("These datasets will not participate in building common " "model: %s" % params.exclude_from_model) if __debug__: # verify that datasets were zscored prior the alignment since it is # assumed/required preprocessing step for ids, ds in enumerate(datasets): for f, fname, tval in ((np.mean, 'means', 0), (np.std, 'stds', 1)): vals = f(ds, axis=0) vals_comp = np.abs(vals - tval) > 1e-5 if np.any(vals_comp): warning( '%d %s are too different (max diff=%g) from %d in ' 'dataset %d to come from a zscored dataset. ' 'Please zscore datasets first for correct operation ' '(unless if was intentional)' % (np.sum(vals_comp), fname, np.max( np.abs(vals)), tval, ids)) # Setting up SearchlightHyperalignment # we need to know which original features where comprising the # individual SL ROIs _shpaldebug('Initializing FeatureSelectionHyperalignment.') hmeasure = FeatureSelectionHyperalignment( ref_ds=params.ref_ds, featsel=params.featsel, hyperalignment=params.hyperalignment, full_matrix=params.combine_neighbormappers, use_same_features=params.use_same_features, exclude_from_model=params.exclude_from_model, dtype=params.dtype) # Performing SL processing manually _shpaldebug("Setting up for searchlights") if params.nproc is None and externals.exists('pprocess'): import pprocess try: params.nproc = pprocess.get_number_of_cores() or 1 except AttributeError: warning("pprocess version %s has no API to figure out maximal " "number of cores. Using 1" % externals.versions['pprocess']) params.nproc = 1 # XXX I think this class should already accept a single dataset only. # It should have a ``space`` setting that names a sample attribute that # can be used to identify individual/original datasets. # Taking a single dataset as argument would be cleaner, because the # algorithm relies on the assumption that there is a coarse feature # alignment, i.e. the SL ROIs cover roughly the same area queryengines = self._get_trained_queryengines(datasets, params.queryengine, params.radius, params.ref_ds) # For surface nodes to voxels queryengines, roi_seed hardly makes sense qe = queryengines[(0 if len(queryengines) == 1 else params.ref_ds)] if isinstance(qe, SurfaceVerticesQueryEngine): self.force_roi_seed = False if not self.params.combine_neighbormappers: raise NotImplementedError( "Mapping from voxels to surface nodes is not " "implmented yet. Try setting combine_neighbormappers to True." ) self.nfeatures = datasets[params.ref_ds].nfeatures _shpaldebug("Performing Hyperalignment in searchlights") # Setting up centers for running SL Hyperalignment if params.sparse_radius is None: roi_ids = self._get_verified_ids(queryengines) \ if params.mask_node_ids is None \ else params.mask_node_ids else: if params.queryengine is not None: raise NotImplementedError( "using sparse_radius whenever custom queryengine is " "provided is not yet supported.") _shpaldebug("Setting up sparse neighborhood") from mvpa2.misc.neighborhood import scatter_neighborhoods if params.mask_node_ids is None: scoords, sidx = scatter_neighborhoods( Sphere(params.sparse_radius), datasets[params.ref_ds].fa.voxel_indices, deterministic=True) roi_ids = sidx else: scoords, sidx = scatter_neighborhoods( Sphere(params.sparse_radius), datasets[params.ref_ds].fa.voxel_indices[ params.mask_node_ids], deterministic=True) roi_ids = [params.mask_node_ids[sid] for sid in sidx] # Initialize projections _shpaldebug('Initializing projection matrices') self.projections = [ csc_matrix((self.nfeatures, self.nfeatures), dtype=params.dtype) for isub in range(self.ndatasets) ] # compute if params.nproc is not None and params.nproc > 1: # split all target ROIs centers into `nproc` equally sized blocks nproc_needed = min(len(roi_ids), params.nproc) params.nblocks = nproc_needed \ if params.nblocks is None else params.nblocks params.nblocks = min(len(roi_ids), params.nblocks) node_blocks = np.array_split(roi_ids, params.nblocks) # the next block sets up the infrastructure for parallel computing # this can easily be changed into a ParallelPython loop, if we # decide to have a PP job server in PyMVPA import pprocess p_results = pprocess.Map(limit=nproc_needed) if __debug__: debug( 'SLC', "Starting off %s child processes for nblocks=%i" % (nproc_needed, params.nblocks)) compute = p_results.manage(pprocess.MakeParallel(self._proc_block)) seed = mvpa2.get_random_seed() for iblock, block in enumerate(node_blocks): # should we maybe deepcopy the measure to have a unique and # independent one per process? compute(block, datasets, copy.copy(hmeasure), queryengines, seed=seed, iblock=iblock) else: # otherwise collect the results in an 1-item list _shpaldebug('Using 1 process to compute mappers.') if params.nblocks is None: params.nblocks = 1 params.nblocks = min(len(roi_ids), params.nblocks) node_blocks = np.array_split(roi_ids, params.nblocks) p_results = [ self._proc_block(block, datasets, hmeasure, queryengines) for block in node_blocks ] results_ds = self.__handle_all_results(p_results) # Dummy iterator for, you know, iteration list(results_ds) _shpaldebug( 'Wrapping projection matrices into StaticProjectionMappers') self.projections = [ StaticProjectionMapper(proj=proj, recon=proj.T) if params.compute_recon else StaticProjectionMapper(proj=proj) for proj in self.projections ] return self.projections
#np.ma.dump(NCS,fname[0:6]+'NC_Ainit') #NCS = ptet.LaplacMesh(inner,NCS,neighbListTet,100,1) #np.ma.dump(NCS,fname[0:6]+'NC_AinitLapl') NCSprev = np.r_[NCS] DispOuter = (NCouter - NCTnc[outer, ]) / Steps for inc in range(Steps): #NCS[outer,] = NCS[outer,]+DispOuter #update boundary displacement # Deform mesh using Gaussian smoothing as suggested in paper by R.Bryan et al. NCp = NCS[outer, ] DD = DispOuter DS = np.zeros(NCS.shape) print 'Do Gaussian Smooth on internal nodes' sigma_k2 = np.power(np.power(1.0715, -(inc + 1)) * 10, 2) results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(ptet.GaussianSmooth)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, np.r_[NCS], NCp, DD, sigma_k2, 2) for j in range(0, LIM): DS[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] DS[range(LIM * NPP1, N1), :] = ptet.GaussianSmooth( np.array(range(LIM * NPP1, N1)), np.r_[NCS], NCp, DD, sigma_k2, 2) NCS[outer, ] = NCS[outer, ] + DispOuter NCS[inner, ] = NCS[inner, ] + DS[inner, ] EQ, delt, Sn2, Sig = qu.elemQual_mu(np.array(range(TetT.shape[0])), NCS, TetT) print ' Average Element Quality: ', np.average(EQ) print ' Degenerate (q<0.15): ', np.where(EQ < 0.15)[0].size
def LineReg(NCB, TriB, RlinesB, VlinesB, NCT, TriT, RlinesT, VlinesT, percentReg, DistMax): N1 = NCB.shape[0] NPP1 = N1 / LIM [RsegmB, VsegmB, RsegmT, VsegmT] = [ np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)), np.array([[], [], []]).reshape((0, 3)) ] [RnodesB, VnodesB, RnodesT, VnodesT] = [ np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)), np.array([[], []]).reshape((0, 2)) ] # Transform Lines into linesegments with i'th segment allocated as [NC_point1,NC_point2,Line_nr] # and list of nodes allocated as [Nd_nr, Line_nr] for i in range(1, RlinesB[0] + 1): Lsize = RlinesB[i].size RnodesB = np.array( np.r_[RnodesB, np.c_[RlinesB[i], np.ones((Lsize, 1)) * i]], int) RsegmB = np.array( np.r_[RsegmB, np.c_[RlinesB[i][0:Lsize - 1], RlinesB[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, VlinesB[0] + 1): Lsize = VlinesB[i].size VnodesB = np.array( np.r_[VnodesB, np.c_[VlinesB[i], np.ones((Lsize, 1)) * i]], int) VsegmB = np.array( np.r_[VsegmB, np.c_[VlinesB[i][0:Lsize - 1], VlinesB[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, RlinesT[0] + 1): Lsize = RlinesT[i].size RnodesT = np.array( np.r_[RnodesT, np.c_[RlinesT[i], np.ones((Lsize, 1)) * i]], int) RsegmT = np.array( np.r_[RsegmT, np.c_[RlinesT[i][0:Lsize - 1], RlinesT[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) for i in range(1, VlinesT[0] + 1): Lsize = VlinesT[i].size VnodesT = np.array( np.r_[VnodesT, np.c_[VlinesT[i], np.ones((Lsize, 1)) * i]], int) VsegmT = np.array( np.r_[VsegmT, np.c_[VlinesT[i][0:Lsize - 1], VlinesT[i][1:Lsize], np.ones((Lsize - 1, 1)) * i]], int) # find average nodal coordinate of base linesegments RsegTNC, VsegTNC = (NCT[RsegmT[:, 0], ] + NCT[RsegmT[:, 1], ]) / 2, ( NCT[VsegmT[:, 0], ] + NCT[VsegmT[:, 1], ]) / 2 # set up k-d tree of nodes and segments in Base model lines print 'TARGET: Determine triangle centroids, triangle normals and weighted vertex normals' TBCT = np.c_[ np. sum(np.c_[NCT[TriT[:, 0], 0], NCT[TriT[:, 1], 0], NCT[TriT[:, 2], 0]], 1) / 3, np. sum(np.c_[NCT[TriT[:, 0], 1], NCT[TriT[:, 1], 1], NCT[TriT[:, 2], 1]], 1) / 3, np. sum(np.c_[NCT[TriT[:, 0], 2], NCT[TriT[:, 1], 2], NCT[TriT[:, 2], 2]], 1) / 3] TNORMT = np.cross(NCT[TriT[:, 1], :] - NCT[TriT[:, 0], :], NCT[TriT[:, 2], :] - NCT[TriT[:, 0], :]) TNORMT = (TNORMT.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([TNORMT * TNORMT]), 2)))).T VNORMT = vrtxnormal(NCT, TriT, TBCT, TNORMT) print 'TARGET: Determine segment normal directions' SegNormRT, SegNormVT = (VNORMT[RsegmT[:, 0], ] + VNORMT[RsegmT[:, 1], ]), (VNORMT[VsegmT[:, 0], ] + VNORMT[VsegmT[:, 1], ]) SegNormRT = (SegNormRT.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([SegNormRT * SegNormRT]), 2)))).T SegNormVT = (SegNormVT.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([SegNormVT * SegNormVT]), 2)))).T # find average nodal coordinate of base linesegments RsegBNC, VsegBNC = (NCB[RsegmB[:, 0], ] + NCB[RsegmB[:, 1], ]) / 2, ( NCB[VsegmB[:, 0], ] + NCB[VsegmB[:, 1], ]) / 2 # set up k-d tree of nodes and segments in Base model lines print 'BASE: Determine triangle centroids, triangle normals and weighted vertex normals' TBCB = np.c_[ np. sum(np.c_[NCB[TriB[:, 0], 0], NCB[TriB[:, 1], 0], NCB[TriB[:, 2], 0]], 1) / 3, np. sum(np.c_[NCB[TriB[:, 0], 1], NCB[TriB[:, 1], 1], NCB[TriB[:, 2], 1]], 1) / 3, np. sum(np.c_[NCB[TriB[:, 0], 2], NCB[TriB[:, 1], 2], NCB[TriB[:, 2], 2]], 1) / 3] TNORMB = np.cross(NCB[TriB[:, 1], :] - NCB[TriB[:, 0], :], NCB[TriB[:, 2], :] - NCB[TriB[:, 0], :]) TNORMB = (TNORMB.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([TNORMB * TNORMB]), 2)))).T VNORMB = vrtxnormal(NCB, TriB, TBCB, TNORMB) print 'BASE: Determine segment normal directions' SegNormRB, SegNormVB = (VNORMB[RsegmB[:, 0], ] + VNORMB[RsegmB[:, 1], ]), (VNORMB[VsegmB[:, 0], ] + VNORMB[VsegmB[:, 1], ]) SegNormRB = (SegNormRB.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([SegNormRB * SegNormRB]), 2)))).T SegNormVB = (SegNormVB.T / (np.ones( (3, 1)) * np.sqrt(np.sum(np.array([SegNormVB * SegNormVB]), 2)))).T deform, k = 2, 0 while (deform > 0.0001) & (k < 100): Supp = 50. / (1 + k / 5) if Supp < 10: Supp = 10 k = k + 1 print ' ITERATION ', k ## find average nodal coordinate of base linesegments #RsegBNC,VsegBNC = (NCB[RsegmB[:,0],]+NCB[RsegmB[:,1],])/2,(NCB[VsegmB[:,0],]+NCB[VsegmB[:,1],])/2 ## set up k-d tree of nodes and segments in Base model lines #print 'BASE: Determine triangle centroids, triangle normals and weighted vertex normals' #TBCB = np.c_[np.sum(np.c_[NCB[TriB[:,0],0],NCB[TriB[:,1],0],NCB[TriB[:,2],0]],1)/3, #np.sum(np.c_[NCB[TriB[:,0],1],NCB[TriB[:,1],1],NCB[TriB[:,2],1]],1)/3,np.sum(np.c_[NCB[TriB[:,0],2],NCB[TriB[:,1],2],NCB[TriB[:,2],2]],1)/3] #TNORMB = np.cross(NCB[TriB[:,1],:]-NCB[TriB[:,0],:],NCB[TriB[:,2],:]-NCB[TriB[:,0],:]) #TNORMB = (TNORMB.T/(np.ones((3,1))*np.sqrt(np.sum(np.array([TNORMB*TNORMB]),2)))).T #VNORMB = vrtxnormal(NCB,TriB,TBCB,TNORMB) #print 'BASE: Determine segment normal directions' #SegNormRB,SegNormVB = (VNORMB[RsegmB[:,0],]+VNORMB[RsegmB[:,1],]),(VNORMB[VsegmB[:,0],]+VNORMB[VsegmB[:,1],]) #SegNormRB = (SegNormRB.T/(np.ones((3,1))*np.sqrt(np.sum(np.array([SegNormRB*SegNormRB]),2)))).T #SegNormVB = (SegNormVB.T/(np.ones((3,1))*np.sqrt(np.sum(np.array([SegNormVB*SegNormVB]),2)))).T ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~REGISTRATION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~## print 'Register feature lines' R12R = FeatReg(RnodesB, NCB, VNORMB, RsegmT, RsegTNC, SegNormRT, RnodesT, NCT, DistMax) #find registered mesh 1 to 2 Ridges R12V = FeatReg(VnodesB, NCB, VNORMB, VsegmT, VsegTNC, SegNormVT, VnodesT, NCT, DistMax) #find registered mesh 1 to 2 Valleys R21R = FeatReg(RnodesT, NCT, VNORMT, RsegmB, RsegBNC, SegNormRB, RnodesB, NCB, DistMax) #find registered mesh 2 to 1 Ridges R21V = FeatReg(VnodesT, NCT, VNORMT, VsegmB, VsegBNC, SegNormVB, VnodesB, NCB, DistMax) #find registered mesh 2 to 1 Valleys print 'Build topological map to disregard unmatched features' R12R, R21R = BuildMap(NCB, NCT, RnodesB, RnodesT, RsegmB, RsegmT, R12R, R21R, percentReg) R12V, R21V = BuildMap(NCB, NCT, VnodesB, VnodesT, VsegmB, VsegmT, R12V, R21V, percentReg) print 'Determine Base deformation required' B12R, T12R = NCB[np.array(R12R[:, 0], int), ], R12R[:, 1:4] B12V, T12V = NCB[np.array(R12V[:, 0], int), ], R12V[:, 1:4] B21R, T21R = R21R[:, 1:4], NCT[np.array(R21R[:, 0], int), ] B21V, T21V = R21V[:, 1:4], NCT[np.array(R21V[:, 0], int), ] Base_i = np.r_[B12R, B12V, B21R, B21V] Target_i = np.r_[T12R, T12V, T21R, T21V] DISP = Target_i - Base_i DISP_Full = np.zeros(NCB.shape) print 'Determine smooth defromation' results = pprocess.Map(limit=LIM) calc = results.manage(pprocess.MakeParallel(GaussMove)) for j in range(0, LIM): calc( np.array(range(0, NPP1)) + j * NPP1, NCB, Base_i, DISP, DISP_Full, Supp) for j in range(0, LIM): DISP_Full[np.array(range(0, NPP1)) + j * NPP1, :] = results[j] DISP_Full[range(LIM * NPP1, N1), :] = GaussMove( range(LIM * NPP1, N1), NCB, Base_i, DISP, DISP_Full, Supp) NCB = NCB + DISP_Full deform = np.sum(np.sqrt(np.sum(DISP_Full * DISP_Full, 1))) / N1 print ' TOTAL CURRENT DEFORMATION: ', deform # return which NODES within surface features are registered: keepB = np.r_[R12R[:, 0], R12V[:, 0]] keepT = np.r_[R21R[:, 0], R21V[:, 0]] keepB, keepT = np.array(keepB, int), np.array(keepT, int) return NCB, keepB, keepT
def gauss_fitter(region='Cepheus_L1251', snr_min=3.0, mol='C2S', vmin=5.0, vmax=10.0, convolve=False, use_old_conv=False, multicore=1, file_extension=None): """ Fit a Gaussian to non-NH3 emission lines from GAS. It creates a cube for the best-fit Gaussian, a cube for the best-fit Gaussian with noise added back into the spectrum, and a parameter map of Tpeak, Vlsr, and FWHM Parameters ---------- region : str Name of region to reduce snr_min : float Lowest signal-to-noise pixels to include in the line-fitting mol : str name of molecule to fit vmin : numpy.float Minimum centroid velocity, in km/s. vmax : numpy.float Maximum centroid velocity, in km/s. convolve : bool or float If not False, specifies the beam-size to convolve the original map with Beam-size must be given in arcseconds use_old_conv : bool If True, use an already convolved map with name: region + '_' + mol + file_extension + '_conv.fits' This convolved map must be in units of km/s multicore : int Maximum number of simultaneous processes desired file_extension: str filename extension """ if file_extension: root = file_extension else: # root = 'base{0}'.format(blorder) root = 'all' molecules = ['C2S', 'HC7N_22_21', 'HC7N_21_20', 'HC5N'] MolFile = '{0}/{0}_{2}_{1}.fits'.format(region, root, mol) ConvFile = '{0}/{0}_{2}_{1}_conv.fits'.format(region, root, mol) GaussOut = '{0}/{0}_{2}_{1}_gauss_cube.fits'.format(region, root, mol) GaussNoiseOut = '{0}/{0}_{2}_{1}_gauss_cube_noise.fits'.format( region, root, mol) ParamOut = '{0}/{0}_{2}_{1}_param_cube.fits'.format(region, root, mol) # Load the spectral cube and convert to velocity units cube = SpectralCube.read(MolFile) cube_km = cube.with_spectral_unit(u.km / u.s, velocity_convention='radio') # If desired, convolve map with larger beam # or load previously created convolved cube if convolve: cube = SpectralCube.read(MolFile) cube_km_1 = cube.with_spectral_unit(u.km / u.s, velocity_convention='radio') beam = radio_beam.Beam(major=convolve * u.arcsec, minor=convolve * u.arcsec, pa=0 * u.deg) cube_km = cube_km_1.convolve_to(beam) cube_km.write(ConvFile, format='fits', overwrite=True) if use_old_conv: cube_km = SpectralCube.read(ConvFile) # Define the spectral axis in km/s spectra_x_axis_kms = np.array(cube_km.spectral_axis) # Find the channel range corresponding to vmin and vmax # -- This is a hold-over from when I originally set up the code to # use a channel range rather than velocity range. # Can change later, but this should work for now. low_channel = np.where(spectra_x_axis_kms <= vmax )[0][0] + 1 # Add ones to change index to channel high_channel = np.where(spectra_x_axis_kms >= vmin )[0][-1] + 1 # Again, hold-over from older setup peak_channels = [low_channel, high_channel] # Create cubes for storing the fitted Gaussian profiles # and the Gaussians with noise added back into the spectrum header = cube_km.header cube_gauss = np.array(cube_km.unmasked_data[:, :, :]) cube_gauss_noise = np.array(cube_km.unmasked_data[:, :, :]) shape = np.shape(cube_gauss) # Set up a cube for storing fitted parameters param_cube = np.zeros((6, shape[1], shape[2])) param_header = cube_km.header # Define the Gaussian profile def p_eval(x, a, x0, sigma): return a * np.exp(-(x - x0)**2 / (2 * sigma**2)) # Create some arrays full of NANs # To be used in output cubes if fits fail nan_array = np.empty(shape[0]) # For gauss cubes nan_array[:] = np.NAN nan_array2 = np.empty(param_cube.shape[0]) # For param cubes nan_array2[:] = np.NAN # Loop through each pixel and find those # with SNR above snr_min x = [] y = [] pixels = 0 for (i, j), value in np.ndenumerate(cube_gauss[0]): spectra = np.array(cube_km.unmasked_data[:, i, j]) if (False in np.isnan(spectra)): rms = np.nanstd( np.append(spectra[0:(peak_channels[0] - 1)], spectra[(peak_channels[1] + 1):len(spectra)])) if (max(spectra[peak_channels[0]:peak_channels[1]]) / rms) > snr_min: pixels += 1 x.append(i) y.append(j) else: cube_gauss[:, i, j] = nan_array param_cube[:, i, j] = nan_array2 cube_gauss_noise[:, i, j] = nan_array print str(pixels) + ' Pixels above SNR=' + str(snr_min) # Define a Gaussian fitting function for each pixel # i, j are the x,y coordinates of the pixel being fit def pix_fit(i, j): spectra = np.array(cube_km.unmasked_data[:, i, j]) # Use the peak brightness Temp within specified channel # range as the initial guess for Gaussian height max_ch = np.argmax(spectra[peak_channels[0]:peak_channels[1]]) Tpeak = spectra[peak_channels[0]:peak_channels[1]][max_ch] # Use the velocity of the brightness Temp peak as # initial guess for Gaussian mean vpeak = spectra_x_axis_kms[peak_channels[0]:peak_channels[1]][max_ch] rms = np.std( np.append(spectra[0:(peak_channels[0] - 1)], spectra[(peak_channels[1] + 1):len(spectra)])) err1 = np.zeros(shape[0]) + rms # Create a noise spectrum based on rms of off-line channels # This will be added to best-fit Gaussian to obtain a noisy Gaussian noise = np.random.normal(0., rms, len(spectra_x_axis_kms)) # Define initial guesses for Gaussian fit guess = [Tpeak, vpeak, 0.3] # [height, mean, sigma] try: coeffs, covar_mat = curve_fit(p_eval, xdata=spectra_x_axis_kms, ydata=spectra, p0=guess, sigma=err1, maxfev=500) gauss = np.array( p_eval(spectra_x_axis_kms, coeffs[0], coeffs[1], coeffs[2])) noisy_gauss = np.array( p_eval(spectra_x_axis_kms, coeffs[0], coeffs[1], coeffs[2])) + noise params = np.append(coeffs, (covar_mat[0][0]**0.5, covar_mat[1][1]** 0.5, covar_mat[2][2]**0.5)) # params = ['Tpeak', 'VLSR','sigma','Tpeak_err','VLSR_err','sigma_err'] # Don't accept fit if fitted parameters are non-physical or too uncertain if (params[0] < 0.01) or (params[3] > 1.0) or ( params[2] < 0.05) or (params[5] > 0.5) or (params[4] > 0.75): noisy_gauss = nan_array gauss = nan_array params = nan_array2 # Don't accept fit if the SNR for fitted spectrum is less than SNR threshold #if max(gauss)/rms < snr_min: # noisy_gauss = nan_array # gauss = nan_array # params = nan_array2 except RuntimeError: noisy_gauss = nan_array gauss = nan_array params = nan_array2 return i, j, gauss, params, noisy_gauss # Parallel computation: nproc = multicore # maximum number of simultaneous processes desired queue = pprocess.Queue(limit=nproc) calc = queue.manage(pprocess.MakeParallel(pix_fit)) tic = time.time() counter = 0 # Uncomment to see some plots of the fitted spectra #for i,j in zip(x,y): #pix_fit(i,j) #plt.plot(spectra_x_axis_kms, spectra, color='blue', drawstyle='steps') #plt.plot(spectra_x_axis_kms, gauss, color='red') #plt.show() #plt.close() # Begin parallel computations # Store the best-fit Gaussians and parameters # in their correct positions in the previously created cubes for i, j in zip(x, y): calc(i, j) for i, j, gauss_spec, parameters, noisy_gauss_spec in queue: cube_gauss[:, i, j] = gauss_spec param_cube[:, i, j] = parameters cube_gauss_noise[:, i, j] = noisy_gauss_spec counter += 1 print str(counter) + ' of ' + str(pixels) + ' pixels completed \r', sys.stdout.flush() print "\n %f s for parallel computation." % (time.time() - tic) # Save final cubes # These will be in km/s units. # Spectra will have larger values to the left, lower values to right cube_final_gauss = SpectralCube(data=cube_gauss, wcs=cube_km.wcs, header=cube_km.header) cube_final_gauss.write(GaussOut, format='fits', overwrite=True) cube_final_gauss_noise = SpectralCube(data=cube_gauss_noise, wcs=cube_km.wcs, header=cube_km.header) cube_final_gauss_noise.write(GaussNoiseOut, format='fits', overwrite=True) # Construct appropriate header for param_cube param_header['NAXIS3'] = len(nan_array2) param_header['WCSAXES'] = 3 param_header['CRPIX3'] = 1 param_header['CDELT3'] = 1 param_header['CRVAL3'] = 0 param_header['PLANE1'] = 'Tpeak' param_header['PLANE2'] = 'VLSR' param_header['PLANE3'] = 'sigma' param_header['PLANE5'] = 'Tpeak_err' param_header['PLANE6'] = 'VLSR_err' param_header['PLANE7'] = 'sigma_err' fits.writeto(ParamOut, param_cube, header=param_header, clobber=True)