def get_files(self, file_def, **subdirs): # Some sanity checks: subdir_labels = set(subdirs.keys()) if not self.mandatory_dirs.issubset(subdir_labels): #print ssubdirs #print self.mandatory_dirs diff = self.mandatory_dirs.difference(subdir_labels) raise Exception('\n'.join(['Missing mandatory directories:'] + \ list(diff))) for dir_label in subdirs.iterkeys(): assert dir_label in self.dir_labels #TODO check that input subdirs are all in allowed_subdirectories all_dirs = self.dir_values.copy() all_dirs.update(subdirs) all_dirs = apply_to_dict(all_dirs, self._forge_sub_dir) selected_dirs = cartesian(*all_dirs.values()) dir_ids = dict([(i, l) for l, i in enumerate(all_dirs)]) pathes = [join(*[d[dir_ids[l]] for l in self.dir_labels]) \ for d in selected_dirs] if self.root is None: raise Exception('Root is not defined!') if not exists(self.root): raise Exception('Root does not exist: %s' % self.root) subdirs_not_found = check_subdirs(self.root, self.dir_labels, all_dirs) if len(subdirs_not_found) > 0: raise Exception('\n'.join(['Levels not found in subdirs:'] + \ list(reversed(subdirs_not_found)))) #f_defs = chain(*self._resolve_file_defs(file_def)) f_defs = self._resolve_file_defs(file_def) #print list(f_defs) files = [join(*[self.root] + f) for f in cartesian(pathes, f_defs)] print 'files:' print files files_not_found = [f for f in files if not exists(f)] if len(files_not_found) > 0: raise Exception('\n'.join(['Files not found:'] + files_not_found)) return files
def get_files(self, file_def, **subdirs): # Some sanity checks: subdir_labels = set(subdirs.keys()) if not self.mandatory_dirs.issubset(subdir_labels): #print ssubdirs #print self.mandatory_dirs diff = self.mandatory_dirs.difference(subdir_labels) raise Exception('\n'.join(['Missing mandatory directories:'] + \ list(diff))) for dir_label in subdirs.iterkeys(): assert dir_label in self.dir_labels #TODO check that input subdirs are all in allowed_subdirectories all_dirs = self.dir_values.copy() all_dirs.update(subdirs) all_dirs = apply_to_dict(all_dirs, self._forge_sub_dir) selected_dirs = cartesian(*all_dirs.values()) dir_ids = dict([(i,l) for l,i in enumerate(all_dirs)]) pathes = [join(*[d[dir_ids[l]] for l in self.dir_labels]) \ for d in selected_dirs] if self.root is None: raise Exception('Root is not defined!') if not exists(self.root): raise Exception('Root does not exist: %s' %self.root) subdirs_not_found = check_subdirs(self.root, self.dir_labels, all_dirs) if len(subdirs_not_found) > 0: raise Exception('\n'.join(['Levels not found in subdirs:'] + \ list(reversed(subdirs_not_found)))) #f_defs = chain(*self._resolve_file_defs(file_def)) f_defs = self._resolve_file_defs(file_def) #print list(f_defs) files = [join(*[self.root]+f) for f in cartesian(pathes,f_defs)] print 'files:' print files files_not_found = [f for f in files if not exists(f)] if len(files_not_found) > 0: raise Exception('\n'.join(['Files not found:'] + files_not_found)) return files
def fromLattice(latticeIndexes, kerMask=None, depth=1, torusFlag=False): """ Creates a NeighbourhoodSystem instance from a n-dimensional lattice """ mpos = NP.where(latticeIndexes != -1) positions = NP.vstack(mpos).transpose() if kerMask == None: # Full neighbourhood, ie 8 in 2D, 26 in 3D ... ndim = latticeIndexes.ndim kerMask = NP.array(list(cartesian(*[[0, -1, 1]] * ndim))[1:], dtype=int) mapper = LatticeIndexMapper(latticeIndexes, torusFlag) nbPositions = latticeIndexes.size # Build lists of closest neighbours: closestNeighbours = NP.empty(nbPositions, dtype=object) for idx, pos in enumerate(positions): ni = map(mapper.indexFromCoord, kerMask + pos) closestNeighbours[idx] = set(filter(lambda x: x is not None, ni)) if depth == 1: return NeighbourhoodSystem(closestNeighbours) neighboursSets = copyModule.deepcopy(closestNeighbours) for idx in xrange(nbPositions): neighboursToTreat = closestNeighbours[idx] visited = set([idx]) for d in xrange(depth - 1): newNeighboursToTreat = set() # In the worst case, neighbours to treat are on the perimeter # of the lattice, say O(sum(lattice shape)) # OR # If the lattice is kind of flat, the worst case is # O(size of kerMask) # So complexity is : # O(max( sum(lattice shape), kerMask size)) * O(kerMask size) for n in neighboursToTreat: # O(kerMask size) newNeighboursToTreat.update(closestNeighbours[n]) # Remember what we just saw: visited.update(neighboursToTreat) # Update neighbours of current position and remove those # already seen: newNeighboursToTreat.difference_update(visited) neighboursSets[idx].update(newNeighboursToTreat) # Define new starting positions for next loop: neighboursToTreat = newNeighboursToTreat return NeighbourhoodSystem(neighboursSets)
def fromLattice(latticeIndexes, kerMask=None, depth=1, torusFlag=False): """ Creates a NeighbourhoodSystem instance from a n-dimensional lattice """ mpos = NP.where(latticeIndexes != -1) positions = NP.vstack(mpos).transpose() if kerMask == None: # Full neighbourhood, ie 8 in 2D, 26 in 3D ... ndim = latticeIndexes.ndim kerMask = NP.array( list(cartesian(*[[0, -1, 1]] * ndim))[1:], dtype=int) mapper = LatticeIndexMapper(latticeIndexes, torusFlag) nbPositions = latticeIndexes.size # Build lists of closest neighbours: closestNeighbours = NP.empty(nbPositions, dtype=object) for idx, pos in enumerate(positions): ni = map(mapper.indexFromCoord, kerMask + pos) closestNeighbours[idx] = set(filter(lambda x: x is not None, ni)) if depth == 1: return NeighbourhoodSystem(closestNeighbours) neighboursSets = copyModule.deepcopy(closestNeighbours) for idx in xrange(nbPositions): neighboursToTreat = closestNeighbours[idx] visited = set([idx]) for d in xrange(depth - 1): newNeighboursToTreat = set() # In the worst case, neighbours to treat are on the perimeter # of the lattice, say O(sum(lattice shape)) # OR # If the lattice is kind of flat, the worst case is # O(size of kerMask) # So complexity is : # O(max( sum(lattice shape), kerMask size)) * O(kerMask size) for n in neighboursToTreat: # O(kerMask size) newNeighboursToTreat.update(closestNeighbours[n]) # Remember what we just saw: visited.update(neighboursToTreat) # Update neighbours of current position and remove those # already seen: newNeighboursToTreat.difference_update(visited) neighboursSets[idx].update(newNeighboursToTreat) # Define new starting positions for next loop: neighboursToTreat = newNeighboursToTreat return NeighbourhoodSystem(neighboursSets)
def mapData(self, data, mappedAxis=0, fillValue=0): ia = mappedAxis nd = self.nbDims ranges = [range(d) for d in data.shape] ranges = ranges[:ia] + [self.positions] + ranges[ia + 1:] coords = cartesian(*ranges) coords = flattenElements(coords) # TODO: remove explicit loop! mask = [NP.array(a) for a in NP.array(coords).transpose()] destshape = data.shape[:ia] + self.targetShape + data.shape[ia + 1:] destData = NP.zeros(destshape, dtype=data.dtype) + fillValue destData[mask] = data.flat return destData
def mapData(self, data, mappedAxis=0, fillValue=0): ia = mappedAxis nd = self.nbDims ranges = [range(d) for d in data.shape] ranges = ranges[:ia] + [self.positions] + ranges[ia+1:] coords = cartesian(*ranges) coords = flattenElements(coords) #TODO: remove explicit loop! mask = [NP.array(a) for a in NP.array(coords).transpose()] destshape = data.shape[:ia] + self.targetShape + data.shape[ia+1:] destData = NP.zeros(destshape, dtype=data.dtype) + fillValue destData[mask] = data.flat return destData
def flattenData(self, data, firstMappedAxis=0): #Note: very similar to pyhrf.ndarray.buildMappingMask ... #TODO: factorize axis expansion stuffs !!! ranges = [range(d) for d in data.shape] ia = firstMappedAxis nd = self.nbDims ranges = ranges[:ia] + [self.positions] + ranges[ia+nd:] coords = cartesian(*ranges) coords = flattenElements(coords) #TODO: remove explicit loop! mask = [NP.array(a) for a in NP.array(coords).transpose()] destshape = data.shape[:ia]+(len(self.positions),)+data.shape[ia+nd:] destData = NP.empty(destshape, dtype=data.dtype) destData.flat = data[mask] return destData
def flattenData(self, data, firstMappedAxis=0): # Note: very similar to pyhrf.ndarray.buildMappingMask ... # TODO: factorize axis expansion stuffs !!! ranges = [range(d) for d in data.shape] ia = firstMappedAxis nd = self.nbDims ranges = ranges[:ia] + [self.positions] + ranges[ia + nd:] coords = cartesian(*ranges) coords = flattenElements(coords) # TODO: remove explicit loop! mask = [NP.array(a) for a in NP.array(coords).transpose()] destshape = data.shape[:ia] + \ (len(self.positions),) + data.shape[ia + nd:] destData = NP.empty(destshape, dtype=data.dtype) destData.flat = data[mask] return destData
def flattenArray(self, array, firstMappedAxis=0): """ Reduce dimensions of 'array'. 'firstMappedAxis' is index of the axis to be reduced (other mapped axes are assumed to follow this one). """ nd = len(self.expandedShape) ia = firstMappedAxis assert len(self.expandedShape) <= array.ndim assert self.expandedShape == array.shape[ia:ia + nd] ranges = [range(d) for d in array.shape] ranges = ranges[:ia] + [self.mapping] + ranges[ia + nd:] coords = cartesian(*ranges) coords = flattenElements(coords) # TODO: remove explicit loop! mask = [NP.array(a) for a in NP.array(coords).transpose()] destshape = array.shape[:ia] + \ (len(self.mapping),) + array.shape[ia + nd:] destData = NP.empty(destshape, dtype=array.dtype) destData.flat = array[mask] return destData
def graph_from_lattice(mask, kerMask=None, depth=1, toroidal=False): """ Creates a graph from a n-dimensional lattice 'mask' define valid positions to build the graph over. 'kerMask' is numpy array mask (tuple of arrays) which defines the neighbourhood system, ie the relative positions of neighbours for a given position in the lattice. """ # print 'size:', latticeIndexes.size if kerMask is not None: assert mask.ndim == len(kerMask) else: # Full neighbourhood, ie 8 in 2D, 26 in 3D ... ndim = mask.ndim neighbourCoords = list(cartesian(*[[0, -1, 1]] * ndim))[1:] kerMask = tuple(np.array(neighbourCoords, dtype=int).transpose()) # loop over valid positions: # print mask.shape positions = mask_to_coords(mask) latticeIndexes = lattice_indexes(mask) # print positions # print positions.shape # Build lists of closest neighbours: closestNeighbours = np.empty(len(positions), dtype=object) for idx, pos in enumerate(positions): # print 'idx :', idx, '- pos:', pos # O(n) : m = center_mask_at(kerMask, pos, latticeIndexes, toroidal) # print 'm:', m closestNeighbours[idx] = latticeIndexes[m][latticeIndexes[m] >= 0] if depth == 1: return closestNeighbours neighboursSets = copyModule.deepcopy(closestNeighbours) for idx in xrange(nbPositions): # Starting positions = closest neighbours of current position neighboursToTreat = closestNeighbours[idx] visited = set([idx]) # O(max( sum(lattice shape), kerMask size))*O(kerMask size)*O(d) for d in xrange(depth - 1): newNeighboursToTreat = set() # In the worst case, neighbours to treat are on the perimeter # of the lattice, say O(sum(lattice shape)) # OR # If the lattice is kind of flat, the worst case is # O(size of kerMask) # So complexity is : # O(max( sum(lattice shape), kerMask size)) * O(kerMask size) for n in neighboursToTreat: # O(kerMask size) newNeighboursToTreat.update(closestNeighbours[n]) # Remember what we just saw: visited.update(neighboursToTreat) # Update neighbours of current position and remove those # already seen: newNeighboursToTreat.difference_update(visited) neighboursSets[idx].update(newNeighboursToTreat) # Define new starting positions for next loop: neighboursToTreat = newNeighboursToTreat return neighboursSets
def testCartesianBasic(self): domains = [(0, 1, 2), ('a', 'b')] cartProd = list(cartesian(*domains)) assert cartProd == [[0, 'a'], [0, 'b'], [1, 'a'], [1, 'b'], [2, 'a'], [2, 'b']]
def graph_from_lattice(mask, kerMask=None, depth=1, toroidal=False): """ Creates a graph from a n-dimensional lattice 'mask' define valid positions to build the graph over. 'kerMask' is numpy array mask (tuple of arrays) which defines the neighbourhood system, ie the relative positions of neighbours for a given position in the lattice. """ #print 'size:', latticeIndexes.size if kerMask is not None: assert mask.ndim == len(kerMask) else: # Full neighbourhood, ie 8 in 2D, 26 in 3D ... ndim = mask.ndim neighbourCoords = list(cartesian(*[[0,-1,1]]*ndim))[1:] kerMask = tuple(np.array(neighbourCoords, dtype=int).transpose()) # loop over valid positions: #print mask.shape positions = mask_to_coords(mask) latticeIndexes = lattice_indexes(mask) #print positions #print positions.shape # Build lists of closest neighbours: closestNeighbours = np.empty(len(positions), dtype=object) for idx, pos in enumerate(positions): #print 'idx :', idx, '- pos:', pos # O(n) : m = center_mask_at(kerMask, pos, latticeIndexes, toroidal) #print 'm:', m closestNeighbours[idx] = latticeIndexes[m][latticeIndexes[m]>=0] if depth == 1: return closestNeighbours neighboursSets = copyModule.deepcopy(closestNeighbours) for idx in xrange(nbPositions): # Starting positions = closest neighbours of current position neighboursToTreat = closestNeighbours[idx] visited = set([idx]) # O(max( sum(lattice shape), kerMask size))*O(kerMask size)*O(d) for d in xrange(depth-1): newNeighboursToTreat = set() # In the worst case, neighbours to treat are on the perimeter # of the lattice, say O(sum(lattice shape)) # OR # If the lattice is kind of flat, the worst case is # O(size of kerMask) # So complexity is : # O(max( sum(lattice shape), kerMask size)) * O(kerMask size) for n in neighboursToTreat: # O(kerMask size) newNeighboursToTreat.update(closestNeighbours[n]) # Remember what we just saw: visited.update(neighboursToTreat) # Update neighbours of current position and remove those # already seen: newNeighboursToTreat.difference_update(visited) neighboursSets[idx].update(newNeighboursToTreat) # Define new starting positions for next loop: neighboursToTreat = newNeighboursToTreat return neighboursSets
class RegularLatticeMapping(SpatialMapping): """ Define a SpatialMapping on a 3D regular lattice. """ order1Mask = NP.array([[1, 0, 0], [0, 1, 0], [0, 0, 1], [-1, 0, 0], [0, -1, 0], [0, 0, -1]]) nbNeighboursOrder1 = 6 order2Mask = NP.array( [c for c in cartesian([0, -1, 1], [0, -1, 1], [0, -1, 1])][1:]) nbNeighboursOrder2 = 26 # TODO AL: handle mapping better # TODO AL: hide advanced options # ) # TODO : add handling of periodic coordinate systems like a torus ... # TODO : check diff between order and depth ... # , **kwargs): def __init__(self, shape=None, mapping=None, order=1, depth=1): """ Construct a spatial configuration corresponding to a regular 3D lattice. 'shape' is a 3-item sequence defining the X, Y and Z sizes. Note on aims dimension conventions : (axial, coronal, sagittal) 'mapping' is a list of integer coordinates defining which positions are valid in the regular lattice. 'order' is the order of the neighbour system in 3D. Examples : - if order=1 => 6 neighbour system (with depth=1) - if order=2 => 26 neighbour system (with depth=1) 'depth' is the depth of the neighbour system. If 'mapping'=None, the default lattice is full and fill order is X->Y->Z. """ if shape != None: assert len(shape) == 3 else: shape = (1, 10, 10) assert len(shape) == 3 assert (order == 1 or order == 2) self.shape = tuple(shape) self.depth = depth if order == 1: self.neighbourMask = self.order1Mask self.nbNeighboursMax = self.nbNeighboursOrder1 else: self.neighbourMask = self.order2Mask self.nbNeighboursMax = self.nbNeighboursOrder2 if mapping == None: # If no mapping is defined then the lattice is "full" : self.nbVoxels = shape[0] * shape[1] * shape[2] # Default fill order of the lattice is : X->Y->Z : lxly = shape[0] * shape[1] lx = shape[0] ly = shape[1] self.mapping = NP.array([ [ (i % (lxly)) / ly, (i % (lxly)) % ly, i / (lxly) ] # creates a mapping by affecting a voxel position to each number corresponding to a voxel for i in xrange(self.nbVoxels) ]) else: # Else valid positions are defined with 'mask' : self.mapping = mapping self.nbVoxels = len(mapping) mask = NP.array(self.mapping) # Center working origin on the "upper left corner" : self.correctedMask = mask - mask.min(0) # TODO : assert that corrected mask fits in volume with given shape self.ndarrayMask = (NP.array([c[0] for c in self.mapping]), NP.array([c[1] for c in self.mapping]), NP.array([c[2] for c in self.mapping])) self.ndarrayCorMask = (NP.array([c[0] for c in self.correctedMask]), NP.array([c[1] for c in self.correctedMask]), NP.array([c[2] for c in self.correctedMask])) # Build the lattice, positions with '-1' are not valid : self.lattice = NP.zeros((self.shape), dtype=int) - 1 # Fill lattice with integer indexes corresponding to valid positions : for iv in xrange(self.nbVoxels): x = self.correctedMask[iv, 0] y = self.correctedMask[iv, 1] z = self.correctedMask[iv, 2] # Fill lattice with index corresponding to a valid position : self.lattice[x, y, z] = iv self.neighboursCoordLists = None self.neighboursIndexLists = None self.buildNeighboursIndexLists() self.buildNeighboursCoordLists() def getNbVoxels(self): return self.nbVoxels def getTargetAxesNames(self): return ['axial', 'coronal', 'sagittal'] # TODO make less static def getMapping(self): return self.mapping def getRoiMask(self): """ Return a binary or n-ary 3D mask which has the shape of the target data """ return NP.ones(self.shape, dtype=int) # every position marked as 1 def getNdArrayMask(self): return self.ndarrayMask def getCoord(self, index): return self.mapping[index] def getIndex(self, coord): return self.lattice[coord] def getNeighboursCoords(self, idvoxel): if self.neighboursCoordLists != None: return self.neighboursCoordLists[idvoxel] else: return [ self.mapping[i] for i in self.getNeighboursIndexes(idvoxel) ] def getClosestNeighboursIndexes(self, idVoxel): cv = self.correctedMask[idVoxel] neighbours = [] # translate center of neighbours mask to voxel position : neighboursTmp = self.neighbourMask + cv for cn in neighboursTmp: if NP.bitwise_and(cn >= 0, cn < self.shape).all(): neighbours.append(cn.tolist()) return [ self.lattice[n[0], n[1], n[2]] for n in neighbours if self.lattice[n[0], n[1], n[2]] != -1 ] def getNeighboursIndexes(self, idVoxel): if self.neighboursIndexLists != None: return self.neighboursIndexLists[idVoxel] else: neighbours = set([idVoxel]) for d in xrange(self.depth): newNeighbours = [] for i in neighbours: newNeighbours.extend(self.getClosestNeighboursIndexes(i)) neighbours.update(newNeighbours) neighbours.remove(idVoxel) return list(neighbours) def buildNeighboursIndexLists(self): neighboursListsTmp = range(self.nbVoxels) for i in xrange(self.nbVoxels): neighboursListsTmp[i] = self.getNeighboursIndexes(i) self.neighboursIndexLists = neighboursListsTmp def buildNeighboursCoordLists(self): neighboursListsTmp = range(self.nbVoxels) for i in xrange(self.nbVoxels): neighboursListsTmp[i] = self.getNeighboursCoords(i) self.neighboursCoordLists = neighboursListsTmp def getNeighboursIndexLists(self): return self.neighboursIndexLists def getNeighboursCoordLists(self): return self.neighboursCoordLists def getNbCliques(self): if not hasattr(self, 'nbCliques') or self.nbCliques is None: self.nbCliques = sum([len(x) for x in self.neighboursIndexLists]) / 2. return self.nbCliques def mapVoxData(self, data, fillValue=0): if len(data.shape) == 1: mappedData = NP.zeros(self.shape, dtype=data.dtype) + fillValue mappedData[self.ndarrayMask] = data elif len(data.shape) == 2: # assume (time, nbVox) sh = tuple([data.shape[0]]) + self.shape mappedData = NP.zeros(sh, dtype=data.dtype) + fillValue m = self.ndarrayCorMask mappedData[:, m[0], m[1], m[2]] = data return mappedData @staticmethod def createFromGUI(GUIobject): """ Creates the actual object based on the parameters """ params = {} params[RegularLatticeMapping.P_XSIZE] = getattr( GUIobject, RegularLatticeMapping.P_XSIZE) params[RegularLatticeMapping.P_YSIZE] = getattr( GUIobject, RegularLatticeMapping.P_YSIZE) params[RegularLatticeMapping.P_ZSIZE] = getattr( GUIobject, RegularLatticeMapping.P_ZSIZE) params[RegularLatticeMapping.P_ORDER] = getattr( GUIobject, RegularLatticeMapping.P_ORDER) params[RegularLatticeMapping.P_DEPTH] = getattr( GUIobject, RegularLatticeMapping.P_DEPTH) return RegularLatticeMapping(**params)
def fromLattice(latticeIndexes, kerMask=None, depth=1, torusFlag=False): """ Creates a NeighbourhoodSystem instance from a n-dimensional lattice """ mpos = NP.where(latticeIndexes != -1) positions = NP.vstack(mpos).transpose() if kerMask == None: # Full neighbourhood, ie 8 in 2D, 26 in 3D ... ndim = latticeIndexes.ndim kerMask = NP.array(list(cartesian(*[[0,-1,1]]*ndim))[1:], dtype=int) #indexLattice = NP.zeros_like(maskLattice) #indexLattice[mpos] = range(nbPositions) mapper = LatticeIndexMapper(latticeIndexes, torusFlag) nbPositions = latticeIndexes.size # Build lists of closest neighbours: closestNeighbours = NP.empty(nbPositions, dtype=object) for idx, pos in enumerate(positions): #print 'idx :', idx, '- pos:', pos # O(n) : #print 'kerMask+pos =', kerMask+pos ni = map(mapper.indexFromCoord, kerMask+pos) closestNeighbours[idx] = set(filter(lambda x: x is not None, ni)) #print '->closestNeighbours[%d]=%s' %(idx,closestNeighbours[idx]) if depth == 1: return NeighbourhoodSystem(closestNeighbours) neighboursSets = copyModule.deepcopy(closestNeighbours) for idx in xrange(nbPositions): ## print 'Treating idx :', idx # Starting positions = closest neighbours of current position neighboursToTreat = closestNeighbours[idx] ## print ' neighboursToTreat =', neighboursToTreat visited = set([idx]) ## print ' already visited =', visited # O(max( sum(lattice shape), kerMask size))*O(kerMask size)*O(d) for d in xrange(depth-1): ## print ' d=', d ## print ' We have to treat:', neighboursToTreat newNeighboursToTreat = set() # In the worst case, neighbours to treat are on the perimeter # of the lattice, say O(sum(lattice shape)) # OR # If the lattice is kind of flat, the worst case is # O(size of kerMask) # So complexity is : # O(max( sum(lattice shape), kerMask size)) * O(kerMask size) for n in neighboursToTreat: ## print ' processing neighbour:', n ## print ' updating to treat with;', closestNeighbours[n] # O(kerMask size) newNeighboursToTreat.update(closestNeighbours[n]) # Remember what we just saw: ## print ' We have just visited:', neighboursToTreat visited.update(neighboursToTreat) ## print ' So all visited are;', visited # Update neighbours of current position and remove those # already seen: newNeighboursToTreat.difference_update(visited) ## print ' neighbours which are left to treat:', \ ## newNeighboursToTreat neighboursSets[idx].update(newNeighboursToTreat) ## print ' list of neighbours for %d is now: %s' \ ## %(idx, str(neighboursSets[idx])) # Define new starting positions for next loop: neighboursToTreat = newNeighboursToTreat return NeighbourhoodSystem(neighboursSets)