def bm29ize(self): """transform the data of id24 in a list of bm29 instance""" self.spectra=[] for n,item in enumerate(scipy.hsplit(self.data[:,1:], self.data.shape[1]-1)): self.spectra.append(bm29file([self.energy, item[:,0]])) self.spectra[n].comments= list(self.comments) self.spectra[n].comments.append("# spectra number "+str(n)+ "\n") self.spectra[n].comments.append("# ---------------------------------"+ "\n") self.spectra[n].comments.append("#L E Mu"+ "\n")
def openSinglefile(filename): inFile = open(filename, 'r') buffero= scipy.loadtxt(inFile) inFile.close() x_array=buffero[:,0] spectra=[] for item in scipy.hsplit(buffero[:,1:], buffero.shape[1]-1): spectra.append(generic(x_array,scipy.ravel(item))) return spectra
def openSinglefile(filename): inFile = open(filename, 'r') buffero = scipy.loadtxt(inFile) inFile.close() x_array = buffero[:, 0] spectra = [] for item in scipy.hsplit(buffero[:, 1:], buffero.shape[1] - 1): spectra.append(generic(x_array, scipy.ravel(item))) return spectra
def idct2(X, blksize): """Calculate the inverse DCT transform of a 2D array, X""" dctm = dct_mat(blksize) #try: blks = [sp.vsplit(x, X.shape[1]/blksize) for x in sp.hsplit(X, X.shape[0]/blksize)] #except: # print "Some error occurred" blks = [sp.dot(sp.dot(dctm.T, b), dctm) for b in blks] return sp.concatenate([blk for blk in blks]).reshape(X.shape)
def bm29ize(self): """transform the data of id24 in a list of bm29 instance""" self.spectra = [] for n, item in enumerate( scipy.hsplit(self.data[:, 1:], self.data.shape[1] - 1)): self.spectra.append(bm29file([self.energy, item[:, 0]])) self.spectra[n].comments = list(self.comments) self.spectra[n].comments.append("# spectra number " + str(n) + "\n") self.spectra[n].comments.append( "# ---------------------------------" + "\n") self.spectra[n].comments.append("#L E Mu" + "\n")
def dct2(X, blksize): """Calculate DCT transform of a 2D array, X In order for this work, we have to split X into blksize chunks""" dctm = dct_mat(blksize) #try: blks = [sp.vsplit(x, X.shape[1]/blksize) for x in sp.hsplit(X, X.shape[0]/blksize)] #except: # print "Some error occurred" blks = [sp.dot(sp.dot(dctm, b), dctm.T) for b in blks] return sp.concatenate([blk for blk in blks]).reshape(X.shape)
def load(cls, path, node_file="throats_cellsThroatsGraph_Nodes.txt", graph_file="throats_cellsThroatsGraph.txt", network=None, voxel_size=None, return_geometry=False): r""" Loads network data from an iMorph processed image stack Parameters ---------- path : string The path of the folder where the subfiles are held node_file : string The file that describes the pores and throats, the default iMorph name is: throats_cellsThroatsGraph_Nodes.txt graph_file : string The file that describes the connectivity of the network, the default iMorph name is: throats_cellsThroatsGraph.txt network : OpenPNM Network Object The OpenPNM Network onto which the data should be loaded. If no network is supplied then an empty import network is created and returned. voxel_size : float Allows the user to define a voxel size different than what is contained in the node_file. The value must be in meters. return_geometry : Boolean If True, then all geometrical related properties are removed from the Network object and added to a GenericGeometry object. In this case the method returns a tuple containing (network, geometry). If False (default) then the returned Network will contain all properties that were in the original file. In this case, the user can call the ```split_geometry``` method explicitly to perform the separation. Returns ------- If no Network object is supplied then one will be created and returned. If return_geometry is True, then a tuple is returned containing both the network and a geometry object. """ # path = Path(path) node_file = os.path.join(path.resolve(), node_file) graph_file = os.path.join(path.resolve(), graph_file) # parsing the nodes file with open(node_file, 'r') as file: Np = sp.fromstring(file.readline().rsplit('=')[1], sep='\t', dtype=int)[0] vox_size = sp.fromstring( file.readline().rsplit(')')[1], sep='\t', )[0] # network always recreated to prevent errors network = GenericNetwork(Np=Np, Nt=0) # Define expected properies network['pore.volume'] = sp.nan scrap_lines = [file.readline() for line in range(4)] while True: vals = file.readline().split('\t') if len(vals) == 1: break network['pore.volume'][int(vals[0])] = float(vals[3]) if 'pore.' + vals[2] not in network.labels(): network['pore.' + vals[2]] = False network['pore.' + vals[2]][int(vals[0])] = True if voxel_size is None: voxel_size = vox_size * 1.0E-6 # file stores value in microns if voxel_size < 0: raise (Exception('Error - Voxel size must be specfied in ' + 'the Nodes file or as a keyword argument.')) # parsing the graph file with open(graph_file, 'r') as file: # Define expected properties network['pore.coords'] = sp.zeros((Np, 3)) * sp.nan network['pore.types'] = sp.nan network['pore.color'] = sp.nan network['pore.radius'] = sp.nan network['pore.dmax'] = sp.nan network['pore.node_number'] = sp.nan # Scan file to get pore coordinate data scrap_lines = [file.readline() for line in range(3)] line = file.readline() xmax = 0.0 ymax = 0.0 zmax = 0.0 node_num = 0 while line != 'connectivity table\n': vals = sp.fromstring(line, sep='\t') xmax = vals[1] if vals[1] > xmax else xmax ymax = vals[2] if vals[2] > ymax else ymax zmax = vals[3] if vals[3] > zmax else zmax network['pore.coords'][int(vals[0]), :] = vals[1:4] network['pore.types'][int(vals[0])] = vals[4] network['pore.color'][int(vals[0])] = vals[5] network['pore.radius'][int(vals[0])] = vals[6] network['pore.dmax'][int(vals[0])] = vals[7] network['pore.node_number'][int(vals[0])] = node_num node_num += 1 line = file.readline() # Scan file to get to connectivity data scrap_lines.append(file.readline()) # Skip line # Create sparse lil array incrementally build adjacency matrix lil = sp.sparse.lil_matrix((Np, Np), dtype=int) while True: vals = sp.fromstring(file.readline(), sep='\t', dtype=int) if len(vals) <= 1: break lil.rows[vals[0]] = vals[2:] lil.data[vals[0]] = sp.ones(vals[1]) # fixing any negative volumes or distances so they are 1 voxel/micron network['pore.volume'][sp.where(network['pore.volume'] < 0)[0]] = 1.0 network['pore.radius'][sp.where(network['pore.radius'] < 0)[0]] = 1.0 network['pore.dmax'][sp.where(network['pore.dmax'] < 0)[0]] = 1.0 # Add adjacency matrix to OpenPNM network conns = sp.sparse.triu(lil, k=1, format='coo') network.update({'throat.all': sp.ones(len(conns.col), dtype=bool)}) network['throat.conns'] = sp.vstack([conns.row, conns.col]).T network['pore.to_trim'] = False network['pore.to_trim'][network.pores('*throat')] = True Ts = network.pores('to_trim') new_conns = network.find_neighbor_pores(pores=Ts, flatten=False) extend(network=network, throat_conns=new_conns, labels='new_conns') for item in network.props('pore'): item = item.split('.')[1] arr = sp.ones_like(network['pore.' + item])[0] arr = sp.tile(A=arr, reps=[network.Nt, 1]) * sp.nan network['throat.' + item] = sp.squeeze(arr) network['throat.'+item][network.throats('new_conns')] = \ network['pore.'+item][Ts] trim(network=network, pores=Ts) # setting up boundary pores x_coord, y_coord, z_coord = sp.hsplit(network['pore.coords'], 3) network['pore.front_boundary'] = sp.ravel(x_coord == 0) network['pore.back_boundary'] = sp.ravel(x_coord == xmax) network['pore.left_boundary'] = sp.ravel(y_coord == 0) network['pore.right_boundary'] = sp.ravel(y_coord == ymax) network['pore.bottom_boundary'] = sp.ravel(z_coord == 0) network['pore.top_boundary'] = sp.ravel(z_coord == zmax) # removing any pores that got classified as a boundary pore but # weren't labled a border_cell_face ps = sp.where(~sp.in1d(network.pores('*_boundary'), network.pores('border_cell_face')))[0] ps = network.pores('*_boundary')[ps] for side in ['front', 'back', 'left', 'right', 'top', 'bottom']: network['pore.' + side + '_boundary'][ps] = False # setting internal label network['pore.internal'] = False network['pore.internal'][network.pores('*_boundary', mode='not')] = True # adding props to border cell face throats and from pores Ts = sp.where( network['throat.conns'][:, 1] > network.pores('border_cell_face')[0] - 1)[0] faces = network['throat.conns'][Ts, 1] for item in network.props('pore'): item = item.split('.')[1] network['throat.' + item][Ts] = network['pore.' + item][faces] network['pore.volume'][faces] = 0.0 # applying unit conversions # TODO: Determine if radius and dmax are indeed microns and not voxels network['pore.coords'] = network['pore.coords'] * 1e-6 network['pore.radius'] = network['pore.radius'] * 1e-6 network['pore.dmax'] = network['pore.dmax'] * 1e-6 network['pore.volume'] = network['pore.volume'] * voxel_size**3 network['throat.coords'] = network['throat.coords'] * 1e-6 network['throat.radius'] = network['throat.radius'] * 1e-6 network['throat.dmax'] = network['throat.dmax'] * 1e-6 network['throat.volume'] = network['throat.volume'] * voxel_size**3 return network.project
def load(cls, path, node_file="throats_cellsThroatsGraph_Nodes.txt", graph_file="throats_cellsThroatsGraph.txt", network=None, voxel_size=None, return_geometry=False): r""" Loads network data from an iMorph processed image stack Parameters ---------- path : string The path of the folder where the subfiles are held node_file : string The file that describes the pores and throats, the default iMorph name is: throats_cellsThroatsGraph_Nodes.txt graph_file : string The file that describes the connectivity of the network, the default iMorph name is: throats_cellsThroatsGraph.txt network : OpenPNM Network Object The OpenPNM Network onto which the data should be loaded. If no network is supplied then an empty import network is created and returned. voxel_size : float Allows the user to define a voxel size different than what is contained in the node_file. The value must be in meters. return_geometry : Boolean If True, then all geometrical related properties are removed from the Network object and added to a GenericGeometry object. In this case the method returns a tuple containing (network, geometry). If False (default) then the returned Network will contain all properties that were in the original file. In this case, the user can call the ```split_geometry``` method explicitly to perform the separation. Returns ------- If no Network object is supplied then one will be created and returned. If return_geometry is True, then a tuple is returned containing both the network and a geometry object. """ # path = Path(path) node_file = os.path.join(path.resolve(), node_file) graph_file = os.path.join(path.resolve(), graph_file) # parsing the nodes file with open(node_file, 'r') as file: Np = sp.fromstring(file.readline().rsplit('=')[1], sep='\t', dtype=int)[0] vox_size = sp.fromstring(file.readline().rsplit(')')[1], sep='\t',)[0] # network always recreated to prevent errors network = GenericNetwork(Np=Np, Nt=0) # Define expected properies network['pore.volume'] = sp.nan scrap_lines = [file.readline() for line in range(4)] while True: vals = file.readline().split('\t') if len(vals) == 1: break network['pore.volume'][int(vals[0])] = float(vals[3]) if 'pore.'+vals[2] not in network.labels(): network['pore.'+vals[2]] = False network['pore.'+vals[2]][int(vals[0])] = True if voxel_size is None: voxel_size = vox_size * 1.0E-6 # file stores value in microns if voxel_size < 0: raise(Exception('Error - Voxel size must be specfied in ' + 'the Nodes file or as a keyword argument.')) # parsing the graph file with open(graph_file, 'r') as file: # Define expected properties network['pore.coords'] = sp.zeros((Np, 3))*sp.nan network['pore.types'] = sp.nan network['pore.color'] = sp.nan network['pore.radius'] = sp.nan network['pore.dmax'] = sp.nan network['pore.node_number'] = sp.nan # Scan file to get pore coordinate data scrap_lines = [file.readline() for line in range(3)] line = file.readline() xmax = 0.0 ymax = 0.0 zmax = 0.0 node_num = 0 while line != 'connectivity table\n': vals = sp.fromstring(line, sep='\t') xmax = vals[1] if vals[1] > xmax else xmax ymax = vals[2] if vals[2] > ymax else ymax zmax = vals[3] if vals[3] > zmax else zmax network['pore.coords'][int(vals[0]), :] = vals[1:4] network['pore.types'][int(vals[0])] = vals[4] network['pore.color'][int(vals[0])] = vals[5] network['pore.radius'][int(vals[0])] = vals[6] network['pore.dmax'][int(vals[0])] = vals[7] network['pore.node_number'][int(vals[0])] = node_num node_num += 1 line = file.readline() # Scan file to get to connectivity data scrap_lines.append(file.readline()) # Skip line # Create sparse lil array incrementally build adjacency matrix lil = sp.sparse.lil_matrix((Np, Np), dtype=int) while True: vals = sp.fromstring(file.readline(), sep='\t', dtype=int) if len(vals) <= 1: break lil.rows[vals[0]] = vals[2:] lil.data[vals[0]] = sp.ones(vals[1]) # fixing any negative volumes or distances so they are 1 voxel/micron network['pore.volume'][sp.where(network['pore.volume'] < 0)[0]] = 1.0 network['pore.radius'][sp.where(network['pore.radius'] < 0)[0]] = 1.0 network['pore.dmax'][sp.where(network['pore.dmax'] < 0)[0]] = 1.0 # Add adjacency matrix to OpenPNM network conns = sp.sparse.triu(lil, k=1, format='coo') network.update({'throat.all': sp.ones(len(conns.col), dtype=bool)}) network['throat.conns'] = sp.vstack([conns.row, conns.col]).T network['pore.to_trim'] = False network['pore.to_trim'][network.pores('*throat')] = True Ts = network.pores('to_trim') new_conns = network.find_neighbor_pores(pores=Ts, flatten=False) extend(network=network, throat_conns=new_conns, labels='new_conns') for item in network.props('pore'): item = item.split('.')[1] arr = sp.ones_like(network['pore.'+item])[0] arr = sp.tile(A=arr, reps=[network.Nt, 1])*sp.nan network['throat.'+item] = sp.squeeze(arr) network['throat.'+item][network.throats('new_conns')] = \ network['pore.'+item][Ts] trim(network=network, pores=Ts) # setting up boundary pores x_coord, y_coord, z_coord = sp.hsplit(network['pore.coords'], 3) network['pore.front_boundary'] = sp.ravel(x_coord == 0) network['pore.back_boundary'] = sp.ravel(x_coord == xmax) network['pore.left_boundary'] = sp.ravel(y_coord == 0) network['pore.right_boundary'] = sp.ravel(y_coord == ymax) network['pore.bottom_boundary'] = sp.ravel(z_coord == 0) network['pore.top_boundary'] = sp.ravel(z_coord == zmax) # removing any pores that got classified as a boundary pore but # weren't labled a border_cell_face ps = sp.where(~sp.in1d(network.pores('*_boundary'), network.pores('border_cell_face')))[0] ps = network.pores('*_boundary')[ps] for side in ['front', 'back', 'left', 'right', 'top', 'bottom']: network['pore.'+side+'_boundary'][ps] = False # setting internal label network['pore.internal'] = False network['pore.internal'][network.pores('*_boundary', mode='not')] = True # adding props to border cell face throats and from pores Ts = sp.where(network['throat.conns'][:, 1] > network.pores('border_cell_face')[0] - 1)[0] faces = network['throat.conns'][Ts, 1] for item in network.props('pore'): item = item.split('.')[1] network['throat.'+item][Ts] = network['pore.'+item][faces] network['pore.volume'][faces] = 0.0 # applying unit conversions # TODO: Determine if radius and dmax are indeed microns and not voxels network['pore.coords'] = network['pore.coords'] * 1e-6 network['pore.radius'] = network['pore.radius'] * 1e-6 network['pore.dmax'] = network['pore.dmax'] * 1e-6 network['pore.volume'] = network['pore.volume'] * voxel_size**3 network['throat.coords'] = network['throat.coords'] * 1e-6 network['throat.radius'] = network['throat.radius'] * 1e-6 network['throat.dmax'] = network['throat.dmax'] * 1e-6 network['throat.volume'] = network['throat.volume'] * voxel_size**3 return network.project
def generate_node_connectivity_array(index_map, data_array): r""" Generates a node connectivity array based on faces, edges and corner adjacency """ # logger.info('generating network connections...') # # setting up some constants x_dim, y_dim, z_dim = data_array.shape conn_map = list(product([0, -1, 1], [0, -1, 1], [0, -1, 1])) # conn_map = sp.array(conn_map, dtype=int) conn_map = conn_map[1:] # # creating slice list to process data chunks slice_list = [slice(0, 10000)] for i in range(slice_list[0].stop, index_map.shape[0], slice_list[0].stop): slice_list.append(slice(i, i + slice_list[0].stop)) slice_list[-1] = slice(slice_list[-1].start, index_map.shape[0]) # conns = sp.ones((0, 2), dtype=data_array.index_int_type) logger.debug('\tnumber of slices to process: {}'.format(len(slice_list))) percent = 10 for n, sect in enumerate(slice_list): # getting coordinates of nodes and their neighbors nodes = index_map[sect] inds = sp.repeat(nodes, conn_map.shape[0], axis=0) inds += sp.tile(conn_map, (nodes.shape[0], 1)) # # calculating the flattened index of the central nodes and storing nodes = sp.ravel_multi_index(sp.hsplit(nodes, 3), data_array.shape) inds = sp.hstack([inds, sp.repeat(nodes, conn_map.shape[0], axis=0)]) # # removing neigbors with negative indicies mask = ~inds[:, 0:3] < 0 inds = inds[sp.sum(mask, axis=1) == 3] # removing neighbors with indicies outside of bounds mask = (inds[:, 0] < x_dim, inds[:, 1] < y_dim, inds[:, 2] < z_dim) mask = sp.stack(mask, axis=1) inds = inds[sp.sum(mask, axis=1) == 3] # removing indices with zero-weight connection mask = data_array[inds[:, 0], inds[:, 1], inds[:, 2]] inds = inds[mask] if inds.size: # calculating flattened index of remaining nieghbor nodes nodes = sp.ravel_multi_index(sp.hsplit(inds[:, 0:3], 3), data_array.shape) inds = sp.hstack([sp.reshape(inds[:, -1], (-1, 1)), nodes]) # ensuring conns[0] is always < conns[1] for duplicate removal mask = inds[:, 0] > inds[:, 1] inds[mask] = inds[mask][:, ::-1] # appending section connectivity data to conns array conns = sp.append(conns, inds.astype(sp.uint32), axis=0) if int(n / len(slice_list) * 100) == percent: logger.debug('\tprocessed slice {:5d}, {}% complete'.format( n, percent)) percent += 10 # # using scipy magic from stackoverflow to remove dupilcate connections logger.info('removing duplicate connections...') dim0 = conns.shape[0] conns = sp.ascontiguousarray(conns) dtype = sp.dtype((sp.void, conns.dtype.itemsize * conns.shape[1])) dim1 = conns.shape[1] conns = sp.unique(conns.view(dtype)).view(conns.dtype).reshape(-1, dim1) logger.debug('\tremoved {} duplicates'.format(dim0 - conns.shape[0])) # return conns
def generate_node_connectivity_array(index_map, data_array): r""" Generates a node connectivity array based on faces, edges and corner adjacency """ # logger.info('generating network connections...') # # setting up some constants x_dim, y_dim, z_dim = data_array.shape conn_map = list(product([0, -1, 1], [0, -1, 1], [0, -1, 1])) conn_map = sp.array(conn_map, dtype=int) conn_map = conn_map[1:] # # creating slice list to process data chunks slice_list = [slice(0, 10000)] for i in range(slice_list[0].stop, index_map.shape[0], slice_list[0].stop): slice_list.append(slice(i, i+slice_list[0].stop)) slice_list[-1] = slice(slice_list[-1].start, index_map.shape[0]) # conns = sp.ones((0, 2), dtype=sp.uint32) logger.debug(' number of slices to process: {}'.format(len(slice_list))) for sect in slice_list: # getting coordinates of nodes and their neighbors nodes = index_map[sect] inds = sp.repeat(nodes, conn_map.shape[0], axis=0) inds += sp.tile(conn_map, (nodes.shape[0], 1)) # # calculating the flattened index of the central nodes and storing nodes = sp.ravel_multi_index(sp.hsplit(nodes, 3), data_array.shape) inds = sp.hstack([inds, sp.repeat(nodes, conn_map.shape[0], axis=0)]) # # removing neigbors with negative indicies mask = ~inds[:, 0:3] < 0 inds = inds[sp.sum(mask, axis=1) == 3] # removing neighbors with indicies outside of bounds mask = (inds[:, 0] < x_dim, inds[:, 1] < y_dim, inds[:, 2] < z_dim) mask = sp.stack(mask, axis=1) inds = inds[sp.sum(mask, axis=1) == 3] # removing indices with zero-weight connection mask = data_array[inds[:, 0], inds[:, 1], inds[:, 2]] inds = inds[mask] if inds.size: # calculating flattened index of remaining nieghbor nodes nodes = sp.ravel_multi_index(sp.hsplit(inds[:, 0:3], 3), data_array.shape) inds = sp.hstack([sp.reshape(inds[:, -1], (-1, 1)), nodes]) # ensuring conns[0] is always < conns[1] for duplicate removal mask = inds[:, 0] > inds[:, 1] inds[mask] = inds[mask][:, ::-1] # appending section connectivity data to conns array conns = sp.append(conns, inds.astype(sp.uint32), axis=0) # # using scipy magic from stackoverflow to remove dupilcate connections logger.info('removing duplicate connections...') dim0 = conns.shape[0] conns = sp.ascontiguousarray(conns) dtype = sp.dtype((sp.void, conns.dtype.itemsize*conns.shape[1])) dim1 = conns.shape[1] conns = sp.unique(conns.view(dtype)).view(conns.dtype).reshape(-1, dim1) logger.debug(' removed {} duplicates'.format(dim0 - conns.shape[0])) # return conns