def main_loop(init_param, X, K, iter=1000, tol=1e-6): """ Gaussian Mixture Model Arguments: - `X`: Input data (2D array, [[x11, x12, ..., x1D], ..., [xN1, ... xND]]). - `K`: Number of clusters. - `iter`: Number of iterations to run. - `tol`: Tolerance. """ X = sp.asarray(X) N, D = X.shape pi = sp.asarray(init_param["coff"]) mu = sp.asarray(init_param["mean"]) sigma = sp.asarray(init_param["cov"]) L = sp.inf for i in xrange(iter): # E-step gamma = sp.apply_along_axis( lambda x: sp.fromiter( (pi[k] * gauss_mixture_calculate(x, mu[k], sigma[k]) for k in xrange(K)), dtype=float ), 1, X, ) gamma /= sp.sum(gamma, 1)[:, sp.newaxis] # M-step Nk = sp.sum(gamma, 0) mu = sp.sum(X * gamma.T[..., sp.newaxis], 1) / Nk[..., sp.newaxis] xmu = X[:, sp.newaxis, :] - mu sigma = ( sp.sum(gamma[..., sp.newaxis, sp.newaxis] * xmu[:, :, sp.newaxis, :] * xmu[:, :, :, sp.newaxis], 0) / Nk[..., sp.newaxis, sp.newaxis] ) pi = Nk / N # Likelihood Lnew = sp.sum( sp.log2( sp.sum( sp.apply_along_axis( lambda x: sp.fromiter( (pi[k] * gauss_mixture_calculate(x, mu[k], sigma[k]) for k in xrange(K)), dtype=float ), 1, X, ), 1, ) ) ) if abs(L - Lnew) < tol: break L = Lnew print "log likelihood=%s" % L return dict(pi=pi, mu=mu, sigma=sigma, gamma=gamma)
def main_loop(init_param, X, K, iter=1000, tol=1e-6): """ Gaussian Mixture Model Arguments: - `X`: Input data (2D array, [[x11, x12, ..., x1D], ..., [xN1, ... xND]]). - `K`: Number of clusters. - `iter`: Number of iterations to run. - `tol`: Tolerance. """ X = sp.asarray(X) N, D = X.shape pi = sp.asarray(init_param['coff']) mu = sp.asarray(init_param['mean']) sigma = sp.asarray(init_param['cov']) L = sp.inf for i in xrange(iter): # E-step gamma = sp.apply_along_axis( lambda x: sp.fromiter( (pi[k] * gauss_mixture_calculate(x, mu[k], sigma[k]) for k in xrange(K)), dtype=float), 1, X) gamma /= sp.sum(gamma, 1)[:, sp.newaxis] # M-step Nk = sp.sum(gamma, 0) mu = sp.sum(X * gamma.T[..., sp.newaxis], 1) / Nk[..., sp.newaxis] xmu = X[:, sp.newaxis, :] - mu sigma = sp.sum( gamma[..., sp.newaxis, sp.newaxis] * xmu[:, :, sp.newaxis, :] * xmu[:, :, :, sp.newaxis], 0) / Nk[..., sp.newaxis, sp.newaxis] pi = Nk / N # Likelihood Lnew = sp.sum( sp.log2( sp.sum( sp.apply_along_axis( lambda x: sp.fromiter((pi[k] * gauss_mixture_calculate( x, mu[k], sigma[k]) for k in xrange(K)), dtype=float), 1, X), 1))) if abs(L - Lnew) < tol: break L = Lnew print "log likelihood=%s" % L return dict(pi=pi, mu=mu, sigma=sigma, gamma=gamma)
def print_all_stats(ctx, series): ftime = get_ftime(series) start = 0 end = ctx.interval print('start-time, samples, min, avg, median, 90%, 95%, 99%, max') while (start < ftime): # for each time interval end = ftime if ftime < end else end sample_arrays = [ s.get_samples(start, end) for s in series ] samplevalue_arrays = [] for sample_array in sample_arrays: samplevalue_arrays.append( [ sample.value for sample in sample_array ] ) #print('samplevalue_arrays len: %d' % len(samplevalue_arrays)) #print('samplevalue_arrays elements len: ' + \ #str(map( lambda l: len(l), samplevalue_arrays))) # collapse list of lists of sample values into list of sample values samplevalues = reduce( array_collapser, samplevalue_arrays, [] ) #print('samplevalues: ' + str(sorted(samplevalues))) # compute all stats and print them myarray = scipy.fromiter(samplevalues, float) mymin = scipy.amin(myarray) myavg = scipy.average(myarray) mymedian = scipy.median(myarray) my90th = scipy.percentile(myarray, 90) my95th = scipy.percentile(myarray, 95) my99th = scipy.percentile(myarray, 99) mymax = scipy.amax(myarray) print( '%f, %d, %f, %f, %f, %f, %f, %f, %f' % ( start, len(samplevalues), mymin, myavg, mymedian, my90th, my95th, my99th, mymax)) # advance to next interval start += ctx.interval end += ctx.interval
def __repr__(self): """Returns a representation of the MatrixGraph object. Returns ------- str The string representation of the object to be printed by the Python interpreter. """ max_vertex_length = sp.fromiter((len(str(vertex)) for vertex in self.vertices), int, len(self)).max() max_value_length = len(str(self.__graph.max())) max_length = max([max_vertex_length, max_value_length]) representation = "<MatrixGraph object>\n" representation += ((max_length + 2) * " " + " ".join([ str(x).ljust(max_length + 1, " ") for x in self.vertices_list.keys() ]) + "\n\n") for label, index in self.label_to_index_mapping: representation += (str(label).ljust(max_length + 2, " ") + " ".join([ str(n).ljust(max_length + 1, " ") for n in self.__graph[int(index)] ]) + "\n") return representation[:-1]
def __repr__(self): """Returns a representation of the ListGraph object. Returns ------- str The string representation of the object to be printed by the Python interpreter. """ max_length = sp.fromiter( (len(str(vertex)) for vertex in self.vertices), int, len(self)).max() representation = "<ListGraph object>\n" for vertex in self.vertices: if self.is_pondered: adjacency = [ f"{destination}: {weight}" for destination, weight in self[vertex].items() ] else: adjacency = [ f"{destination}" for destination in self[vertex].keys() ] representation += (str(vertex).ljust(max_length, " ") + " -> " + ", ".join(adjacency) + "\n") return representation
def find_neighbor_throats(self, pores, mode='union', flatten=True): r""" Returns a list of throats neighboring the given pore(s) Parameters ---------- pores : array_like Indices of pores whose neighbors are sought flatten : boolean, optional If flatten is True (default) a 1D array of unique throat ID numbers is returned. If flatten is False the returned array contains arrays of neighboring throat ID numbers for each input pore, in the order they were sent. mode : string, optional Specifies which neighbors should be returned. The options are: **'union'** : All neighbors of the input pores **'intersection'** : Only neighbors shared by all input pores **'not_intersection'** : Only neighbors not shared by any input pores Returns ------- neighborTs : 1D array (if flatten is True) or ndarray of arrays (if flatten if False) Examples -------- >>> import OpenPNM >>> pn = OpenPNM.Network.TestNet() >>> pn.find_neighbor_throats(pores=[0, 1]) array([0, 1, 2, 3, 4, 5]) >>> pn.find_neighbor_throats(pores=[0, 1],flatten=False) array([array([0, 1, 2]), array([0, 3, 4, 5])], dtype=object) """ pores = self._parse_locations(pores) if sp.size(pores) == 0: return sp.array([], ndmin=1, dtype=int) # Test for existence of incidence matrix try: neighborTs = self._incidence_matrix['lil'].rows[[pores]] except: temp = self.create_incidence_matrix(sprsfmt='lil') self._incidence_matrix['lil'] = temp neighborTs = self._incidence_matrix['lil'].rows[[pores]] if flatten: # Convert rows of lil into single flat list neighborTs = itertools.chain.from_iterable(neighborTs) # Convert list to numpy array neighborTs = sp.fromiter(neighborTs, dtype=int) if mode == 'not_intersection': neighborTs = sp.unique(sp.where(sp.bincount(neighborTs) == 1)[0]) elif mode == 'union': neighborTs = sp.unique(neighborTs) elif mode == 'intersection': neighborTs = sp.unique(sp.where(sp.bincount(neighborTs) > 1)[0]) return sp.array(neighborTs, ndmin=1, dtype=int) else: # Convert lists in array to numpy arrays neighborTs = [sp.array(neighborTs[i]) for i in range(0, len(pores))] return sp.array(neighborTs, ndmin=1)
def find_neighbor_pores(self, pores, mode='union', flatten=True, excl_self=True): r""" Returns a list of pores neighboring the given pore(s) Parameters ---------- pores : array_like ID numbers of pores whose neighbors are sought. flatten : boolean, optional If flatten is True a 1D array of unique pore ID numbers is returned. If flatten is False the returned array contains arrays of neighboring pores for each input pore, in the order they were sent. excl_self : bool, optional (Default is False) If this is True then the input pores are not included in the returned list. This option only applies when input pores are in fact neighbors to each other, otherwise they are not part of the returned list anyway. mode : string, optional Specifies which neighbors should be returned. The options are: **'union'** : All neighbors of the input pores **'intersection'** : Only neighbors shared by all input pores **'not_intersection'** : Only neighbors not shared by any input pores Returns ------- neighborPs : 1D array (if flatten is True) or ndarray of ndarrays (if flatten if False) Examples -------- >>> import OpenPNM >>> pn = OpenPNM.Network.TestNet() >>> pn.find_neighbor_pores(pores=[0, 2]) array([ 1, 3, 5, 7, 25, 27]) >>> pn.find_neighbor_pores(pores=[0, 1]) array([ 2, 5, 6, 25, 26]) >>> pn.find_neighbor_pores(pores=[0, 1], mode='union', excl_self=False) array([ 0, 1, 2, 5, 6, 25, 26]) >>> pn.find_neighbor_pores(pores=[0, 2], flatten=False) array([array([ 1, 5, 25]), array([ 1, 3, 7, 27])], dtype=object) >>> pn.find_neighbor_pores(pores=[0, 2], mode='intersection') array([1]) >>> pn.find_neighbor_pores(pores=[0, 2], mode='not_intersection') array([ 3, 5, 7, 25, 27]) """ pores = self._parse_locations(pores) allowed_modes = ['union', 'intersection', 'not_intersection'] mode = self._parse_mode(mode, allowed=allowed_modes, single=True) if sp.size(pores) == 0: return sp.array([], ndmin=1, dtype=int) # Test for existence of incidence matrix try: neighborPs = self._adjacency_matrix['lil'].rows[[pores]] except: temp = self.create_adjacency_matrix(sprsfmt='lil') self._adjacency_matrix['lil'] = temp neighborPs = self._adjacency_matrix['lil'].rows[[pores]] if flatten: # Convert rows of lil into single flat list neighborPs = itertools.chain.from_iterable(neighborPs) # Add input pores to list neighborPs = itertools.chain.from_iterable([neighborPs, pores]) # Convert list to numpy array neighborPs = sp.fromiter(neighborPs, dtype=int) # Apply logic to include/exclude items of the set if mode == 'not_intersection': temp = sp.where(sp.bincount(neighborPs) == 1)[0] neighborPs = sp.unique(temp) elif mode == 'union': neighborPs = sp.unique(neighborPs) elif mode == 'intersection': temp = sp.where(sp.bincount(neighborPs) > 1)[0] neighborPs = sp.unique(temp) if excl_self: neighborPs = neighborPs[~sp.in1d(neighborPs, pores)] return sp.array(neighborPs, ndmin=1, dtype=int) else: # Convert lists in array to numpy arrays neighborPs = [sp.array(neighborPs[i]) for i in range(0, len(pores))] return sp.array(neighborPs, ndmin=1)
def _find_neighbors(self, pores, element, mode, flatten, excl_self): r""" Private method for finding the neighboring pores or throats connected directly to given set of pores. Parameters ---------- pores : array_like The list of pores whose neighbors are sought element : string, either 'pore' or 'throat' Whether to find neighboring pores or throats mode : string Controls how the neighbors are filtered. Options are: **'union'** : All neighbors of the input pores **'intersection'** : Only neighbors shared by all input pores **'not_intersection'** : Only neighbors not shared by any input pores flatten : boolean If flatten is True (default) a 1D array of unique neighbors is returned. If flatten is False the returned array contains arrays of neighboring throat ID numbers for each input pore, in the order they were sent. excl_self : bool When True the input pores are not included in the returned list of neighboring pores. This option only applies when input pores are in fact neighbors to each other, otherwise they are not part of the returned list anyway. This is ignored with the element is 'throats'. See Also -------- find_neighbor_pores find_neighbor_throats num_neighors """ element = self._parse_element(element=element, single=True) pores = self._parse_locations(pores) if sp.size(pores) == 0: return sp.array([], ndmin=1, dtype=int) # Test for existence of incidence or adjacency matrix if element == 'pore': try: neighbors = self._adjacency_matrix['lil'].rows[[pores]] except: temp = self.create_adjacency_matrix(sprsfmt='lil') self._adjacency_matrix['lil'] = temp neighbors = self._adjacency_matrix['lil'].rows[[pores]] elif element == 'throat': try: neighbors = self._incidence_matrix['lil'].rows[[pores]] except: temp = self.create_incidence_matrix(sprsfmt='lil') self._incidence_matrix['lil'] = temp neighbors = self._incidence_matrix['lil'].rows[[pores]] if flatten: # Convert rows of lil into single flat list neighbors = itertools.chain.from_iterable(neighbors) if element == 'pore': # Add input pores to list neighbors = itertools.chain.from_iterable([neighbors, pores]) # Convert list to numpy array neighbors = sp.fromiter(neighbors, dtype=int) if mode == 'not_intersection': neighbors = sp.unique(sp.where(sp.bincount(neighbors) == 1)[0]) elif mode == 'union': neighbors = sp.unique(neighbors) elif mode == 'intersection': neighbors = sp.unique(sp.where(sp.bincount(neighbors) > 1)[0]) if excl_self and element == 'pore': # Remove input pores from list neighbors = neighbors[~sp.in1d(neighbors, pores)] return sp.array(neighbors, ndmin=1, dtype=int) else: # Convert lists in array to numpy arrays neighbors = [sp.array(neighbors[i]) for i in range(0, len(pores))] return sp.array(neighbors, ndmin=1)
def wavefunction(r): iterable = (basis_function(r, k, w) for (k, w) in zip(*contour)) basis_vec = sp.fromiter(iterable, complex, count=length) return sp.dot(basis_vec, eigvec)
def wavefunction(r): iterable = (basis_function(r, n) for n in range(length)) basis_vec = sp.fromiter(iterable, complex, count=length) return sp.dot(basis_vec, eigvec)
def find_neighbor_throats(self, pores, mode='union', flatten=True): r""" Returns a list of throats neighboring the given pore(s) Parameters ---------- pores : array_like Indices of pores whose neighbors are sought flatten : boolean, optional If flatten is True (default) a 1D array of unique throat ID numbers is returned. If flatten is False the returned array contains arrays of neighboring throat ID numbers for each input pore, in the order they were sent. mode : string, optional Specifies which neighbors should be returned. The options are: **'union'** : All neighbors of the input pores **'intersection'** : Only neighbors shared by all input pores **'not_intersection'** : Only neighbors not shared by any input pores Returns ------- neighborTs : 1D array (if flatten is True) or ndarray of arrays (if flatten if False) Examples -------- >>> import OpenPNM >>> pn = OpenPNM.Network.TestNet() >>> pn.find_neighbor_throats(pores=[0, 1]) array([0, 1, 2, 3, 4, 5]) >>> pn.find_neighbor_throats(pores=[0, 1],flatten=False) array([array([0, 1, 2]), array([0, 3, 4, 5])], dtype=object) """ pores = self._parse_locations(pores) if sp.size(pores) == 0: return sp.array([], ndmin=1, dtype=int) # Test for existence of incidence matrix try: neighborTs = self._incidence_matrix['lil'].rows[[pores]] except: temp = self.create_incidence_matrix(sprsfmt='lil') self._incidence_matrix['lil'] = temp neighborTs = self._incidence_matrix['lil'].rows[[pores]] if flatten: # Convert rows of lil into single flat list neighborTs = itertools.chain.from_iterable(neighborTs) # Convert list to numpy array neighborTs = sp.fromiter(neighborTs, dtype=int) if mode == 'not_intersection': neighborTs = sp.unique( sp.where(sp.bincount(neighborTs) == 1)[0]) elif mode == 'union': neighborTs = sp.unique(neighborTs) elif mode == 'intersection': neighborTs = sp.unique( sp.where(sp.bincount(neighborTs) > 1)[0]) return sp.array(neighborTs, ndmin=1, dtype=int) else: # Convert lists in array to numpy arrays neighborTs = [ sp.array(neighborTs[i]) for i in range(0, len(pores)) ] return sp.array(neighborTs, ndmin=1)
def find_neighbor_pores(self, pores, mode='union', flatten=True, excl_self=True): r""" Returns a list of pores neighboring the given pore(s) Parameters ---------- pores : array_like ID numbers of pores whose neighbors are sought. flatten : boolean, optional If flatten is True a 1D array of unique pore ID numbers is returned. If flatten is False the returned array contains arrays of neighboring pores for each input pore, in the order they were sent. excl_self : bool, optional (Default is False) If this is True then the input pores are not included in the returned list. This option only applies when input pores are in fact neighbors to each other, otherwise they are not part of the returned list anyway. mode : string, optional Specifies which neighbors should be returned. The options are: **'union'** : All neighbors of the input pores **'intersection'** : Only neighbors shared by all input pores **'not_intersection'** : Only neighbors not shared by any input pores Returns ------- neighborPs : 1D array (if flatten is True) or ndarray of ndarrays (if flatten if False) Examples -------- >>> import OpenPNM >>> pn = OpenPNM.Network.TestNet() >>> pn.find_neighbor_pores(pores=[0, 2]) array([ 1, 3, 5, 7, 25, 27]) >>> pn.find_neighbor_pores(pores=[0, 1]) array([ 2, 5, 6, 25, 26]) >>> pn.find_neighbor_pores(pores=[0, 1], mode='union', excl_self=False) array([ 0, 1, 2, 5, 6, 25, 26]) >>> pn.find_neighbor_pores(pores=[0, 2], flatten=False) array([array([ 1, 5, 25]), array([ 1, 3, 7, 27])], dtype=object) >>> pn.find_neighbor_pores(pores=[0, 2], mode='intersection') array([1]) >>> pn.find_neighbor_pores(pores=[0, 2], mode='not_intersection') array([ 3, 5, 7, 25, 27]) """ pores = self._parse_locations(pores) allowed_modes = ['union', 'intersection', 'not_intersection'] mode = self._parse_mode(mode, allowed=allowed_modes, single=True) if sp.size(pores) == 0: return sp.array([], ndmin=1, dtype=int) # Test for existence of incidence matrix try: neighborPs = self._adjacency_matrix['lil'].rows[[pores]] except: temp = self.create_adjacency_matrix(sprsfmt='lil') self._adjacency_matrix['lil'] = temp neighborPs = self._adjacency_matrix['lil'].rows[[pores]] if flatten: # Convert rows of lil into single flat list neighborPs = itertools.chain.from_iterable(neighborPs) # Add input pores to list neighborPs = itertools.chain.from_iterable([neighborPs, pores]) # Convert list to numpy array neighborPs = sp.fromiter(neighborPs, dtype=int) # Apply logic to include/exclude items of the set if mode == 'not_intersection': temp = sp.where(sp.bincount(neighborPs) == 1)[0] neighborPs = sp.unique(temp) elif mode == 'union': neighborPs = sp.unique(neighborPs) elif mode == 'intersection': temp = sp.where(sp.bincount(neighborPs) > 1)[0] neighborPs = sp.unique(temp) if excl_self: neighborPs = neighborPs[~sp.in1d(neighborPs, pores)] return sp.array(neighborPs, ndmin=1, dtype=int) else: # Convert lists in array to numpy arrays neighborPs = [ sp.array(neighborPs[i]) for i in range(0, len(pores)) ] return sp.array(neighborPs, ndmin=1)
def get_source_info(*, pixel_array, stddev_array, mask_array, source_positions, aperture, bg_radii): """ Return field array containing source positions, fluxes and backgrounds. Args: pixel_array (2-D array like): The measured values of the image pixels. stddev_array (2-D array like): The estimated variance of the image pixels. mask_array (2-D array like): Quality flags for the image pixels. source_positions: The return value from get_source_positions(). aperture: The size of the aperture to use for measuring the flux. bg_radii((float, float)): The inner and outer radius to use for the background annulus. Returns: (scipy field array): All relevant source information in the following fields: - x: The x coordinates of the sources. - y: The y coordinates of the sources. - flux: The measured fluxes of the sources. - flux_err: Estimated error of the flux. - bg: The measured backgrounds of the sources. - bg_err: Estimated error of the background. - bg_npix: The number of pixels used in determining the background. """ def add_flux_info(result, measure_background): """Measure the flux of the sources and add to result.""" fit_star_shape = FitStarShape( mode='PSF', shape_terms='{1}', grid=[[-1.1 * aperture, 1.1 * aperture], [-1.1 * aperture, 1.1 * aperture]], initial_aperture=aperture, bg_min_pix=0, src_min_pix=0, src_min_signal_to_noise=0.0, src_max_aperture=1000.0 ) result_tree = fit_star_shape.fit( [ ( pixel_array, stddev_array, mask_array, result ) ], [measure_background] ) get_flux = SubPixPhot(apertures=[aperture]) get_flux( (pixel_array, stddev_array, mask_array), result_tree, ) result['flux'] = flux_from_magnitude( result_tree.get(quantity='apphot.mag.0.0', shape=result.shape), get_flux.configuration['magnitude_1adu'] ) result = scipy.empty(len(source_positions), dtype=[('ID', 'S5'), ('x', scipy.float64), ('y', scipy.float64), ('flux', scipy.float64), ('bg', scipy.float64), ('bg_err', scipy.float64), ('bg_npix', scipy.uint64), ('enabled', scipy.float64)]) result['enabled'][:] = True src_x = scipy.fromiter((pos[0] for pos in source_positions), dtype=c_double) src_y = scipy.fromiter((pos[1] for pos in source_positions), dtype=c_double) for int_id in range(result.size): result['ID'][int_id] = '%5.5d' % int_id result['x'] = src_x result['y'] = src_y measure_background = BackgroundExtractor(pixel_array, *bg_radii) result['bg'], result['bg_err'], result['bg_npix'] = measure_background( src_x, src_y ) add_flux_info(result, measure_background) return result