def index_select(inp, dim, index) -> 'Tensor': _check_tensors(inp) engine = _get_engine(inp) return _create_tensor( inp, data=engine.take_along_axis(inp.data, index.data.astype('int'), dim), func=wrapped_partial(index_select_backward, inp=inp, index=index, dim=dim) )
def extract_top_n_words_per_topic(self, tf_idf, vectorizer, labels, n=30): """Based on tf_idf scores per topic, extract the top n words per topic Arguments: tf_idf: A c-TF-IDF matrix from which to calculate the top words vectorizer: object of class CountVecWrapper, which is derived from CountVectorizer labels: A list of unique topic labels n: number of words per topic (Default: 30) Returns: top_n_words: The top n words per topic topic_names: Dictionary containing key as Topic ID and value as top 4 words from that topic cluster """ words = vectorizer.get_feature_names().to_arrow().to_pylist() labels = sorted(labels.to_arrow().to_pylist()) indices, scores = top_n_sparse(tf_idf, n) sorted_indices = cp.argsort(scores, 1) indices = cp.take_along_axis(indices, sorted_indices, axis=1).get() scores = cp.take_along_axis(scores, sorted_indices, axis=1).get() # Get top 30 words per topic based on c-TF-IDF score topics = { label: [ (words[word_index], score) if word_index and score > 0 else ("", 0.00001) for word_index, score in zip(indices[index][::-1], scores[index][::-1]) ] for index, label in enumerate(labels) } topic_names = { key: f"{key}_" + "_".join([word[0] for word in values[:4]]) for key, values in topics.items() } return topics, topic_names
def GetSASE_gpu(CentralEnergy, dE_FWHM, dt_FWHM, samples=0, onlyT=False): from cupy import interp import cupy as cp h = 4.135667662 #in eV*fs dE = dE_FWHM / 2.355 #in eV, converts to sigma dt = dt_FWHM / 2.355 #in fs, converts to sigma if samples == 0: samples = int(600. * dt * CentralEnergy / h) else: if (samples < 400. * dt * CentralEnergy / h): print( "Number of samples is a little small, proceeding anyway. Got", samples, "prefer more than", 400. * dt * CentralEnergy / h) EnAxis = cp.linspace(0., 20. * CentralEnergy, num=samples, dtype=cp.float32) newEaxis = cp.linspace(0, 140, 32 * 1024) # EnInput=cp.zeros(samples, dtype=cp.complex64) # for i in range(samples): EnInput = cp.exp(-(EnAxis - CentralEnergy)**2 / 2. / dE**2 + 2 * cp.pi * 1j * cp.random.random(size=samples), dtype=cp.complex64) EnInput = interp(newEaxis, EnAxis, EnInput) newTaxis = cp.fft.fftfreq(32 * 1024, d=140 / (32 * 1024)) * h TOutput = cp.exp(-newTaxis**2 / 2. / dt**2) * cp.fft.fft(EnInput) # sort TOutput and newTaxis ind = cp.argsort(newTaxis, axis=0) newTaxis = cp.sort(newTaxis) TOutput = cp.take_along_axis(TOutput, ind, axis=0) # En_FFT=cp.fft.fft(EnInput) # TAxis=cp.fft.fftfreq(samples,d=(20.*CentralEnergy)/samples)*h # TOutput=cp.exp(-TAxis**2/2./dt**2)*En_FFT if not (onlyT): EnOutput = cp.fft.ifft(TOutput) if (onlyT): return newTaxis, TOutput else: return newEaxis, EnOutput, newTaxis, TOutput
def main(vectors, gpu_id=-1, n_neighbors=100, batch_size=1024, cutoff=0, start=0, end=None): """ Step 6: Precompute nearest-neighbor queries (optional) Precompute nearest-neighbor queries for every entry in the vocab to make Sense2Vec.most_similar faster. The --cutoff option lets you define the number of earliest rows to limit the neighbors to. For instance, if cutoff is 100000, no word will have a nearest neighbor outside of the top 100k vectors. """ if gpu_id == -1: xp = numpy else: import cupy as xp import cupy.cuda.device cupy.take_along_axis = take_along_axis device = cupy.cuda.device.Device(gpu_id) device.use() vectors_dir = Path(vectors) vectors_file = vectors_dir / "vectors" if not vectors_dir.is_dir() or not vectors_file.exists(): err = "Are you passing in the exported sense2vec directory containing a vectors file?" msg.fail(f"Can't load vectors from {vectors}", err, exits=1) with msg.loading(f"Loading vectors from {vectors}"): vectors = xp.load(str(vectors_file)) msg.good( f"Loaded {vectors.shape[0]:,} vectors with dimension {vectors.shape[1]}" ) norms = xp.linalg.norm(vectors, axis=1, keepdims=True) norms[norms == 0] = 1 # Normalize to unit norm vectors /= norms if cutoff < 1: cutoff = vectors.shape[0] if end is None: end = vectors.shape[0] mean = float(norms.mean()) var = float(norms.var()) msg.good(f"Normalized (mean {mean:,.2f}, variance {var:,.2f})") msg.info( f"Finding {n_neighbors:,} neighbors among {cutoff:,} most frequent") n = min(n_neighbors, vectors.shape[0]) subset = vectors[:cutoff] best_rows = xp.zeros((end - start, n), dtype="i") scores = xp.zeros((end - start, n), dtype="f") for i in tqdm.tqdm(list(range(start, end, batch_size))): size = min(batch_size, end - i) batch = vectors[i:i + size] sims = xp.dot(batch, subset.T) # Set self-similarities to -inf, so that we don't return them. for j in range(size): if i + j < sims.shape[1]: sims[j, i + j] = -xp.inf # This used to use argpartition, to do a partial sort...But this ended # up being a ratsnest of terrible numpy crap. Just sorting the whole # list isn't really slower, and it's much simpler to read. ranks = xp.argsort(sims, axis=1) batch_rows = ranks[:, -n:] # Reverse batch_rows = batch_rows[:, ::-1] batch_scores = xp.take_along_axis(sims, batch_rows, axis=1) best_rows[i:i + size] = batch_rows scores[i:i + size] = batch_scores msg.info("Saving output") if not isinstance(best_rows, numpy.ndarray): best_rows = best_rows.get() if not isinstance(scores, numpy.ndarray): scores = scores.get() output = { "indices": best_rows, "scores": scores.astype("float16"), "start": start, "end": end, "cutoff": cutoff, } output_file = vectors_dir / "cache" with msg.loading("Saving output..."): srsly.write_msgpack(output_file, output) msg.good(f"Saved cache to {output_file}")
print (graph_build_endtime - graph_build_starttime) louvain_starttime = datetime.datetime.now() result, mod = cg.louvain(G) vertex = result['vertex'] partition = result['partition'] size = result['partition'].max() + 1 print('community', size) print('modularity', mod) vertex = cp.fromDlpack(vertex.to_dlpack()) partition = cp.fromDlpack(partition.to_dlpack()) vertex = cp.reshape(vertex, XYZ_C.shape[0]) labelRE = cp.reshape(partition, XYZ_C.shape[0]) index = cp.argsort(vertex) vertex = cp.take_along_axis(vertex, index, axis=0) labelRE = cp.take_along_axis(labelRE, index, axis=0) print(result) print(vertex) print(labelRE) print(index) louvain_endtime = datetime.datetime.now() print(louvain_endtime - louvain_starttime) print(labelRE) labelRE = cp.asfortranarray(labelRE) labelRE = cd.from_dlpack(labelRE.toDlpack()) #labelRE = cd.DataFrame(labelRE) print(labelRE) df = cd.DataFrame({'label': labelRE}) df.to_csv('HW_AI/HW_Final/gpu.csv')
def _clahe(image, kernel_size, clip_limit, nbins): """Contrast Limited Adaptive Histogram Equalization. Parameters ---------- image : (N1,...,NN) ndarray Input image. kernel_size: int or N-tuple of int Defines the shape of contextual regions used in the algorithm. clip_limit : float Normalized clipping limit between 0 and 1 (higher values give more contrast). nbins : int Number of gray bins for histogram ("data range"). Returns ------- out : (N1,...,NN) ndarray Equalized image. The number of "effective" graylevels in the output image is set by `nbins`; selecting a small value (e.g. 128) speeds up processing and still produces an output image of good quality. A clip limit of 0 or larger than or equal to 1 results in standard (non-contrast limited) AHE. """ ndim = image.ndim dtype = image.dtype # pad the image such that the shape in each dimension # - is a multiple of the kernel_size and # - is preceded by half a kernel size pad_start_per_dim = [k // 2 for k in kernel_size] pad_end_per_dim = [ (k - s % k) % k + math.ceil(k / 2.0) for k, s in zip(kernel_size, image.shape) ] image = cp.pad( image, [(p_i, p_f) for p_i, p_f in zip(pad_start_per_dim, pad_end_per_dim)], mode="reflect", ) bin_size = 1 + NR_OF_GRAY // nbins if True: lut = cp.arange(NR_OF_GRAY) lut //= bin_size image = lut[image] else: lut = np.arange(NR_OF_GRAY) lut //= bin_size image = cp.asarray(lut[image.get()]) # calculate graylevel mappings for each contextual region # rearrange image into flattened contextual regions ns_hist = [int(s / k) - 1 for s, k in zip(image.shape, kernel_size)] hist_blocks_shape = functools.reduce( operator.add, [(s, k) for s, k in zip(ns_hist, kernel_size)] ) hist_blocks_axis_order = tuple(range(0, ndim * 2, 2)) + tuple( range(1, ndim * 2, 2) ) hist_slices = [ slice(k // 2, k // 2 + n * k) for k, n in zip(kernel_size, ns_hist) ] hist_blocks = image[tuple(hist_slices)].reshape(hist_blocks_shape) hist_blocks = hist_blocks.transpose(hist_blocks_axis_order) hist_block_assembled_shape = hist_blocks.shape hist_blocks = hist_blocks.reshape((_prod(ns_hist), -1)) # Calculate actual clip limit if clip_limit > 0.0: clim = int(max(clip_limit * _prod(kernel_size), 1)) else: # largest possible value, i.e., do not clip (AHE) clim = np.product(kernel_size) if True: # faster to loop over the arrays on the host hist_blocks = cp.asnumpy(hist_blocks) hist = np.apply_along_axis( np.bincount, -1, hist_blocks, minlength=nbins ) hist = np.apply_along_axis( clip_histogram, -1, hist, clip_limit=clim, xp=np ) hist = cp.asarray(hist) else: hist = cnp.apply_along_axis( cp.bincount, -1, hist_blocks, minlength=nbins ) hist = cnp.apply_along_axis(clip_histogram, -1, hist, clip_limit=clim) hist = map_histogram(hist, 0, NR_OF_GRAY - 1, _prod(kernel_size)) hist = hist.reshape(hist_block_assembled_shape[:ndim] + (-1,)) # duplicate leading mappings in each dim map_array = cp.pad( hist, [(1, 1) for _ in range(ndim)] + [(0, 0)], mode="edge" ) # Perform multilinear interpolation of graylevel mappings # using the convention described here: # https://en.wikipedia.org/w/index.php?title=Adaptive_histogram_ # equalization&oldid=936814673#Efficient_computation_by_interpolation # rearrange image into blocks for vectorized processing ns_proc = [int(s / k) for s, k in zip(image.shape, kernel_size)] blocks_shape = functools.reduce( operator.add, [(s, k) for s, k in zip(ns_proc, kernel_size)] ) blocks_axis_order = hist_blocks_axis_order blocks = image.reshape(blocks_shape) blocks = blocks.transpose(blocks_axis_order) blocks_flattened_shape = blocks.shape blocks = blocks.reshape((_prod(ns_proc), _prod(blocks.shape[ndim:]))) # calculate interpolation coefficients coeffs = cp.meshgrid( *tuple([cp.arange(k) / k for k in kernel_size[::-1]]), indexing="ij" ) coeffs = [cp.transpose(c).flatten() for c in coeffs] inv_coeffs = [1 - c for c in coeffs] # sum over contributions of neighboring contextual # regions in each direction result = cp.zeros(blocks.shape, dtype=cp.float32) for iedge, edge in enumerate(itertools.product(*((range(2),) * ndim))): edge_maps = map_array[ tuple([slice(e, e + n) for e, n in zip(edge, ns_proc)]) ] edge_maps = edge_maps.reshape((_prod(ns_proc), -1)) # apply map # edge_mapped = cp.asarray(np.take_along_axis(edge_maps.get(), blocks.get(), axis=-1)) edge_mapped = cp.take_along_axis(edge_maps, blocks, axis=-1) # interpolate edge_coeffs = functools.reduce( operator.mul, [[inv_coeffs, coeffs][e][d] for d, e in enumerate(edge[::-1])], ) result += (edge_mapped * edge_coeffs).astype(result.dtype) result = result.astype(dtype) # rebuild result image from blocks result = result.reshape(blocks_flattened_shape) blocks_axis_rebuild_order = functools.reduce( operator.add, [(s, k) for s, k in zip(range(0, ndim), range(ndim, ndim * 2))], ) result = result.transpose(blocks_axis_rebuild_order) result = result.reshape(image.shape) # undo padding unpad_slices = tuple( [ slice(p_i, s - p_f) for p_i, p_f, s in zip( pad_start_per_dim, pad_end_per_dim, image.shape ) ] ) result = result[unpad_slices] return result
def move(self, move_preference_matrix, move_probability_matrix, ratio_random_move=0.1): """ 1. Select all living agents and their neighbours. 2. Create a movement matrix. All occupied by agent cells should be unavailable for move. add {move_preference_matrix} for values of neighbours. 3. If agent does not have any available cells for moving - it should die. Drop all died agents from current moving agents. 4. 10% of the time the agent moves randomly. Agent can't go to unavailable cells, so we recalculate probability for available neighbours. (sum of prob should be 1). 5. Vectorized way to get random indices from array of probs. Like random.choice, but for 2d array. 6. Find new flat indexes for random moving agents. 7. Find new flat indexes for normal moving agents. Before argmax selection we shuffle neighbours, otherwise we will use always first max index. 8. Create an array with new agents positions. 9. If two agents want to occupy same cell - then we accept only first. All agents, which was declined to move because of collision will die. 10. If agent reach top - it dies too. :param move_preference_matrix: The agent decides which space to move to by adding this move preference array to the value array of the surrounding environment. :param move_probability_matrix: 10% of the time the agent moves randomly to an adjacent space. It is the move probability matrix. :return: """ # (1) live_agents_neighbour_flat_positions = self.agents_neighbour_flat_positions[ self.agents_state] # (2) move_candidates = self.env.ravel( )[live_agents_neighbour_flat_positions].copy() is_available = self.is_available_env.ravel( )[live_agents_neighbour_flat_positions] move_candidates[~is_available] = cp.nan move_candidates = move_candidates + cp.asarray(move_preference_matrix) # (3) should_die = cp.all(cp.isnan(move_candidates.reshape(-1, 27)), axis=1) should_die_agents = cp.flatnonzero(self.agents_state)[should_die] self.agents_state[should_die_agents] = False move_candidates = move_candidates[~should_die] live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[ ~should_die] # (4) is_random_move = cp.random.binomial( 1, ratio_random_move, live_agents_neighbour_flat_positions.shape[0]) is_random_move = is_random_move.astype(cp.bool) random_move_candidates = move_candidates[is_random_move] random_move_probs = (~cp.isnan(random_move_candidates) * cp.asarray(move_probability_matrix)).reshape( -1, 27) random_move_probs /= random_move_probs.sum(axis=1)[:, None] # (5) random_vals = cp.expand_dims(cp.random.rand( random_move_probs.shape[0]), axis=1) random_indexes = (random_move_probs.cumsum(axis=1) > random_vals).argmax(axis=1) # (6) random_live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[ is_random_move] random_new_positions = cp.take_along_axis( random_live_agents_neighbour_flat_positions.reshape(-1, 27), random_indexes[:, None], axis=1).T[0] # (7) normal_move_candidates = move_candidates[~is_random_move] # normal_move_indexes = cp.nanargmax(normal_move_candidates.reshape(-1, 27), axis=1)[:, None] # smart analog of cp.nanargmax(normal_move_candidates.reshape(-1, 27), axis=1)[:, None] normal_flattened_move_candidates = normal_move_candidates.reshape( -1, 27) normal_shuffled_candidates_idx = cp.random.rand( *normal_flattened_move_candidates.shape).argsort(axis=1) normal_shuffled_flattened_move_candidates = cp.take_along_axis( normal_flattened_move_candidates, normal_shuffled_candidates_idx, axis=1) normal_shuffled_candidates_max_idx = cp.nanargmax( normal_shuffled_flattened_move_candidates, axis=1)[:, None] normal_move_indexes = cp.take_along_axis( normal_shuffled_candidates_idx, normal_shuffled_candidates_max_idx, axis=1) #### normal_live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[ ~is_random_move] normal_move_new_positions = cp.take_along_axis( normal_live_agents_neighbour_flat_positions.reshape(-1, 27), normal_move_indexes, axis=1).T[0] # (8) moving_agents_flat_positions = self.agents_flat_positions[ self.agents_state] new_agents_flat_positions = moving_agents_flat_positions.copy() new_agents_flat_positions[is_random_move] = random_new_positions new_agents_flat_positions[~is_random_move] = normal_move_new_positions live_agents_indexes = cp.flatnonzero(self.agents_state) # (9) _, flat_positions_first_entry = cp.unique(new_agents_flat_positions, return_index=True) is_live = cp.zeros_like(new_agents_flat_positions).astype(cp.bool) is_live[flat_positions_first_entry] = True new_agents_flat_positions[~is_live] = moving_agents_flat_positions[ ~is_live] new_agents_positions = cp.array( cp.unravel_index(new_agents_flat_positions, self.env.shape)).T # (10) is_live[new_agents_positions[:, 2] == 1] = False self._agents_positions[live_agents_indexes] = new_agents_positions self.agents_state[live_agents_indexes] = is_live self.is_available_env.ravel()[moving_agents_flat_positions] = True self.is_available_env.ravel()[new_agents_flat_positions] = False self._agents_positions_all_time.append( cp.asnumpy(self._agents_positions))