def index_select(inp, dim, index) -> 'Tensor':
_check_tensors(inp)
engine = _get_engine(inp)
return _create_tensor(
inp,
data=engine.take_along_axis(inp.data, index.data.astype('int'), dim),
func=wrapped_partial(index_select_backward, inp=inp, index=index, dim=dim)
)
def extract_top_n_words_per_topic(self, tf_idf, vectorizer, labels, n=30):
"""Based on tf_idf scores per topic, extract the top n words per topic
Arguments:
tf_idf: A c-TF-IDF matrix from which to calculate the top words
vectorizer: object of class CountVecWrapper, which is derived from CountVectorizer
labels: A list of unique topic labels
n: number of words per topic (Default: 30)
Returns:
top_n_words: The top n words per topic
topic_names: Dictionary containing key as Topic ID and value
as top 4 words from that topic cluster
"""
words = vectorizer.get_feature_names().to_arrow().to_pylist()
labels = sorted(labels.to_arrow().to_pylist())
indices, scores = top_n_sparse(tf_idf, n)
sorted_indices = cp.argsort(scores, 1)
indices = cp.take_along_axis(indices, sorted_indices, axis=1).get()
scores = cp.take_along_axis(scores, sorted_indices, axis=1).get()
# Get top 30 words per topic based on c-TF-IDF score
topics = {
label: [
(words[word_index], score)
if word_index and score > 0
else ("", 0.00001)
for word_index, score in zip(indices[index][::-1], scores[index][::-1])
]
for index, label in enumerate(labels)
}
topic_names = {
key: f"{key}_" + "_".join([word[0] for word in values[:4]])
for key, values in topics.items()
}
return topics, topic_names
def GetSASE_gpu(CentralEnergy, dE_FWHM, dt_FWHM, samples=0, onlyT=False):
from cupy import interp
import cupy as cp
h = 4.135667662 #in eV*fs
dE = dE_FWHM / 2.355 #in eV, converts to sigma
dt = dt_FWHM / 2.355 #in fs, converts to sigma
if samples == 0:
samples = int(600. * dt * CentralEnergy / h)
else:
if (samples < 400. * dt * CentralEnergy / h):
print(
"Number of samples is a little small, proceeding anyway. Got",
samples, "prefer more than", 400. * dt * CentralEnergy / h)
EnAxis = cp.linspace(0.,
20. * CentralEnergy,
num=samples,
dtype=cp.float32)
newEaxis = cp.linspace(0, 140, 32 * 1024)
# EnInput=cp.zeros(samples, dtype=cp.complex64)
# for i in range(samples):
EnInput = cp.exp(-(EnAxis - CentralEnergy)**2 / 2. / dE**2 +
2 * cp.pi * 1j * cp.random.random(size=samples),
dtype=cp.complex64)
EnInput = interp(newEaxis, EnAxis, EnInput)
newTaxis = cp.fft.fftfreq(32 * 1024, d=140 / (32 * 1024)) * h
TOutput = cp.exp(-newTaxis**2 / 2. / dt**2) * cp.fft.fft(EnInput)
# sort TOutput and newTaxis
ind = cp.argsort(newTaxis, axis=0)
newTaxis = cp.sort(newTaxis)
TOutput = cp.take_along_axis(TOutput, ind, axis=0)
# En_FFT=cp.fft.fft(EnInput)
# TAxis=cp.fft.fftfreq(samples,d=(20.*CentralEnergy)/samples)*h
# TOutput=cp.exp(-TAxis**2/2./dt**2)*En_FFT
if not (onlyT):
EnOutput = cp.fft.ifft(TOutput)
if (onlyT):
return newTaxis, TOutput
else:
return newEaxis, EnOutput, newTaxis, TOutput
def main(vectors,
gpu_id=-1,
n_neighbors=100,
batch_size=1024,
cutoff=0,
start=0,
end=None):
"""
Step 6: Precompute nearest-neighbor queries (optional)
Precompute nearest-neighbor queries for every entry in the vocab to make
Sense2Vec.most_similar faster. The --cutoff option lets you define the
number of earliest rows to limit the neighbors to. For instance, if cutoff
is 100000, no word will have a nearest neighbor outside of the top 100k
vectors.
"""
if gpu_id == -1:
xp = numpy
else:
import cupy as xp
import cupy.cuda.device
cupy.take_along_axis = take_along_axis
device = cupy.cuda.device.Device(gpu_id)
device.use()
vectors_dir = Path(vectors)
vectors_file = vectors_dir / "vectors"
if not vectors_dir.is_dir() or not vectors_file.exists():
err = "Are you passing in the exported sense2vec directory containing a vectors file?"
msg.fail(f"Can't load vectors from {vectors}", err, exits=1)
with msg.loading(f"Loading vectors from {vectors}"):
vectors = xp.load(str(vectors_file))
msg.good(
f"Loaded {vectors.shape[0]:,} vectors with dimension {vectors.shape[1]}"
)
norms = xp.linalg.norm(vectors, axis=1, keepdims=True)
norms[norms == 0] = 1
# Normalize to unit norm
vectors /= norms
if cutoff < 1:
cutoff = vectors.shape[0]
if end is None:
end = vectors.shape[0]
mean = float(norms.mean())
var = float(norms.var())
msg.good(f"Normalized (mean {mean:,.2f}, variance {var:,.2f})")
msg.info(
f"Finding {n_neighbors:,} neighbors among {cutoff:,} most frequent")
n = min(n_neighbors, vectors.shape[0])
subset = vectors[:cutoff]
best_rows = xp.zeros((end - start, n), dtype="i")
scores = xp.zeros((end - start, n), dtype="f")
for i in tqdm.tqdm(list(range(start, end, batch_size))):
size = min(batch_size, end - i)
batch = vectors[i:i + size]
sims = xp.dot(batch, subset.T)
# Set self-similarities to -inf, so that we don't return them.
for j in range(size):
if i + j < sims.shape[1]:
sims[j, i + j] = -xp.inf
# This used to use argpartition, to do a partial sort...But this ended
# up being a ratsnest of terrible numpy crap. Just sorting the whole
# list isn't really slower, and it's much simpler to read.
ranks = xp.argsort(sims, axis=1)
batch_rows = ranks[:, -n:]
# Reverse
batch_rows = batch_rows[:, ::-1]
batch_scores = xp.take_along_axis(sims, batch_rows, axis=1)
best_rows[i:i + size] = batch_rows
scores[i:i + size] = batch_scores
msg.info("Saving output")
if not isinstance(best_rows, numpy.ndarray):
best_rows = best_rows.get()
if not isinstance(scores, numpy.ndarray):
scores = scores.get()
output = {
"indices": best_rows,
"scores": scores.astype("float16"),
"start": start,
"end": end,
"cutoff": cutoff,
}
output_file = vectors_dir / "cache"
with msg.loading("Saving output..."):
srsly.write_msgpack(output_file, output)
msg.good(f"Saved cache to {output_file}")
print (graph_build_endtime - graph_build_starttime)
louvain_starttime = datetime.datetime.now()
result, mod = cg.louvain(G)
vertex = result['vertex']
partition = result['partition']
size = result['partition'].max() + 1
print('community', size)
print('modularity', mod)
vertex = cp.fromDlpack(vertex.to_dlpack())
partition = cp.fromDlpack(partition.to_dlpack())
vertex = cp.reshape(vertex, XYZ_C.shape[0])
labelRE = cp.reshape(partition, XYZ_C.shape[0])
index = cp.argsort(vertex)
vertex = cp.take_along_axis(vertex, index, axis=0)
labelRE = cp.take_along_axis(labelRE, index, axis=0)
print(result)
print(vertex)
print(labelRE)
print(index)
louvain_endtime = datetime.datetime.now()
print(louvain_endtime - louvain_starttime)
print(labelRE)
labelRE = cp.asfortranarray(labelRE)
labelRE = cd.from_dlpack(labelRE.toDlpack())
#labelRE = cd.DataFrame(labelRE)
print(labelRE)
df = cd.DataFrame({'label': labelRE})
df.to_csv('HW_AI/HW_Final/gpu.csv')
def _clahe(image, kernel_size, clip_limit, nbins):
"""Contrast Limited Adaptive Histogram Equalization.
Parameters
----------
image : (N1,...,NN) ndarray
Input image.
kernel_size: int or N-tuple of int
Defines the shape of contextual regions used in the algorithm.
clip_limit : float
Normalized clipping limit between 0 and 1 (higher values give more
contrast).
nbins : int
Number of gray bins for histogram ("data range").
Returns
-------
out : (N1,...,NN) ndarray
Equalized image.
The number of "effective" graylevels in the output image is set by `nbins`;
selecting a small value (e.g. 128) speeds up processing and still produces
an output image of good quality. A clip limit of 0 or larger than or equal
to 1 results in standard (non-contrast limited) AHE.
"""
ndim = image.ndim
dtype = image.dtype
# pad the image such that the shape in each dimension
# - is a multiple of the kernel_size and
# - is preceded by half a kernel size
pad_start_per_dim = [k // 2 for k in kernel_size]
pad_end_per_dim = [
(k - s % k) % k + math.ceil(k / 2.0)
for k, s in zip(kernel_size, image.shape)
]
image = cp.pad(
image,
[(p_i, p_f) for p_i, p_f in zip(pad_start_per_dim, pad_end_per_dim)],
mode="reflect",
)
bin_size = 1 + NR_OF_GRAY // nbins
if True:
lut = cp.arange(NR_OF_GRAY)
lut //= bin_size
image = lut[image]
else:
lut = np.arange(NR_OF_GRAY)
lut //= bin_size
image = cp.asarray(lut[image.get()])
# calculate graylevel mappings for each contextual region
# rearrange image into flattened contextual regions
ns_hist = [int(s / k) - 1 for s, k in zip(image.shape, kernel_size)]
hist_blocks_shape = functools.reduce(
operator.add, [(s, k) for s, k in zip(ns_hist, kernel_size)]
)
hist_blocks_axis_order = tuple(range(0, ndim * 2, 2)) + tuple(
range(1, ndim * 2, 2)
)
hist_slices = [
slice(k // 2, k // 2 + n * k) for k, n in zip(kernel_size, ns_hist)
]
hist_blocks = image[tuple(hist_slices)].reshape(hist_blocks_shape)
hist_blocks = hist_blocks.transpose(hist_blocks_axis_order)
hist_block_assembled_shape = hist_blocks.shape
hist_blocks = hist_blocks.reshape((_prod(ns_hist), -1))
# Calculate actual clip limit
if clip_limit > 0.0:
clim = int(max(clip_limit * _prod(kernel_size), 1))
else:
# largest possible value, i.e., do not clip (AHE)
clim = np.product(kernel_size)
if True:
# faster to loop over the arrays on the host
hist_blocks = cp.asnumpy(hist_blocks)
hist = np.apply_along_axis(
np.bincount, -1, hist_blocks, minlength=nbins
)
hist = np.apply_along_axis(
clip_histogram, -1, hist, clip_limit=clim, xp=np
)
hist = cp.asarray(hist)
else:
hist = cnp.apply_along_axis(
cp.bincount, -1, hist_blocks, minlength=nbins
)
hist = cnp.apply_along_axis(clip_histogram, -1, hist, clip_limit=clim)
hist = map_histogram(hist, 0, NR_OF_GRAY - 1, _prod(kernel_size))
hist = hist.reshape(hist_block_assembled_shape[:ndim] + (-1,))
# duplicate leading mappings in each dim
map_array = cp.pad(
hist, [(1, 1) for _ in range(ndim)] + [(0, 0)], mode="edge"
)
# Perform multilinear interpolation of graylevel mappings
# using the convention described here:
# https://en.wikipedia.org/w/index.php?title=Adaptive_histogram_
# equalization&oldid=936814673#Efficient_computation_by_interpolation
# rearrange image into blocks for vectorized processing
ns_proc = [int(s / k) for s, k in zip(image.shape, kernel_size)]
blocks_shape = functools.reduce(
operator.add, [(s, k) for s, k in zip(ns_proc, kernel_size)]
)
blocks_axis_order = hist_blocks_axis_order
blocks = image.reshape(blocks_shape)
blocks = blocks.transpose(blocks_axis_order)
blocks_flattened_shape = blocks.shape
blocks = blocks.reshape((_prod(ns_proc), _prod(blocks.shape[ndim:])))
# calculate interpolation coefficients
coeffs = cp.meshgrid(
*tuple([cp.arange(k) / k for k in kernel_size[::-1]]), indexing="ij"
)
coeffs = [cp.transpose(c).flatten() for c in coeffs]
inv_coeffs = [1 - c for c in coeffs]
# sum over contributions of neighboring contextual
# regions in each direction
result = cp.zeros(blocks.shape, dtype=cp.float32)
for iedge, edge in enumerate(itertools.product(*((range(2),) * ndim))):
edge_maps = map_array[
tuple([slice(e, e + n) for e, n in zip(edge, ns_proc)])
]
edge_maps = edge_maps.reshape((_prod(ns_proc), -1))
# apply map
# edge_mapped = cp.asarray(np.take_along_axis(edge_maps.get(), blocks.get(), axis=-1))
edge_mapped = cp.take_along_axis(edge_maps, blocks, axis=-1)
# interpolate
edge_coeffs = functools.reduce(
operator.mul,
[[inv_coeffs, coeffs][e][d] for d, e in enumerate(edge[::-1])],
)
result += (edge_mapped * edge_coeffs).astype(result.dtype)
result = result.astype(dtype)
# rebuild result image from blocks
result = result.reshape(blocks_flattened_shape)
blocks_axis_rebuild_order = functools.reduce(
operator.add,
[(s, k) for s, k in zip(range(0, ndim), range(ndim, ndim * 2))],
)
result = result.transpose(blocks_axis_rebuild_order)
result = result.reshape(image.shape)
# undo padding
unpad_slices = tuple(
[
slice(p_i, s - p_f)
for p_i, p_f, s in zip(
pad_start_per_dim, pad_end_per_dim, image.shape
)
]
)
result = result[unpad_slices]
return result
def move(self,
move_preference_matrix,
move_probability_matrix,
ratio_random_move=0.1):
"""
1. Select all living agents and their neighbours.
2. Create a movement matrix. All occupied by agent cells should be unavailable for move.
add {move_preference_matrix} for values of neighbours.
3. If agent does not have any available cells for moving - it should die.
Drop all died agents from current moving agents.
4. 10% of the time the agent moves randomly.
Agent can't go to unavailable cells, so we recalculate probability for available neighbours.
(sum of prob should be 1).
5. Vectorized way to get random indices from array of probs. Like random.choice, but for 2d array.
6. Find new flat indexes for random moving agents.
7. Find new flat indexes for normal moving agents. Before argmax selection we shuffle neighbours,
otherwise we will use always first max index.
8. Create an array with new agents positions.
9. If two agents want to occupy same cell - then we accept only first.
All agents, which was declined to move because of collision will die.
10. If agent reach top - it dies too.
:param move_preference_matrix: The agent decides which space to move to by adding this move
preference array to the value array of the surrounding environment.
:param move_probability_matrix: 10% of the time the agent moves randomly to an adjacent space.
It is the move probability matrix.
:return:
"""
# (1)
live_agents_neighbour_flat_positions = self.agents_neighbour_flat_positions[
self.agents_state]
# (2)
move_candidates = self.env.ravel(
)[live_agents_neighbour_flat_positions].copy()
is_available = self.is_available_env.ravel(
)[live_agents_neighbour_flat_positions]
move_candidates[~is_available] = cp.nan
move_candidates = move_candidates + cp.asarray(move_preference_matrix)
# (3)
should_die = cp.all(cp.isnan(move_candidates.reshape(-1, 27)), axis=1)
should_die_agents = cp.flatnonzero(self.agents_state)[should_die]
self.agents_state[should_die_agents] = False
move_candidates = move_candidates[~should_die]
live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[
~should_die]
# (4)
is_random_move = cp.random.binomial(
1, ratio_random_move,
live_agents_neighbour_flat_positions.shape[0])
is_random_move = is_random_move.astype(cp.bool)
random_move_candidates = move_candidates[is_random_move]
random_move_probs = (~cp.isnan(random_move_candidates) *
cp.asarray(move_probability_matrix)).reshape(
-1, 27)
random_move_probs /= random_move_probs.sum(axis=1)[:, None]
# (5)
random_vals = cp.expand_dims(cp.random.rand(
random_move_probs.shape[0]),
axis=1)
random_indexes = (random_move_probs.cumsum(axis=1) >
random_vals).argmax(axis=1)
# (6)
random_live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[
is_random_move]
random_new_positions = cp.take_along_axis(
random_live_agents_neighbour_flat_positions.reshape(-1, 27),
random_indexes[:, None],
axis=1).T[0]
# (7)
normal_move_candidates = move_candidates[~is_random_move]
# normal_move_indexes = cp.nanargmax(normal_move_candidates.reshape(-1, 27), axis=1)[:, None]
# smart analog of cp.nanargmax(normal_move_candidates.reshape(-1, 27), axis=1)[:, None]
normal_flattened_move_candidates = normal_move_candidates.reshape(
-1, 27)
normal_shuffled_candidates_idx = cp.random.rand(
*normal_flattened_move_candidates.shape).argsort(axis=1)
normal_shuffled_flattened_move_candidates = cp.take_along_axis(
normal_flattened_move_candidates,
normal_shuffled_candidates_idx,
axis=1)
normal_shuffled_candidates_max_idx = cp.nanargmax(
normal_shuffled_flattened_move_candidates, axis=1)[:, None]
normal_move_indexes = cp.take_along_axis(
normal_shuffled_candidates_idx,
normal_shuffled_candidates_max_idx,
axis=1)
####
normal_live_agents_neighbour_flat_positions = live_agents_neighbour_flat_positions[
~is_random_move]
normal_move_new_positions = cp.take_along_axis(
normal_live_agents_neighbour_flat_positions.reshape(-1, 27),
normal_move_indexes,
axis=1).T[0]
# (8)
moving_agents_flat_positions = self.agents_flat_positions[
self.agents_state]
new_agents_flat_positions = moving_agents_flat_positions.copy()
new_agents_flat_positions[is_random_move] = random_new_positions
new_agents_flat_positions[~is_random_move] = normal_move_new_positions
live_agents_indexes = cp.flatnonzero(self.agents_state)
# (9)
_, flat_positions_first_entry = cp.unique(new_agents_flat_positions,
return_index=True)
is_live = cp.zeros_like(new_agents_flat_positions).astype(cp.bool)
is_live[flat_positions_first_entry] = True
new_agents_flat_positions[~is_live] = moving_agents_flat_positions[
~is_live]
new_agents_positions = cp.array(
cp.unravel_index(new_agents_flat_positions, self.env.shape)).T
# (10)
is_live[new_agents_positions[:, 2] == 1] = False
self._agents_positions[live_agents_indexes] = new_agents_positions
self.agents_state[live_agents_indexes] = is_live
self.is_available_env.ravel()[moving_agents_flat_positions] = True
self.is_available_env.ravel()[new_agents_flat_positions] = False
self._agents_positions_all_time.append(
cp.asnumpy(self._agents_positions))