def test_bincount_unspecified_minlength():
x = np.array([1, 1, 3, 7, 0])
d = da.from_array(x, chunks=2)
e = da.bincount(d)
assert_eq(e, np.bincount(x))
assert same_keys(da.bincount(d), e)
assert len(e.compute()) == 8 # shape is (nan,) so must compute for len()
def test_bincount_unspecified_minlength():
x = np.array([1, 1, 3, 7, 0])
d = da.from_array(x, chunks=2)
e = da.bincount(d)
assert_eq(e, np.bincount(x))
assert same_keys(da.bincount(d), e)
assert len(e.compute()) == 8 # shape is (nan,) so must compute for len()
def make_images(data: Data, image_size: Tuple[int, int],
bins: ArrayLike) -> da.Array:
"""
Bin LATRD events data into images of event counts.
Given a collection of events data, a known image shape and an array of the
desired time bin edges, make an image for each time bin, representing the number
of events recorded at each pixel.
Args:
data: A LATRD data dictionary (a dictionary with data set names as keys
and Dask arrays as values). Must contain one entry for event
location messages and one for event timestamps. The two arrays are
assumed to have the same length.
image_size: The (y, x), i.e. (slow, fast) dimensions (number of pixels) of
the image.
bins: The time bin edges of the images (in clock cycles, to match the
event timestamps).
Returns:
A dask array representing the calculations required to obtain the
resulting image.
"""
# We need to ensure that the chunk layout of the event location array matches
# that of the event time array, so that we can perform matching blockwise iterations
event_locations = data[event_location_key].rechunk(
data[event_time_key].chunks)
event_locations = pixel_index(event_locations, image_size)
num_images = len(bins) - 1
if num_images > 1:
# We cannot perform a single bincount of the entire data set because that
# would require allocating enough memory for the entire image stack.
# Find the index of the image to which each event belongs.
image_indices = da.digitize(data[event_time_key], bins) - 1
(images_in_block, ) = da.compute(map(np.unique, image_indices.blocks))
# Construct a stack of images using dask.array.bincount.
images = []
for i in range(num_images):
# When searching for events with a given image index, we already know we
# can exclude some blocks and thereby save some computation time.
contains_index = [i in indices for indices in images_in_block]
image_events = event_locations.blocks[contains_index][
image_indices.blocks[contains_index] == i]
images.append(da.bincount(image_events,
minlength=mul(*image_size)))
images = da.stack(images)
else:
images = da.bincount(event_locations, minlength=mul(*image_size))
return images.astype(np.uint32).reshape(num_images, *image_size)
def test_bincount_with_weights():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
weights = np.array([1, 2, 1, 0.5, 1])
dweights = da.from_array(weights, chunks=2)
e = da.bincount(d, weights=dweights, minlength=6)
assert_eq(e, np.bincount(x, weights=dweights, minlength=6))
assert same_keys(da.bincount(d, weights=dweights, minlength=6), e)
def test_bincount_raises_informative_error_on_missing_minlength_kwarg():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
try:
da.bincount(d)
except Exception as e:
assert 'minlength' in str(e)
else:
assert False
def test_bincount_with_weights():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
weights = np.array([1, 2, 1, 0.5, 1])
dweights = da.from_array(weights, chunks=2)
e = da.bincount(d, weights=dweights, minlength=6)
assert eq(e, np.bincount(x, weights=dweights, minlength=6))
assert same_keys(da.bincount(d, weights=dweights, minlength=6), e)
def test_bincount_raises_informative_error_on_missing_minlength_kwarg():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
try:
da.bincount(d)
except Exception as e:
assert 'minlength' in str(e)
else:
assert False
def compute_class_weight(class_weight, *, classes, y):
if not DaskToolBox.is_dask_object(y):
return sk_utils.class_weight.compute_class_weight(class_weight,
classes=classes,
y=y)
y = DaskToolBox.make_chunk_size_known(y)
if set(dask.compute(da.unique(y))[0]) - set(classes):
raise ValueError(
"classes should include all valid labels that can be in y")
if class_weight == 'balanced':
# Find the weight of each class as present in y.
le = dm_pre.LabelEncoder()
y_ind = le.fit_transform(y)
# if not all(np.in1d(classes, le.classes_)):
# raise ValueError("classes should have valid labels that are in y")
# recip_freq = len(y) / (len(le.classes_) *
# np.bincount(y_ind).astype(np.float64))
# weight = recip_freq[le.transform(classes)]
y_shape, y_ind_bincount, le_classes_ = dask.compute(
y.shape, da.bincount(y_ind), le.classes_)
if not all(np.in1d(classes, le_classes_)):
raise ValueError(
"classes should have valid labels that are in y")
recip_freq = y_shape[0] / (len(le_classes_) *
y_ind_bincount.astype(np.float64))
weight = recip_freq[np.searchsorted(le_classes_, classes)]
else:
raise ValueError("Only class_weight == 'balanced' is supported.")
return weight
def van_hove_distinct(onset, frame, bins, box=None, use_dask=True, comp=False, bincount=True):
r"""
Compute the distinct part of the Van Hove autocorrelation function.
..math::
G(r, t) = \sum_{i, j} \delta(|\vec r_i(0) - \vec r_j(t)| - r)
"""
if box is None:
box = onset.box.diagonal()
dimension = len(box)
N = len(onset)
if use_dask:
onset = darray.from_array(onset, chunks=(500, dimension)).reshape(1, N, dimension)
frame = darray.from_array(frame, chunks=(500, dimension)).reshape(N, 1, dimension)
dist = ((pbc_diff(onset, frame, box)**2).sum(axis=-1)**0.5)
if np.diff(bins).std() < 1e6:
dx = bins[0] - bins[1]
hist = darray.bincount((dist // dx).astype(int), minlength=(len(bins) - 1))
else:
hist = darray.histogram(dist, bins=bins)[0]
return hist.compute() / N
else:
if comp:
dx = bins[1] - bins[0]
minlength = len(bins) - 1
def f(x):
d = (pbc_diff(x, frame, box)**2).sum(axis=-1)**0.5
return np.bincount((d // dx).astype(int), minlength=minlength)[:minlength]
hist = sum(f(x) for x in onset)
else:
dist = (pbc_diff(onset.reshape(1, -1, 3), frame.reshape(-1, 1, 3), box)**2).sum(axis=-1)**0.5
hist = histogram(dist, bins=bins)[0]
return hist / N
def _kmeans_single_lloyd(X, n_clusters, max_iter=300, init='k-means||',
verbose=False, x_squared_norms=None,
random_state=None, tol=1e-4,
precompute_distances=True,
oversampling_factor=2,
init_max_iter=None):
centers = k_init(X, n_clusters, init=init,
oversampling_factor=oversampling_factor,
random_state=random_state, max_iter=init_max_iter)
dt = X.dtype
X = X.astype(np.float32)
P = X.shape[1]
for i in range(max_iter):
t0 = tic()
centers = centers.astype('f4')
labels, distances = pairwise_distances_argmin_min(
X, centers, metric='euclidean', metric_kwargs={"squared": True}
)
labels = labels.astype(np.int32)
distances = distances.astype(np.float32)
r = da.atop(_centers_dense, 'ij',
X, 'ij',
labels, 'i',
n_clusters, None,
distances, 'i',
adjust_chunks={"i": n_clusters, "j": P},
dtype='f8')
new_centers = da.from_delayed(
sum(r.to_delayed().flatten()),
(n_clusters, P),
X.dtype
)
counts = da.bincount(labels, minlength=n_clusters)
new_centers = new_centers / counts[:, None]
new_centers, = compute(new_centers)
# Convergence check
shift = squared_norm(centers - new_centers)
t1 = tic()
logger.info("Lloyd loop %2d. Shift: %0.4f [%.2f s]", i, shift, t1 - t0)
if shift < tol:
break
centers = new_centers
if shift > 1e-7:
labels, distances = pairwise_distances_argmin_min(X, centers)
inertia = distances.astype(dt).sum()
centers = centers.astype(dt)
labels = labels.astype(np.int64)
return labels, inertia, centers, i + 1
def test_bincount():
x = cupy.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2, asarray=False)
e = da.bincount(d, minlength=6)
assert_eq(e, np.bincount(x, minlength=6))
assert same_keys(da.bincount(d, minlength=6), e)
assert da.bincount(d, minlength=6).name != da.bincount(d, minlength=7).name
assert da.bincount(d, minlength=6).name == da.bincount(d, minlength=6).name
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
e = da.bincount(d, minlength=6)
assert_eq(e, np.bincount(x, minlength=6))
assert same_keys(da.bincount(d, minlength=6), e)
assert da.bincount(d, minlength=6).name != da.bincount(d, minlength=7).name
assert da.bincount(d, minlength=6).name == da.bincount(d, minlength=6).name
def triclustering(Z,
nclusters_row,
nclusters_col,
nclusters_bnd,
errobj,
niters,
epsilon,
row_clusters_init=None,
col_clusters_init=None,
bnd_clusters_init=None):
"""
Run the tri-clustering, Dask implementation
:param Z: d x m x n data matrix
:param nclusters_row: number of row clusters
:param nclusters_col: number of column clusters
:param nclusters_bnd: number of band clusters
:param errobj: convergence threshold for the objective function
:param niters: maximum number of iterations
:param epsilon: numerical parameter, avoids zero arguments in log
:param row_clusters_init: initial row cluster assignment
:param col_clusters_init: initial column cluster assignment
:param bnd_clusters_init: initial column cluster assignment
:return: has converged, number of iterations performed. final row,
column, and band clustering, error value
"""
client = get_client()
Z = da.array(Z) if not isinstance(Z, da.Array) else Z
[d, m, n] = Z.shape
bnd_chunks, row_chunks, col_chunks = Z.chunksize
row_clusters = da.array(row_clusters_init) \
if row_clusters_init is not None \
else _initialize_clusters(m, nclusters_row, chunks=row_chunks)
col_clusters = da.array(col_clusters_init) \
if col_clusters_init is not None \
else _initialize_clusters(n, nclusters_col, chunks=col_chunks)
bnd_clusters = da.array(bnd_clusters_init) \
if bnd_clusters_init is not None \
else _initialize_clusters(d, nclusters_bnd, chunks=bnd_chunks)
R = _setup_cluster_matrix(nclusters_row, row_clusters)
C = _setup_cluster_matrix(nclusters_col, col_clusters)
B = _setup_cluster_matrix(nclusters_bnd, bnd_clusters)
e, old_e = 2 * errobj, 0
s = 0
converged = False
Gavg = Z.mean()
while (not converged) & (s < niters):
logger.debug(f'Iteration # {s} ..')
# Calculate number of elements in each tri-cluster
nel_row_clusters = da.bincount(row_clusters, minlength=nclusters_row)
nel_col_clusters = da.bincount(col_clusters, minlength=nclusters_col)
nel_bnd_clusters = da.bincount(bnd_clusters, minlength=nclusters_bnd)
logger.debug(
'num of populated clusters: row {}, col {}, bnd {}'.format(
da.sum(nel_row_clusters > 0).compute(),
da.sum(nel_col_clusters > 0).compute(),
da.sum(nel_bnd_clusters > 0).compute()))
nel_clusters = da.einsum('i,j->ij', nel_row_clusters, nel_col_clusters)
nel_clusters = da.einsum('i,jk->ijk', nel_bnd_clusters, nel_clusters)
# calculate tri-cluster averages (epsilon takes care of empty clusters)
# first sum values in each tri-cluster ..
TriCavg = da.einsum('ij,ilm->jlm', B, Z) # .. along band axis
TriCavg = da.einsum('ij,kim->kjm', R, TriCavg) # .. along row axis
TriCavg = da.einsum('ij,kli->klj', C, TriCavg) # .. along col axis
# finally divide by number of elements in each tri-cluster
TriCavg = (TriCavg + Gavg * epsilon) / (nel_clusters + epsilon)
# unpack tri-cluster averages ..
avg_unpck = da.einsum('ij,jkl->ikl', B, TriCavg) # .. along band axis
avg_unpck = da.einsum('ij,klj->kli', C, avg_unpck) # .. along col axis
# use these for the row cluster assignment
idx = (1, 0, 2)
d_row = _distance(Z.transpose(idx), avg_unpck.transpose(idx), epsilon)
row_clusters = da.argmin(d_row, axis=1)
R = _setup_cluster_matrix(nclusters_row, row_clusters)
# unpack tri-cluster averages ..
avg_unpck = da.einsum('ij,jkl->ikl', B, TriCavg) # .. along band axis
avg_unpck = da.einsum('ij,kjl->kil', R, avg_unpck) # .. along row axis
# use these for the col cluster assignment
idx = (2, 0, 1)
d_col = _distance(Z.transpose(idx), avg_unpck.transpose(idx), epsilon)
col_clusters = da.argmin(d_col, axis=1)
C = _setup_cluster_matrix(nclusters_col, col_clusters)
# unpack tri-cluster averages ..
avg_unpck = da.einsum('ij,kjl->kil', R, TriCavg) # .. along row axis
avg_unpck = da.einsum('ij,klj->kli', C, avg_unpck) # .. along col axis
# use these for the band cluster assignment
d_bnd = _distance(Z, avg_unpck, epsilon)
bnd_clusters = da.argmin(d_bnd, axis=1)
B = _setup_cluster_matrix(nclusters_bnd, bnd_clusters)
# Error value (actually just the band component really)
old_e = e
minvals = da.min(d_bnd, axis=1)
# power 1 divergence, power 2 euclidean
e = da.sum(da.power(minvals, 1))
row_clusters, R, col_clusters, C, bnd_clusters, B, e = client.persist(
[row_clusters, R, col_clusters, C, bnd_clusters, B, e])
e = e.compute()
logger.debug(f'Error = {e:+.15e}, dE = {e - old_e:+.15e}')
converged = abs(e - old_e) < errobj
s = s + 1
if converged:
logger.debug(f'Triclustering converged in {s} iterations')
else:
logger.debug(f'Triclustering not converged in {s} iterations')
return converged, s, row_clusters, col_clusters, bnd_clusters, e
def coclustering(Z,
nclusters_row,
nclusters_col,
errobj,
niters,
epsilon,
col_clusters_init=None,
row_clusters_init=None,
run_on_worker=False):
"""
Run the co-clustering, Dask implementation
:param Z: m x n data matrix
:param nclusters_row: num row clusters
:param nclusters_col: number of column clusters
:param errobj: convergence threshold for the objective function
:param niters: maximum number of iterations
:param epsilon: numerical parameter, avoids zero arguments in log
:param row_clusters_init: initial row cluster assignment
:param col_clusters_init: initial column cluster assignment
:param run_on_worker: whether the function is submitted to a Dask worker
:return: has converged, number of iterations performed. final row and
column clustering, error value
"""
client = get_client()
Z = da.array(Z) if not isinstance(Z, da.Array) else Z
[m, n] = Z.shape
row_chunks, col_chunks = Z.chunksize
row_clusters = da.array(row_clusters_init) \
if row_clusters_init is not None \
else _initialize_clusters(m, nclusters_row, chunks=row_chunks)
col_clusters = da.array(col_clusters_init) \
if col_clusters_init is not None \
else _initialize_clusters(n, nclusters_col, chunks=col_chunks)
R = _setup_cluster_matrix(nclusters_row, row_clusters)
C = _setup_cluster_matrix(nclusters_col, col_clusters)
e, old_e = 2 * errobj, 0
s = 0
converged = False
Gavg = Z.mean()
while (not converged) & (s < niters):
logger.debug(f'Iteration # {s} ..')
# Calculate cluster based averages
# nel_clusters is a matrix with the number of elements per co-cluster
# originally computed as: da.dot(da.dot(R.T, da.ones((m, n))), C)
nel_row_clusters = da.bincount(row_clusters, minlength=nclusters_row)
nel_col_clusters = da.bincount(col_clusters, minlength=nclusters_col)
logger.debug('num of populated clusters: row {}, col {}'.format(
da.sum(nel_row_clusters > 0).compute(),
da.sum(nel_col_clusters > 0).compute()))
nel_clusters = da.outer(nel_row_clusters, nel_col_clusters)
CoCavg = (da.matmul(da.matmul(R.T, Z), C) + Gavg * epsilon) / \
(nel_clusters + epsilon)
# Calculate distance based on row approximation
d_row = _distance(Z, da.matmul(C, CoCavg.T), epsilon)
# Assign to best row cluster
row_clusters = da.argmin(d_row, axis=1)
R = _setup_cluster_matrix(nclusters_row, row_clusters)
# Calculate distance based on column approximation
d_col = _distance(Z.T, da.matmul(R, CoCavg), epsilon)
# Assign to best column cluster
col_clusters = da.argmin(d_col, axis=1)
C = _setup_cluster_matrix(nclusters_col, col_clusters)
# Error value (actually just the column components really)
old_e = e
minvals = da.min(d_col, axis=1)
# power 1 divergence, power 2 euclidean
e = da.sum(da.power(minvals, 1))
row_clusters, R, col_clusters, C, e = client.persist(
[row_clusters, R, col_clusters, C, e])
if run_on_worker:
# this is workaround for e.compute() for a function that runs
# on a worker with multiple threads
# https://github.com/dask/distributed/issues/3827
e = client.compute(e)
secede()
e = e.result()
rejoin()
else:
e = e.compute()
logger.debug(f'Error = {e:+.15e}, dE = {e - old_e:+.15e}')
converged = abs(e - old_e) < errobj
s = s + 1
if converged:
logger.debug(f'Coclustering converged in {s} iterations')
else:
logger.debug(f'Coclustering not converged in {s} iterations')
return converged, s, row_clusters, col_clusters, e
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
assert eq(da.bincount(d, minlength=6), np.bincount(x, minlength=6))
client = Client(cluster)
print(client)
# Allow workers to use module
client.upload_file("/nfs/paper-big-data-engines/utils.py")
client.upload_file("/nfs/paper-big-data-engines/kmeans/Kmeans.py")
# Read images
paths = crawl_dir(os.path.abspath(args.bb_dir))
img = [get_voxels(path, start=start, args=args) for path in paths]
voxels = da.concatenate(img).reshape(-1)
start_time = time() - start
bincount = da.bincount(voxels)
bincount = bincount[bincount != 0]
unique = da.unique(voxels)
unique, counts = dask.compute(unique, bincount)
end_time = time() - start
if args.benchmark:
benchmark(
start_time,
end_time,
"all_file",
args.output_dir,
args.experiment,
"find_frequency",
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
e = da.bincount(d, minlength=6)
assert_eq(e, np.bincount(x, minlength=6))
assert same_keys(da.bincount(d, minlength=6), e)
def _kmeans_single_lloyd(
X,
n_clusters,
max_iter=300,
init="k-means||",
verbose=False,
x_squared_norms=None,
random_state=None,
tol=1e-4,
precompute_distances=True,
oversampling_factor=2,
init_max_iter=None,
):
centers = k_init(
X,
n_clusters,
init=init,
oversampling_factor=oversampling_factor,
random_state=random_state,
max_iter=init_max_iter,
)
dt = X.dtype
P = X.shape[1]
for i in range(max_iter):
with _timer("Lloyd loop %2d." % i, _logger=logger):
labels, distances = pairwise_distances_argmin_min(
X,
centers,
metric="euclidean",
metric_kwargs={"squared": True})
labels = labels.astype(np.int32)
# distances is always float64, but we need it to match X.dtype
# for centers_dense, but remain float64 for inertia
r = blockwise(
_centers_dense,
"ij",
X,
"ij",
labels,
"i",
n_clusters,
None,
"i",
adjust_chunks={
"i": n_clusters,
"j": P
},
dtype=X.dtype,
)
new_centers = da.from_delayed(sum(r.to_delayed().flatten()),
(n_clusters, P), X.dtype)
counts = da.bincount(labels, minlength=n_clusters)
# Require at least one per bucket, to avoid division by 0.
counts = da.maximum(counts, 1)
new_centers = new_centers / counts[:, None]
(new_centers, ) = compute(new_centers)
# Convergence check
shift = squared_norm(centers - new_centers)
logger.info("Shift: %0.4f", shift)
if shift < tol:
break
centers = new_centers
if shift > 1e-7:
labels, distances = pairwise_distances_argmin_min(X, centers)
labels = labels.astype(np.int32)
inertia = distances.sum()
centers = centers.astype(dt)
return labels, inertia, centers, i + 1
def test_bincount():
x = np.array([2, 1, 5, 2, 1])
d = da.from_array(x, chunks=2)
assert eq(da.bincount(d, minlength=6), np.bincount(x, minlength=6))
