def shortestPath(ctx, dim, numIters): dist = eager( expr.shuffle( expr.ndarray( (dim, 1), dtype = np.int64, tile_hint = (dim / ctx.num_workers, 1) ), make_dist, ) ) linkMatrix = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int64, tile_hint = (dim, dim / ctx.num_workers)), make_matrix, )) startCompute = time.time() for it in range(numIters): first = expr.add(dist, linkMatrix) second = first.min(axis = 0) dist = second.reshape(dim, 1) dist.evaluate() endCompute = time.time() return endCompute - startCompute
def bfs(ctx, dim):
util.log_info("start to computing......")
sGenerate = time.time()
current = eager(
expr.shuffle(
expr.ndarray(
(dim, 1),
dtype = np.int64,
tile_hint = (dim / ctx.num_workers, 1)),
make_current,
))
linkMatrix = eager(
expr.shuffle(
expr.ndarray(
(dim, dim),
dtype = np.int64,
tile_hint = (dim, dim / ctx.num_workers)),
make_matrix,
))
eGenerate = time.time()
startCompute = time.time()
while(True):
next = expr.dot(linkMatrix, current)
formerNum = expr.count_nonzero(current)
laterNum = expr.count_nonzero(next)
hasNew = expr.equal(formerNum, laterNum).glom()
current = next
if (hasNew):
break
current.evaluate()
endCompute = time.time()
return (eGenerate - sGenerate, endCompute - startCompute)
def connectedConponents(ctx, dim, numIters): linkMatrix = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int64, tile_hint = (dim / ctx.num_workers, dim)), make_matrix, )) power = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int64, tile_hint = (dim / ctx.num_workers, dim)), make_matrix, )) eye = expr.eye(dim, tile_hint = (dim / ctx.num_workers,dim)) startCompute = time.time() result = expr.logical_or(eye, linkMatrix).optimized().glom() for i in range(numIters): power = expr.dot(power, linkMatrix).optimized().glom() result = expr.logical_or(result, power) result.optimized().glom() final = expr.logical_and(result, expr.transpose(result.optimized())).optimized().evaluate() endCompute = time.time() return endCompute - startCompute
def _fit_transform(self, X):
self.nbrs_.fit(X)
self.training_data_ = self.nbrs_._fit_X
self.kernel_pca_ = KernelPCA(n_components=self.n_components,
kernel="precomputed",
eigen_solver=self.eigen_solver,
tol=self.tol, max_iter=self.max_iter)
kng = kneighbors_graph(self.nbrs_, self.n_neighbors, mode="distance")
n_points = X.shape[0]
n_workers = blob_ctx.get().num_workers
if n_points < n_workers:
tile_hint = (1, )
else:
tile_hint = (n_points / n_workers, )
"""
task_array is used for deciding the idx of starting points and idx of endding points
that each tile needs to find the shortest path among.
"""
task_array = expr.ndarray((n_points,), tile_hint=tile_hint)
task_array = task_array.force()
#dist matrix is used to hold the result
dist_matrix = expr.ndarray((n_points, n_points), reduce_fn=lambda a,b:a+b).force()
results = task_array.foreach_tile(mapper_fn = _shortest_path_mapper,
kw = {'kng' : kng,
'directed' : False,
'dist_matrix' : dist_matrix})
self.dist_matrix_ = dist_matrix.glom()
G = self.dist_matrix_ ** 2
G *= -0.5
self.embedding_ = self.kernel_pca_.fit_transform(G)
def fit(self, X, centers = None):
"""Compute k-means clustering.
Parameters
----------
X : spartan matrix, shape=(n_samples, n_features). It should be tiled by rows.
centers : numpy.ndarray. The initial centers. If None, it will be randomly generated.
"""
X = expr.force(X)
num_dim = X.shape[1]
labels = expr.zeros((X.shape[0],1), dtype=np.int, tile_hint=X.tile_shape())
if centers is None:
centers = np.random.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
# Reset them to zero.
new_centers = expr.ndarray((self.n_clusters, num_dim), reduce_fn=lambda a, b: a + b)
new_counts = expr.ndarray((self.n_clusters, 1), dtype=np.int, reduce_fn=lambda a, b: a + b)
_ = expr.shuffle(X,
_find_cluster_mapper,
kw={'d_pts' : X,
'old_centers' : centers,
'new_centers' : new_centers,
'new_counts' : new_counts,
'labels': labels
})
_.force()
new_counts = new_counts.glom()
new_centers = new_centers.glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (new_counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
new_counts[zcount_indices] = 1
new_centers[zcount_indices, :] = np.random.randn(n_points, num_dim)
new_centers = new_centers / new_counts
centers = new_centers
return centers, labels
def benchmark_naive_bayes(ctx, timer):
print "#worker:", ctx.num_workers
#N = 100000 * ctx.num_workers
N = 10000 * 64
D = 128
# create data
data = expr.randint(N, D, low=0, high=D, tile_hint=(N, D/ctx.num_workers))
labels = expr.shuffle(expr.ndarray((data.shape[0], 1), dtype=np.int), _init_label_mapper,
kw={'data': data}, shape_hint=(data.shape[0], 1),
cost_hint={hash(data):{'00': 0, '10': np.prod(data.shape)}}
)
#util.log_warn('data:%s, label:%s', data.glom(), labels.glom())
util.log_warn('begin train')
t1 = datetime.now()
model = fit(data, labels, D)
t2 = datetime.now()
util.log_warn('train time:%s ms', millis(t1,t2))
correct = 0
for i in range(10):
new_data = expr.randint(1, D, low=0, high=D, tile_hint=(1, D))
new_label = predict(model, new_data)
#print 'point %s, predict %s' % (new_data.glom(), new_label)
new_data = new_data.glom()
if np.isclose(new_data[0, new_label], np.max(new_data)):
correct += 1
print 'predict precision:', correct * 1.0 / 10
def fit(data, labels, label_size, alpha=1.0):
'''
Train standard naive bayes model.
Args:
data(Expr): documents to be trained.
labels(Expr): the correct labels of the training data.
label_size(int): the number of different labels.
alpha(float): alpha parameter of naive bayes model.
'''
labels = expr.force(labels)
# calc document freq
df = expr.reduce(data,
axis=0,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis: (data > 0).sum(axis),
accumulate_fn=np.add,
tile_hint=(data.shape[1],))
idf = expr.log(data.shape[0] * 1.0 / (df + 1)) + 1
# Normalized Frequency for a feature in a document is calculated by dividing the feature frequency
# by the root mean square of features frequencies in that document
square_sum = expr.reduce(data,
axis=1,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis: np.square(data).sum(axis),
accumulate_fn=np.add,
tile_hint=(data.shape[0],))
rms = expr.sqrt(square_sum * 1.0 / data.shape[1])
# calculate weight normalized Tf-Idf
data = data / rms.reshape((data.shape[0], 1)) * idf.reshape((1, data.shape[1]))
# add up all the feature vectors with the same labels
sum_instance_by_label = expr.ndarray((label_size, data.shape[1]),
dtype=np.float64,
reduce_fn=np.add,
tile_hint=(label_size / len(labels.tiles), data.shape[1]))
sum_instance_by_label = expr.shuffle(data,
_sum_instance_by_label_mapper,
target=sum_instance_by_label,
kw={'labels': labels, 'label_size': label_size})
# sum up all the weights for each label from the previous step
weights_per_label = expr.sum(sum_instance_by_label, axis=1, tile_hint=(label_size,))
# generate naive bayes per_label_and_feature weights
weights_per_label_and_feature = expr.shuffle(sum_instance_by_label,
_naive_bayes_mapper,
kw={'weights_per_label': weights_per_label,
'alpha':alpha})
return {'scores_per_label_and_feature': weights_per_label_and_feature.force(),
'scores_per_label': weights_per_label.force(),
}
def saveAsTextFile(ctx, dim): matrix = eager( expr.shuffle( expr.ndarray( (dim, dim), dtype = np.int32, tile_hint = (dim, dim / ctx.num_workers)), #tile_hint = (2, 2)), make_matrix, ))
def fit(self, X, y):
"""
Parameters
----------
X : array-like of shape = [n_samples, n_features]
The training input samples.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target values (integers that correspond to classes in
classification, real numbers in regression).
Returns
-------
self : object
Returns self.
"""
if isinstance(X, np.ndarray):
X = expr.from_numpy(X)
if isinstance(y, np.ndarray):
y = expr.from_numpy(y)
X = expr.force(X)
y = expr.force(y)
self.n_classes = np.unique(y.glom()).size
ctx = blob_ctx.get()
n_workers = ctx.num_workers
_ = self._create_task_array(n_workers, self.n_estimators)
task_array = expr.from_numpy(_, tile_hint=(1, )).force()
target_array = expr.ndarray((task_array.shape[0], ),
dtype=object,
tile_hint=(1, )).force()
results = task_array.foreach_tile(mapper_fn=_build_mapper,
kw={
'task_array': task_array,
'target_array': target_array,
'X': X,
'y': y,
'criterion': self.criterion,
'max_depth': self.max_depth,
'min_samples_split':
self.min_samples_split,
'min_samples_leaf':
self.min_samples_leaf,
'max_features':
self.max_features,
'bootstrap': self.bootstrap
})
# Target array stores the local random forest each worker builds,
# it's used for further prediction.
self.target_array = target_array
return self
def pagerank_sparse(num_pages,
num_outlinks,
same_site_prob):
result = expr.ndarray((num_pages, num_pages), dtype=np.float32, sparse=True)
cost = num_pages * num_pages
return expr.shuffle(result,
target=result,
fn=_make_site_sparse,
kw = { 'num_outlinks' : num_outlinks,
'same_site_prob' : same_site_prob },
cost_hint={hash(result):{'11':0, '01':cost, '10':cost, '00':cost}})
def fit(data, labels, label_size, alpha=1.0):
'''
Train standard naive bayes model.
Args:
data(Expr): documents to be trained.
labels(Expr): the correct labels of the training data.
label_size(int): the number of different labels.
alpha(float): alpha parameter of naive bayes model.
'''
# calc document freq
df = expr.reduce(data,
axis=0,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis: (data > 0).sum(axis),
accumulate_fn=np.add)
idf = expr.log(data.shape[0] * 1.0 / (df + 1)) + 1
# Normalized Frequency for a feature in a document is calculated by dividing the feature frequency
# by the root mean square of features frequencies in that document
square_sum = expr.reduce(data,
axis=1,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis: np.square(data).sum(axis),
accumulate_fn=np.add)
rms = expr.sqrt(square_sum * 1.0 / data.shape[1])
# calculate weight normalized Tf-Idf
data = data / rms.reshape((data.shape[0], 1)) * idf.reshape((1, data.shape[1]))
# add up all the feature vectors with the same labels
#weights_per_label_and_feature = expr.ndarray((label_size, data.shape[1]), dtype=np.float64)
#for i in range(label_size):
# i_mask = (labels == i)
# weights_per_label_and_feature = expr.assign(weights_per_label_and_feature, np.s_[i, :], expr.sum(data[i_mask, :], axis=0))
weights_per_label_and_feature = expr.shuffle(expr.retile(data, tile_hint=util.calc_tile_hint(data, axis=0)),
_sum_instance_by_label_mapper,
target=expr.ndarray((label_size, data.shape[1]), dtype=np.float64, reduce_fn=np.add),
kw={'labels': labels, 'label_size': label_size},
cost_hint={hash(labels):{'00':0, '01':np.prod(labels.shape)}})
# sum up all the weights for each label from the previous step
weights_per_label = expr.sum(weights_per_label_and_feature, axis=1)
# generate naive bayes per_label_and_feature weights
weights_per_label_and_feature = expr.log((weights_per_label_and_feature + alpha) /
(weights_per_label.reshape((weights_per_label.shape[0], 1)) +
alpha * weights_per_label_and_feature.shape[1]))
return {'scores_per_label_and_feature': weights_per_label_and_feature.optimized().force(),
'scores_per_label': weights_per_label.optimized().force(),
}
def fuzzy_kmeans(points, k=10, num_iter=10, m=2.0, centers=None):
'''
clustering data points using fuzzy kmeans clustering method.
Args:
points(Expr or DistArray): the input data points matrix.
k(int): the number of clusters.
num_iter(int): the max iterations to run.
m(float): the parameter of fuzzy kmeans.
centers(Expr or DistArray): the initialized centers of each cluster.
'''
points = expr.force(points)
num_dim = points.shape[1]
if centers is None:
centers = expr.rand(k, num_dim, tile_hint=(k, num_dim))
labels = expr.zeros((points.shape[0],), dtype=np.int, tile_hint=(points.shape[0]/len(points.tiles),))
for iter in range(num_iter):
new_centers = expr.ndarray((k, num_dim), reduce_fn=lambda a, b: a + b, tile_hint=(k, num_dim))
new_counts = expr.ndarray((k, 1), dtype=np.float, reduce_fn=lambda a, b: a + b, tile_hint=(k, 1))
expr.shuffle(points, _fuzzy_kmeans_mapper, kw={'old_centers': centers,
'centers': new_centers,
'counts': new_counts,
'labels': labels,
'm': m}).force()
# If any centroids don't have any points assigned to them.
zcount_indices = (new_counts.glom() == 0).reshape(k)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them, which results in their
# position being the zero-vector. We reseed these centroids with new random values
# and set their counts to 1 in order to get rid of dividing by zero.
new_counts[zcount_indices, :] = 1
new_centers[zcount_indices, :] = np.random.rand(np.count_nonzero(zcount_indices), num_dim)
centers = new_centers / new_counts
return labels
def pagerank_sparse(num_pages,
num_outlinks,
same_site_prob,
hint):
return expr.shuffle(
expr.ndarray((num_pages, num_pages),
dtype=np.float32,
tile_hint=hint,
sparse=True),
fn=_make_site_sparse,
kw = { 'num_outlinks' : num_outlinks,
'same_site_prob' : same_site_prob })
def als(A, la=0.065, alpha=40, implicit_feedback=False, num_features=20, num_iter=10):
'''
compute the factorization A = U M' using the alternating least-squares (ALS) method.
where `A` is the "ratings" matrix which maps from a user and item to a rating score,
`U` and `M` are the factor matrices, which represent user and item preferences.
Args:
A(Expr or DistArray): the rating matrix which maps from a user and item to a rating score.
la(float): the parameter of the als.
alpha(int): confidence parameter used on implicit feedback.
implicit_feedback(bool): whether using implicit_feedback method for als.
num_features(int): dimension of the feature space.
num_iter(int): max iteration to run.
'''
A = expr.force(A)
AT = expr.shuffle(expr.ndarray((A.shape[1], A.shape[0]), dtype=A.dtype,
tile_hint=(A.shape[1] / len(A.tiles), A.shape[0])),
_transpose_mapper, kw={'orig_array': A})
num_items = A.shape[1]
avg_rating = expr.sum(A, axis=0, tile_hint=(num_items / len(A.tiles),)) * 1.0 / \
expr.count_nonzero(A, axis=0, tile_hint=(num_items / len(A.tiles),))
M = expr.shuffle(expr.ndarray((num_items, num_features),
tile_hint=(num_items / len(A.tiles), num_features)),
_init_M_mapper, kw={'avg_rating': avg_rating})
#util.log_warn('avg_rating:%s M:%s', avg_rating.glom(), M.glom())
for i in range(num_iter):
# Recomputing U
U = expr.shuffle(A, _solve_U_or_M_mapper, kw={'U_or_M': M, 'la': la, 'alpha': alpha, 'implicit_feedback': implicit_feedback})
# Recomputing M
M = expr.shuffle(AT, _solve_U_or_M_mapper, kw={'U_or_M': U, 'la': la, 'alpha': alpha, 'implicit_feedback': implicit_feedback})
return U, M
def fit(self, X, y):
"""
Parameters
----------
X : array-like of shape = [n_samples, n_features]
The training input samples.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target values (integers that correspond to classes in
classification, real numbers in regression).
Returns
-------
self : object
Returns self.
"""
if isinstance(X, np.ndarray):
X = expr.from_numpy(X)
if isinstance(y, np.ndarray):
y = expr.from_numpy(y)
X = X.evaluate()
y = y.evaluate()
self.n_classes = np.unique(y.glom()).size
ctx = blob_ctx.get()
n_workers = ctx.num_workers
_ = self._create_task_array(n_workers, self.n_estimators)
task_array = expr.from_numpy(_, tile_hint=(1, )).evaluate()
target_array = expr.ndarray((task_array.shape[0], ), dtype=object, tile_hint=(1,)).evaluate()
results = task_array.foreach_tile(mapper_fn=_build_mapper,
kw={'task_array': task_array,
'target_array': target_array,
'X': X,
'y': y,
'criterion': self.criterion,
'max_depth': self.max_depth,
'min_samples_split': self.min_samples_split,
'min_samples_leaf': self.min_samples_leaf,
'max_features': self.max_features,
'bootstrap': self.bootstrap})
# Target array stores the local random forest each worker builds,
# it's used for further prediction.
self.target_array = target_array
return self
def fit(data, labels, num_tiles, T=50, la=1.0):
'''
Train an SVM model using the disdca (2013) algorithm.
Args:
data(Expr): points to be trained.
labels(Expr): the correct labels of the training data.
num_tiles(int): the total tiles of the training data.
T(int): max training iterations.
la(float): lambda parameter of this SVM model.
'''
w = None
m = data.shape[0] / num_tiles
alpha = expr.zeros((m * num_tiles, 1), dtype=np.float64, tile_hint=(m,1)).force()
for i in range(T):
new_weight = expr.ndarray((data.shape[1], 1), dtype=np.float64, reduce_fn=np.add, tile_hint=(data.shape[1], 1))
new_weight = expr.shuffle(data, _svm_mapper, target=new_weight, kw={'labels': labels, 'alpha': alpha, 'w': w, 'm': m, 'scale': num_tiles, 'lambda_n': la * data.shape[0]})
w = new_weight / num_tiles
return w
def benchmark_pagerank(ctx, timer):
num_pages = PAGES_PER_WORKER * ctx.num_workers
util.log_info('Total pages: %s', num_pages)
wts = eager(
expr.shuffle(
expr.ndarray(
(num_pages, num_pages),
dtype=np.float32,
tile_hint=(num_pages, PAGES_PER_WORKER / 8)),
make_weights,
))
p = eager(expr.ones((num_pages, 1),
tile_hint=(PAGES_PER_WORKER / 8, 1),
dtype=np.float32))
for i in range(3):
timer.time_op('pagerank', lambda: expr.dot(wts, p).force())
def benchmark_pagerank(ctx, timer):
num_pages = PAGES_PER_WORKER * ctx.num_workers
util.log_info('Total pages: %s', num_pages)
wts = eager(
expr.shuffle(
expr.ndarray((num_pages, num_pages),
dtype=np.float32,
tile_hint=(num_pages, PAGES_PER_WORKER / 8)),
make_weights,
))
p = eager(
expr.ones((num_pages, 1),
tile_hint=(PAGES_PER_WORKER / 8, 1),
dtype=np.float32))
for i in range(3):
timer.time_op('pagerank', lambda: expr.dot(wts, p).force())
def pagerank_sparse(num_pages, num_outlinks, same_site_prob):
result = expr.ndarray((num_pages, num_pages),
dtype=np.float32,
sparse=True)
cost = num_pages * num_pages
return expr.shuffle(result,
target=result,
fn=_make_site_sparse,
kw={
'num_outlinks': num_outlinks,
'same_site_prob': same_site_prob
},
cost_hint={
hash(result): {
'11': 0,
'01': cost,
'10': cost,
'00': cost
}
})
def pagerankDistributed(ctx, numPage, numIters, alpha):
sGenerate = time.time()
rank = eager(expr.ones((numPage, 1), tile_hint = (numPage / ctx.num_workers, 1), dtype = np.float32))
linkMatrix = eager(
expr.shuffle(
expr.ndarray(
(numPage, numPage),
dtype = np.float32,
tile_hint = (numPage, numPage / ctx.num_workers)),
make_weights,
))
eGenerate = time.time()
util.log_info("**pagerank** rank init finished")
startCompute = time.time()
for i in range(numIters):
#rank = ((1 - alpha) * expr.dot(linkMatrix, rank,tile_hint = (numPage, numPage/10))) + belta
rank = expr.dot(linkMatrix, rank, tile_hint = (numPage, numPage/10))
rank.evaluate()
endCompute = time.time()
util.log_info("**pagerank** compute finished")
return (eGenerate - sGenerate, endCompute - startCompute)
def learn_topics(terms_docs_matrix, k_topics, alpha=0.1, eta=0.1, max_iter=10, max_iter_per_doc=1):
'''
Using Collapsed Variational Bayes method (Mahout implementation) to train LDA topic model.
Args:
terms_docs_matrix(Expr or DistArray): the count of each term in each document.
k_topics: the number of topics we need to find.
alpha(float): parameter of LDA model.
eta(float): parameter of LDA model.
max_iter(int):the max iterations to train LDA topic model.
max_iter_per_doc: the max iterations to train each document.
'''
topic_term_counts = expr.rand(k_topics, terms_docs_matrix.shape[0],
tile_hint=(k_topics, terms_docs_matrix.shape[0]))
for i in range(max_iter):
new_topic_term_counts = expr.ndarray((k_topics, terms_docs_matrix.shape[0]),
dtype=np.float64,
reduce_fn=np.add,
tile_hint=(k_topics, terms_docs_matrix.shape[0]))
topic_term_counts = expr.shuffle(terms_docs_matrix, _lda_mapper, target=new_topic_term_counts,
kw={'k_topics': k_topics, 'alpha': alpha, 'eta':eta,
'max_iter_per_doc': max_iter_per_doc,
'topic_term_counts': topic_term_counts})
# calculate the doc-topic inference
doc_topics = expr.shuffle(terms_docs_matrix, _lda_doc_topic_mapper,
kw={'k_topics': k_topics, 'alpha': alpha, 'eta':eta,
'max_iter_per_doc': max_iter_per_doc,
'topic_term_counts': topic_term_counts})
# normalize the topic-term distribution
norm_val = expr.reduce(topic_term_counts, axis=1,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis:np.abs(data).sum(axis),
accumulate_fn=np.add)
topic_term_counts = topic_term_counts / norm_val.reshape((topic_term_counts.shape[0], 1))
return doc_topics, topic_term_counts
def test_pagerank(self):
_skip_if_travis()
OUTLINKS_PER_PAGE = 10
PAGES_PER_WORKER = 1000000
num_pages = PAGES_PER_WORKER * self.ctx.num_workers
wts = expr.shuffle(
expr.ndarray(
(num_pages, num_pages),
dtype=np.float32,
tile_hint=(num_pages, PAGES_PER_WORKER / 8)),
make_weights,
)
start = time.time()
p = expr.eager(expr.ones((num_pages, 1), tile_hint=(PAGES_PER_WORKER / 8, 1),
dtype=np.float32))
expr.dot(wts, p, tile_hint=(PAGES_PER_WORKER / 8, 1)).evaluate()
cost = time.time() - start
self._verify_cost("pagerank", cost)
def fit(data, labels, label_size, alpha=1.0):
'''
Train standard naive bayes model.
Args:
data(Expr): documents to be trained.
labels(Expr): the correct labels of the training data.
label_size(int): the number of different labels.
alpha(float): alpha parameter of naive bayes model.
'''
# calc document freq
df = expr.reduce(data,
axis=0,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis:
(data > 0).sum(axis),
accumulate_fn=np.add)
idf = expr.log(data.shape[0] * 1.0 / (df + 1)) + 1
# Normalized Frequency for a feature in a document is calculated by dividing the feature frequency
# by the root mean square of features frequencies in that document
square_sum = expr.reduce(
data,
axis=1,
dtype_fn=lambda input: input.dtype,
local_reduce_fn=lambda ex, data, axis: np.square(data).sum(axis),
accumulate_fn=np.add)
rms = expr.sqrt(square_sum * 1.0 / data.shape[1])
# calculate weight normalized Tf-Idf
data = data / rms.reshape((data.shape[0], 1)) * idf.reshape(
(1, data.shape[1]))
# add up all the feature vectors with the same labels
#weights_per_label_and_feature = expr.ndarray((label_size, data.shape[1]), dtype=np.float64)
#for i in range(label_size):
# i_mask = (labels == i)
# weights_per_label_and_feature = expr.assign(weights_per_label_and_feature, np.s_[i, :], expr.sum(data[i_mask, :], axis=0))
weights_per_label_and_feature = expr.shuffle(
expr.retile(data, tile_hint=util.calc_tile_hint(data, axis=0)),
_sum_instance_by_label_mapper,
target=expr.ndarray((label_size, data.shape[1]),
dtype=np.float64,
reduce_fn=np.add),
kw={
'labels': labels,
'label_size': label_size
},
cost_hint={hash(labels): {
'00': 0,
'01': np.prod(labels.shape)
}})
# sum up all the weights for each label from the previous step
weights_per_label = expr.sum(weights_per_label_and_feature, axis=1)
# generate naive bayes per_label_and_feature weights
weights_per_label_and_feature = expr.log(
(weights_per_label_and_feature + alpha) /
(weights_per_label.reshape((weights_per_label.shape[0], 1)) +
alpha * weights_per_label_and_feature.shape[1]))
return {
'scores_per_label_and_feature':
weights_per_label_and_feature.optimized().force(),
'scores_per_label':
weights_per_label.optimized().force(),
}
def fit(self, X, centers=None, implementation='map2'):
"""Compute k-means clustering.
Parameters
----------
X : spartan matrix, shape=(n_samples, n_features). It should be tiled by rows.
centers : numpy.ndarray. The initial centers. If None, it will be randomly generated.
"""
num_dim = X.shape[1]
num_points = X.shape[0]
labels = expr.zeros((num_points, 1), dtype=np.int)
if implementation == 'map2':
if centers is None:
centers = np.random.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
labels = expr.map2(X,
0,
fn=kmeans_map2_dist_mapper,
fn_kw={"centers": centers},
shape=(X.shape[0], ))
counts = expr.map2(labels,
0,
fn=kmeans_count_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], ))
new_centers = expr.map2(
(X, labels), (0, 0),
fn=kmeans_center_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], centers.shape[1]))
counts = counts.optimized().glom()
centers = new_centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(
n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
return centers, labels
elif implementation == 'outer':
if centers is None:
centers = expr.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
labels = expr.outer((X, centers), (0, None),
fn=kmeans_outer_dist_mapper,
shape=(X.shape[0], ))
#labels = expr.argmin(distances, axis=1)
counts = expr.map2(labels,
0,
fn=kmeans_count_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], ))
new_centers = expr.map2(
(X, labels), (0, 0),
fn=kmeans_center_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], centers.shape[1]))
counts = counts.optimized().glom()
centers = new_centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(
n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
centers = expr.from_numpy(centers)
return centers, labels
elif implementation == 'broadcast':
if centers is None:
centers = expr.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
util.log_warn("k_means_ %d %d", i, time.time())
X_broadcast = expr.reshape(X, (X.shape[0], 1, X.shape[1]))
centers_broadcast = expr.reshape(
centers, (1, centers.shape[0], centers.shape[1]))
distances = expr.sum(expr.square(X_broadcast -
centers_broadcast),
axis=2)
labels = expr.argmin(distances, axis=1)
center_idx = expr.arange((1, centers.shape[0]))
matches = expr.reshape(labels,
(labels.shape[0], 1)) == center_idx
matches = matches.astype(np.int64)
counts = expr.sum(matches, axis=0)
centers = expr.sum(
X_broadcast *
expr.reshape(matches,
(matches.shape[0], matches.shape[1], 1)),
axis=0)
counts = counts.optimized().glom()
centers = centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(
n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
centers = expr.from_numpy(centers)
return centers, labels
elif implementation == 'shuffle':
if centers is None:
centers = np.random.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
# Reset them to zero.
new_centers = expr.ndarray((self.n_clusters, num_dim),
reduce_fn=lambda a, b: a + b)
new_counts = expr.ndarray((self.n_clusters, 1),
dtype=np.int,
reduce_fn=lambda a, b: a + b)
_ = expr.shuffle(X,
_find_cluster_mapper,
kw={
'd_pts': X,
'old_centers': centers,
'new_centers': new_centers,
'new_counts': new_counts,
'labels': labels
},
shape_hint=(1, ),
cost_hint={
hash(labels): {
'00': 0,
'01': np.prod(labels.shape)
}
})
_.force()
new_counts = new_counts.glom()
new_centers = new_centers.glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (new_counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
new_counts[zcount_indices] = 1
new_centers[zcount_indices, :] = np.random.randn(
n_points, num_dim)
new_centers = new_centers / new_counts
centers = new_centers
return centers, labels
def fit(self, X, centers=None, implementation='outer'):
"""Compute k-means clustering.
Parameters
----------
X : spartan matrix, shape=(n_samples, n_features). It should be tiled by rows.
centers : numpy.ndarray. The initial centers. If None, it will be randomly generated.
"""
num_dim = X.shape[1]
num_points = X.shape[0]
labels = expr.zeros((num_points, 1), dtype=np.int)
if implementation == 'map2':
if centers is None:
centers = np.random.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
labels = expr.map2(X, 0, fn=kmeans_map2_dist_mapper, fn_kw={"centers": centers},
shape=(X.shape[0], ))
counts = expr.map2(labels, 0, fn=kmeans_count_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], ))
new_centers = expr.map2((X, labels), (0, 0), fn=kmeans_center_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], centers.shape[1]))
counts = counts.optimized().glom()
centers = new_centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
return centers, labels
elif implementation == 'outer':
if centers is None:
centers = expr.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
labels = expr.outer((X, centers), (0, None), fn=kmeans_outer_dist_mapper,
shape=(X.shape[0],))
#labels = expr.argmin(distances, axis=1)
counts = expr.map2(labels, 0, fn=kmeans_count_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], ))
new_centers = expr.map2((X, labels), (0, 0), fn=kmeans_center_mapper,
fn_kw={'centers_count': self.n_clusters},
shape=(centers.shape[0], centers.shape[1]))
counts = counts.optimized().glom()
centers = new_centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
centers = expr.from_numpy(centers)
return centers, labels
elif implementation == 'broadcast':
if centers is None:
centers = expr.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
util.log_warn("k_means_ %d %d", i, time.time())
X_broadcast = expr.reshape(X, (X.shape[0], 1, X.shape[1]))
centers_broadcast = expr.reshape(centers, (1, centers.shape[0],
centers.shape[1]))
distances = expr.sum(expr.square(X_broadcast - centers_broadcast), axis=2)
labels = expr.argmin(distances, axis=1)
center_idx = expr.arange((1, centers.shape[0]))
matches = expr.reshape(labels, (labels.shape[0], 1)) == center_idx
matches = matches.astype(np.int64)
counts = expr.sum(matches, axis=0)
centers = expr.sum(X_broadcast * expr.reshape(matches, (matches.shape[0],
matches.shape[1], 1)),
axis=0)
counts = counts.optimized().glom()
centers = centers.optimized().glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
counts[zcount_indices] = 1
centers[zcount_indices, :] = np.random.randn(n_points, num_dim)
centers = centers / counts.reshape(centers.shape[0], 1)
centers = expr.from_numpy(centers)
return centers, labels
elif implementation == 'shuffle':
if centers is None:
centers = np.random.rand(self.n_clusters, num_dim)
for i in range(self.n_iter):
# Reset them to zero.
new_centers = expr.ndarray((self.n_clusters, num_dim),
reduce_fn=lambda a, b: a + b)
new_counts = expr.ndarray((self.n_clusters, 1), dtype=np.int,
reduce_fn=lambda a, b: a + b)
_ = expr.shuffle(X,
_find_cluster_mapper,
kw={'d_pts': X,
'old_centers': centers,
'new_centers': new_centers,
'new_counts': new_counts,
'labels': labels},
shape_hint=(1,),
cost_hint={hash(labels): {'00': 0,
'01': np.prod(labels.shape)}})
_.force()
new_counts = new_counts.glom()
new_centers = new_centers.glom()
# If any centroids don't have any points assigined to them.
zcount_indices = (new_counts == 0).reshape(self.n_clusters)
if np.any(zcount_indices):
# One or more centroids may not have any points assigned to them,
# which results in their position being the zero-vector. We reseed these
# centroids with new random values.
n_points = np.count_nonzero(zcount_indices)
# In order to get rid of dividing by zero.
new_counts[zcount_indices] = 1
new_centers[zcount_indices, :] = np.random.randn(n_points, num_dim)
new_centers = new_centers / new_counts
centers = new_centers
return centers, labels
