def _partial_mp_gammai_sparse(batch, matrix, idx, n, A, B, verbose):
"""Parallel helper function for MP gammai for sparse similarity matrices.
Please do not directly use this function, but invoke via
mutual_proximity_gammai()
"""
log = ConsoleLogging()
S_mp = lil_matrix((len(batch), n), dtype=np.float32)
for i, b in enumerate(batch):
if verbose and ((batch[i]+1)%1000 == 0 or batch[i]+1 == n
or i == len(batch)-1 or i == 0):
log.message("MP_gammai_sparse: {} of {}. On {}.".format(
batch[i]+1, n, mp.current_process().name, flush=True))
j_idx = slice(b+1, n)
if b+1 >= n:
continue # nothing to do in the last row
#if j_idx.size == 0:
# continue
# avoiding fancy indexing for efficiency reasons
S_ij = matrix[b, j_idx].toarray().ravel() #Extract dense rows temporarily
p1 = _local_gamcdf(S_ij, A[b], B[b])
del S_ij
S_ji = matrix[j_idx, b].toarray().ravel() #for vectorization below.
p2 = _local_gamcdf(S_ji, A[j_idx], B[j_idx])
del S_ji
val = (p1 * p2).ravel()
S_mp[i, j_idx] = val
#need to mirror later!!
return batch, S_mp
def _partial_mp_emp_sparse(batch, matrix, idx, n, verbose):
"""Parallel helper function for MP empiric for sparse similarity matrices.
Please do not directly use this function, but invoke via
mutual_proximity_empiric()
"""
log = ConsoleLogging()
S_mp = lil_matrix((len(batch), n), dtype=np.float32)
# TODO implement faster version from serial MP emp sparse
for i, b in enumerate(batch):
if verbose and ((batch[i]+1)%1000 == 0 or batch[i] == n-1
or i == len(batch)-1 or i == 0):
log.message("MP_empiric_sparse: {} of {}. On {}.".format(
batch[i]+1, n, mp.current_process().name), flush=True)
for j in range(b+1, n):
d = matrix[b, j]
if d > 0:
dI = matrix.getrow(b).toarray()
dJ = matrix.getrow(j).toarray()
# non-zeros elements
nz = (dI > 0) & (dJ > 0)
S_mp[i, j] = (nz & (dI <= d) & (dJ <= d)).sum() / (nz.sum() - 1)
# need to mirror later
else:
pass # skip zero entries
return (batch, S_mp)
def _partial_mp_gaussi_sparse(batch, matrix, idx, n, mu, sd, verbose):
"""Parallel helper function for MP gaussi for sparse similarity matrices.
Please do not directly use this function, but invoke via
mutual_proximity_gaussi()
"""
log = ConsoleLogging()
Dmp = lil_matrix((len(batch), n), dtype=np.float32)
#non-vectorized code
for i, b in enumerate(batch):
if verbose and ((batch[i]+1)%1000 == 0 or batch[i]+1==n
or i==len(batch)-1 or i==0):
log.message("MP_gaussi_sparse: {} of {}. On {}.".format(
batch[i]+1, n, mp.current_process().name, flush=True))
for j in range(b+1, n):
if matrix[b, j] > 0:
p1 = norm.cdf(matrix[b, j], mu[b], sd[b])
p2 = norm.cdf(matrix[j, b], mu[j], sd[j])
Dmp[i, j] = (p1 * p2).ravel()
return batch, Dmp
def test_message(self):
log = ConsoleLogging()
log.message("Message")
return self