def test_distributed_array_bincount_gaps(comm):
from nbodykit.utils import DistributedArray, EmptyRank
data = numpy.array(
comm.scatter([
numpy.array([
0,
1,
], 'i4'),
numpy.array([
3,
3,
3,
3,
], 'i4'),
numpy.array([], 'i4'),
numpy.array([5, 5], 'i4'),
]))
da = DistributedArray(data, comm)
N = da.bincount(shared_edges=True)
assert_array_equal(numpy.concatenate(comm.allgather(N.local)),
[1, 1, 0, 4, 0, 2])
N = da.bincount(shared_edges=False)
assert_array_equal(numpy.concatenate(comm.allgather(N.local)),
[1, 1, 0, 4, 0, 2])
def test_distributed_array_bincount(comm):
from nbodykit.utils import DistributedArray, EmptyRank
data = numpy.array(
comm.scatter([
numpy.array([
0,
1,
2,
3,
], 'i4'),
numpy.array([
3,
3,
3,
3,
], 'i4'),
numpy.array([], 'i4'),
numpy.array([3, 5], 'i4'),
]))
da = DistributedArray(data, comm)
N = da.bincount()
assert_array_equal(numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 6, 6, 0, 1])
weights = numpy.ones_like(data)
N = da.bincount(weights)
assert_array_equal(numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 6, 6, 0, 1])
N = da.bincount(weights, shared_edges=False)
assert_array_equal(numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 0, 1])
def getinsat(self, mHIsat, satid, totalsize, localsize, comm):
da = DistributedArray(satid, comm)
mHI = da.bincount(mHIsat, shared_edges=False)
zerosize = totalsize - mHI.cshape[0]
zeros = DistributedArray.cempty(cshape=(zerosize, ), dtype=mHI.local.dtype, comm=comm)
zeros.local[...] = 0
mHItotal = DistributedArray.concat(mHI, zeros, localsize=localsize)
return mHItotal
def test_distributed_array_bincount_gaps(comm):
from nbodykit.utils import DistributedArray, EmptyRank
data = numpy.array(comm.scatter(
[numpy.array([0, 1, ], 'i4'),
numpy.array([3, 3, 3, 3, ], 'i4'),
numpy.array([], 'i4'),
numpy.array([5, 5], 'i4'),
]))
da = DistributedArray(data, comm)
N = da.bincount(shared_edges=True)
assert_array_equal( numpy.concatenate(comm.allgather(N.local)),
[1, 1, 0, 4, 0, 2])
N = da.bincount(shared_edges=False)
assert_array_equal( numpy.concatenate(comm.allgather(N.local)),
[1, 1, 0, 4, 0, 2])
def test_distributed_array_bincount(comm):
from nbodykit.utils import DistributedArray, EmptyRank
data = numpy.array(comm.scatter(
[numpy.array([0, 1, 2, 3, ], 'i4'),
numpy.array([3, 3, 3, 3, ], 'i4'),
numpy.array([], 'i4'),
numpy.array([3, 5], 'i4'),
]))
da = DistributedArray(data, comm)
N = da.bincount()
assert_array_equal( numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 6, 6, 0, 1])
weights = numpy.ones_like(data)
N = da.bincount(weights)
assert_array_equal( numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 6, 6, 0, 1])
N = da.bincount(weights, shared_edges=False)
assert_array_equal( numpy.concatenate(comm.allgather(N.local)),
[1, 1, 1, 6, 0, 1])
def getinsat(self, mHIsat, satid, totalsize, localsize, comm):
#print(comm.rank, np.all(np.diff(satid) >=0))
#diff = np.diff(satid)
#if comm.rank == 260:
# print(satid[:-1][diff <0], satid[1:][diff < 0])
da = DistributedArray(satid, comm)
mHI = da.bincount(mHIsat, shared_edges=False)
zerosize = totalsize - mHI.cshape[0]
zeros = DistributedArray.cempty(cshape=(zerosize, ), dtype=mHI.local.dtype, comm=comm)
zeros.local[...] = 0
mHItotal = DistributedArray.concat(mHI, zeros, localsize=localsize)
return mHItotal
(aa, suff))
sat = BigFileCatalog(myscratch + sim + '/fastpm_%0.4f/satcat-%s/' %
(aa, suff))
##
hmass = halos['Length'].compute() * mp
cmass = cen["Mass"].compute()
chmass = cen["HaloMass"].compute()
smass = sat["Mass"].compute()
hpos, cpos, spos = halos['Position'].compute(
), cen['Position'].compute(), sat['Position'].compute()
chid, shid = cen['GlobalID'].compute(), sat['GlobalID'].compute()
cnsat = cen['Nsat'].compute()
da = DistributedArray(shid, comm)
N = da.bincount(shared_edges=False)
print('rank, shid, N : ', rank, shid[:10], N.local[:10])
print('rank, chid, csat, N : ', rank, chid[:10], cnsat[:10],
N.local[:10])
print('rank, cen.csize, N.cshape : ', rank, cen.csize, N.cshape)
print('rank, cen.size, Nlocal.size : ', rank, cen.size, N.local.size)
print(cen.csize - N.cshape)
zerosize = (cen.csize - N.cshape[0])
#start = (zerosize *rank // wsize)
#end = (zerosize *(rank+1) // wsize)
#zeros = DistributedArray(np.zeros(end-start), comm=comm)
print(zerosize, N.local.dtype)
zeros = DistributedArray.cempty(cshape=(zerosize, ),
dtype=N.local.dtype,