def test_distributed_array_bincount_gaps(comm): from nbodykit.utils import DistributedArray, EmptyRank data = numpy.array( comm.scatter([ numpy.array([ 0, 1, ], 'i4'), numpy.array([ 3, 3, 3, 3, ], 'i4'), numpy.array([], 'i4'), numpy.array([5, 5], 'i4'), ])) da = DistributedArray(data, comm) N = da.bincount(shared_edges=True) assert_array_equal(numpy.concatenate(comm.allgather(N.local)), [1, 1, 0, 4, 0, 2]) N = da.bincount(shared_edges=False) assert_array_equal(numpy.concatenate(comm.allgather(N.local)), [1, 1, 0, 4, 0, 2])
def test_distributed_array_bincount(comm): from nbodykit.utils import DistributedArray, EmptyRank data = numpy.array( comm.scatter([ numpy.array([ 0, 1, 2, 3, ], 'i4'), numpy.array([ 3, 3, 3, 3, ], 'i4'), numpy.array([], 'i4'), numpy.array([3, 5], 'i4'), ])) da = DistributedArray(data, comm) N = da.bincount() assert_array_equal(numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 6, 6, 0, 1]) weights = numpy.ones_like(data) N = da.bincount(weights) assert_array_equal(numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 6, 6, 0, 1]) N = da.bincount(weights, shared_edges=False) assert_array_equal(numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 0, 1])
def getinsat(self, mHIsat, satid, totalsize, localsize, comm): da = DistributedArray(satid, comm) mHI = da.bincount(mHIsat, shared_edges=False) zerosize = totalsize - mHI.cshape[0] zeros = DistributedArray.cempty(cshape=(zerosize, ), dtype=mHI.local.dtype, comm=comm) zeros.local[...] = 0 mHItotal = DistributedArray.concat(mHI, zeros, localsize=localsize) return mHItotal
def test_distributed_array_bincount_gaps(comm): from nbodykit.utils import DistributedArray, EmptyRank data = numpy.array(comm.scatter( [numpy.array([0, 1, ], 'i4'), numpy.array([3, 3, 3, 3, ], 'i4'), numpy.array([], 'i4'), numpy.array([5, 5], 'i4'), ])) da = DistributedArray(data, comm) N = da.bincount(shared_edges=True) assert_array_equal( numpy.concatenate(comm.allgather(N.local)), [1, 1, 0, 4, 0, 2]) N = da.bincount(shared_edges=False) assert_array_equal( numpy.concatenate(comm.allgather(N.local)), [1, 1, 0, 4, 0, 2])
def test_distributed_array_bincount(comm): from nbodykit.utils import DistributedArray, EmptyRank data = numpy.array(comm.scatter( [numpy.array([0, 1, 2, 3, ], 'i4'), numpy.array([3, 3, 3, 3, ], 'i4'), numpy.array([], 'i4'), numpy.array([3, 5], 'i4'), ])) da = DistributedArray(data, comm) N = da.bincount() assert_array_equal( numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 6, 6, 0, 1]) weights = numpy.ones_like(data) N = da.bincount(weights) assert_array_equal( numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 6, 6, 0, 1]) N = da.bincount(weights, shared_edges=False) assert_array_equal( numpy.concatenate(comm.allgather(N.local)), [1, 1, 1, 6, 0, 1])
def getinsat(self, mHIsat, satid, totalsize, localsize, comm): #print(comm.rank, np.all(np.diff(satid) >=0)) #diff = np.diff(satid) #if comm.rank == 260: # print(satid[:-1][diff <0], satid[1:][diff < 0]) da = DistributedArray(satid, comm) mHI = da.bincount(mHIsat, shared_edges=False) zerosize = totalsize - mHI.cshape[0] zeros = DistributedArray.cempty(cshape=(zerosize, ), dtype=mHI.local.dtype, comm=comm) zeros.local[...] = 0 mHItotal = DistributedArray.concat(mHI, zeros, localsize=localsize) return mHItotal
(aa, suff)) sat = BigFileCatalog(myscratch + sim + '/fastpm_%0.4f/satcat-%s/' % (aa, suff)) ## hmass = halos['Length'].compute() * mp cmass = cen["Mass"].compute() chmass = cen["HaloMass"].compute() smass = sat["Mass"].compute() hpos, cpos, spos = halos['Position'].compute( ), cen['Position'].compute(), sat['Position'].compute() chid, shid = cen['GlobalID'].compute(), sat['GlobalID'].compute() cnsat = cen['Nsat'].compute() da = DistributedArray(shid, comm) N = da.bincount(shared_edges=False) print('rank, shid, N : ', rank, shid[:10], N.local[:10]) print('rank, chid, csat, N : ', rank, chid[:10], cnsat[:10], N.local[:10]) print('rank, cen.csize, N.cshape : ', rank, cen.csize, N.cshape) print('rank, cen.size, Nlocal.size : ', rank, cen.size, N.local.size) print(cen.csize - N.cshape) zerosize = (cen.csize - N.cshape[0]) #start = (zerosize *rank // wsize) #end = (zerosize *(rank+1) // wsize) #zeros = DistributedArray(np.zeros(end-start), comm=comm) print(zerosize, N.local.dtype) zeros = DistributedArray.cempty(cshape=(zerosize, ), dtype=N.local.dtype,