def test_avg_sequential_3_4_1(): ''' CubeMPI for AVG: Exact squential values (ne=3, ngq=4, nproc=1) ''' ne, ngq = 3, 4 nproc, myrank = 1, 0 cubegrid = CubeGridMPI(ne, ngq, nproc, myrank) cubempi = CubeMPI(cubegrid, method='AVG') a_equal(cubegrid.local_gids, np.arange(6*ne*ne*ngq*ngq)) a_equal(cubempi.recv_schedule.shape, (0,3)) a_equal(cubempi.send_schedule.shape, (0,3)) a_equal(cubempi.recv_buf_size, 6*ne*ne*12) a_equal(cubempi.send_buf_size, 0) #----------------------------------------------------- # Generate a sequential field on the cubed-sphere #----------------------------------------------------- f = np.arange(cubegrid.local_ep_size, dtype='f8') #----------------------------------------------------- # Average the element boundary for the spectral element method #----------------------------------------------------- recv_buf = np.zeros(cubempi.recv_buf_size, 'f8') send_buf = np.zeros(cubempi.send_buf_size, 'f8') pre_send(cubempi, f, recv_buf, send_buf) post_recv(cubempi, f, recv_buf) #----------------------------------------------------- # Check if mvps have same values #----------------------------------------------------- fs = [f] ranks, lids = cubegrid.ranks, cubegrid.lids cs_fpath = 'cs_grid_ne%dngq%d.nc'%(ne, ngq) cs_ncf = nc.Dataset(cs_fpath, 'r', format='NETCDF4') mvps = cs_ncf.variables['mvps'][:] for seq, mvp in enumerate(mvps): eff_mvp = [k for k in mvp if k != -1] for gid in eff_mvp: rank, lid = ranks[gid], lids[gid] ok_( feq(fs[rank][lid], np.mean(eff_mvp), 15) )
def test_jacobian_area_30_2(): ''' CubeTensor: Jacobian area test (ne=30, ngq=4, nproc=2) ''' ne, ngq = 30, 4 nproc = 2 # Rank 0 cubegrid0 = CubeGridMPI(ne, ngq, nproc, myrank=0) cubetensor0 = CubeTensor(cubegrid0) local_ep_size0 = cubegrid0.local_ep_size local_gq_indices0 = cubegrid0.local_gq_indices J0 = cubetensor0.J gq_wts0 = cubetensor0.gq_wts areas0 = np.zeros(local_ep_size0) for seq in xrange(local_ep_size0): panel, ei, ej, gi, gj = local_gq_indices0[seq] areas0[seq] = J0[seq]*gq_wts0[gi-1]*gq_wts0[gj-1] # Rank 1 cubegrid1 = CubeGridMPI(ne, ngq, nproc, myrank=1) cubetensor1 = CubeTensor(cubegrid1) local_ep_size1 = cubegrid1.local_ep_size local_gq_indices1 = cubegrid1.local_gq_indices J1 = cubetensor1.J gq_wts1 = cubetensor1.gq_wts areas1 = np.zeros(local_ep_size1) for seq in xrange(local_ep_size1): panel, ei, ej, gi, gj = local_gq_indices1[seq] areas1[seq] = J1[seq]*gq_wts1[gi-1]*gq_wts1[gj-1] ok_( feq(fsum(areas0)+fsum(areas1), 4*np.pi, 13) )
def test_jacobian_area_30(): ''' CubeTensor: Jacobian area test (ne=30, ngq=4) ''' ne, ngq = 30, 4 nproc, myrank = 1, 0 cubegrid = CubeGridMPI(ne, ngq, nproc, myrank) cubetensor = CubeTensor(cubegrid) local_ep_size = cubegrid.local_ep_size local_gq_indices = cubegrid.local_gq_indices J = cubetensor.J gq_wts = cubetensor.gq_wts areas = np.zeros(local_ep_size) for seq in xrange(local_ep_size): panel, ei, ej, gi, gj = local_gq_indices[seq] areas[seq] = J[seq]*gq_wts[gi-1]*gq_wts[gj-1] #aa_equal(fsum(areas), 4*np.pi, 11) #aa_equal(fsum(areas)/100, 4*np.pi/100, 13) # /100 for accuracy comparison ok_( feq(fsum(areas), 4*np.pi, 13) )
def check_avg_sequential_mpi(ne, ngq, comm): ''' CubeMPI for AVG ''' myrank = comm.Get_rank() nproc = comm.Get_size() cubegrid = CubeGridMPI(ne, ngq, nproc, myrank) cubempi = CubeMPI(cubegrid, method='AVG', comm=None) # Generate a sequential field on the cubed-sphere f = cubegrid.local_gids.astype('f8') # Average the element boundary for the spectral element method send_buf = np.zeros(cubempi.send_buf_size, 'f8') recv_buf = np.zeros(cubempi.recv_buf_size, 'f8') # Send/Recv pre_send(cubempi, f, recv_buf, send_buf) req_send_list = list() req_recv_list = list() for dest, start, size in cubempi.send_schedule: req = comm.Isend(send_buf[start:start+size], dest, 0) req_send_list.append(req) for dest, start, size in cubempi.recv_schedule: req = comm.Irecv(recv_buf[start:start+size], dest, 0) req_recv_list.append(req) MPI.Request.Waitall(req_send_list) MPI.Request.Waitall(req_recv_list) # After receive post_recv(cubempi, f, recv_buf) #----------------------------------------------------- # Check if mvps have same values #----------------------------------------------------- if myrank == 0: fs = [f] for src in xrange(1,nproc): size = cubegrid.local_ep_size fs.append( np.zeros(size, 'f8') ) comm.Recv(fs[-1], src, 10) cs_fpath = 'cs_grid_ne%dngq%d.nc'%(ne, ngq) cs_ncf = nc.Dataset(cs_fpath, 'r', format='NETCDF4') mvps = cs_ncf.variables['mvps'][:] ranks, lids = cubegrid.ranks, cubegrid.lids for seq, mvp in enumerate(mvps): eff_mvp = [k for k in mvp if k != -1] for gid in eff_mvp: rank, lid = ranks[gid], lids[gid] ok_( feq(fs[rank][lid], np.mean(eff_mvp), 15) ) else: comm.Send(f, 0, 10)
def test_avg_sequential_3_4_3(): ''' CubeMPI for AVG: Exact squential values (ne=3, ngq=4, nproc=3) ''' ne, ngq = 3, 4 nproc = 3 cubegrid0 = CubeGridMPI(ne, ngq, nproc, 0) cubempi0 = CubeMPI(cubegrid0, method='AVG') cubegrid1 = CubeGridMPI(ne, ngq, nproc, 1) cubempi1 = CubeMPI(cubegrid1, method='AVG') cubegrid2 = CubeGridMPI(ne, ngq, nproc, 2) cubempi2 = CubeMPI(cubegrid2, method='AVG') # Check send/recv pair in send_group and recv_group a_equal(cubempi0.send_group[1].keys(), cubempi1.recv_group[0]) a_equal(cubempi0.send_group[2].keys(), cubempi2.recv_group[0]) a_equal(cubempi1.send_group[0].keys(), cubempi0.recv_group[1]) a_equal(cubempi1.send_group[2].keys(), cubempi2.recv_group[1]) a_equal(cubempi2.send_group[0].keys(), cubempi0.recv_group[2]) a_equal(cubempi2.send_group[1].keys(), cubempi1.recv_group[2]) # Check send/recv pair in send_buf and recv_buf rank0, i0, n0 = cubempi0.send_schedule[0] # send 0->1 rank1, i1, n1 = cubempi1.recv_schedule[0] # recv a_equal(cubempi0.send_buf[i0:i0+n0], cubempi1.recv_buf[i1:i1+n1]) rank0, i0, n0 = cubempi1.send_schedule[0] # send 1->0 rank1, i1, n1 = cubempi0.recv_schedule[0] # recv a_equal(cubempi1.send_buf[i0:i0+n0], cubempi0.recv_buf[i1:i1+n1]) rank0, i0, n0 = cubempi0.send_schedule[1] # send 0->2 rank1, i1, n1 = cubempi2.recv_schedule[0] # recv a_equal(cubempi0.send_buf[i0:i0+n0], cubempi2.recv_buf[i1:i1+n1]) rank0, i0, n0 = cubempi2.send_schedule[0] # send 2->0 rank1, i1, n1 = cubempi0.recv_schedule[1] # recv a_equal(cubempi2.send_buf[i0:i0+n0], cubempi0.recv_buf[i1:i1+n1]) rank0, i0, n0 = cubempi1.recv_schedule[1] # send 1->2 rank1, i1, n1 = cubempi2.send_schedule[1] # recv a_equal(cubempi1.recv_buf[i0:i0+n0], cubempi2.send_buf[i1:i1+n1]) rank0, i0, n0 = cubempi2.recv_schedule[1] # send 2->1 rank1, i1, n1 = cubempi1.send_schedule[1] # recv a_equal(cubempi2.recv_buf[i0:i0+n0], cubempi1.send_buf[i1:i1+n1]) #----------------------------------------------------- # Generate a sequential field on the cubed-sphere #----------------------------------------------------- f0 = cubegrid0.local_gids.astype('f8') f1 = cubegrid1.local_gids.astype('f8') f2 = cubegrid2.local_gids.astype('f8') #----------------------------------------------------- # Average the element boundary for the spectral element method #----------------------------------------------------- recv_buf0 = np.zeros(cubempi0.recv_buf_size, 'f8') send_buf0 = np.zeros(cubempi0.send_buf_size, 'f8') recv_buf1 = np.zeros(cubempi1.recv_buf_size, 'f8') send_buf1 = np.zeros(cubempi1.send_buf_size, 'f8') recv_buf2 = np.zeros(cubempi2.recv_buf_size, 'f8') send_buf2 = np.zeros(cubempi2.send_buf_size, 'f8') # Prepare to send pre_send(cubempi0, f0, recv_buf0, send_buf0) pre_send(cubempi1, f1, recv_buf1, send_buf1) pre_send(cubempi2, f2, recv_buf2, send_buf2) # Send/Recv rank0, i0, n0 = cubempi0.send_schedule[0] # send 0->1 rank1, i1, n1 = cubempi1.recv_schedule[0] recv_buf1[i1:i1+n1] = send_buf0[i0:i0+n0] rank1, i1, n1 = cubempi1.send_schedule[0] # send 1->0 rank0, i0, n0 = cubempi0.recv_schedule[0] recv_buf0[i0:i0+n0] = send_buf1[i1:i1+n1] rank1, i1, n1 = cubempi0.send_schedule[1] # send 0->2 rank0, i0, n0 = cubempi2.recv_schedule[0] recv_buf2[i0:i0+n0] = send_buf0[i1:i1+n1] rank1, i1, n1 = cubempi2.send_schedule[0] # send 2->0 rank0, i0, n0 = cubempi0.recv_schedule[1] recv_buf0[i0:i0+n0] = send_buf2[i1:i1+n1] rank1, i1, n1 = cubempi1.send_schedule[1] # send 1->2 rank0, i0, n0 = cubempi2.recv_schedule[1] recv_buf2[i0:i0+n0] = send_buf1[i1:i1+n1] rank1, i1, n1 = cubempi2.send_schedule[1] # send 2->1 rank0, i0, n0 = cubempi1.recv_schedule[1] recv_buf1[i0:i0+n0] = send_buf2[i1:i1+n1] # After receive post_recv(cubempi0, f0, recv_buf0) post_recv(cubempi1, f1, recv_buf1) post_recv(cubempi2, f2, recv_buf2) #----------------------------------------------------- # Check if mvps have same values #----------------------------------------------------- fs = [f0, f1, f2] ranks, lids = cubegrid0.ranks, cubegrid0.lids cs_fpath = 'cs_grid_ne%dngq%d.nc'%(ne, ngq) cs_ncf = nc.Dataset(cs_fpath, 'r', format='NETCDF4') mvps = cs_ncf.variables['mvps'][:] for seq, mvp in enumerate(mvps): eff_mvp = [k for k in mvp if k != -1] for gid in eff_mvp: rank, lid = ranks[gid], lids[gid] ok_( feq(fs[rank][lid], np.mean(eff_mvp), 15) )
def check_exact_value_mvp(ne, ngq, dsts, srcs, weights, mvps, gq_indices): ''' Check the exact value on MVP ''' f = np.arange(mvps.shape[0], dtype='f8') check_sparse_matrix_on_mvp(f, dsts, srcs, weights, mvps) ij2seq = dict([(tuple(ij),seq) for seq, ij in enumerate(gq_indices)]) ret = (ij2seq[(1,1,1,1,1)] + ij2seq[(4,ne,1,ngq,1)] + ij2seq[(5,1,ne,1,ngq)])/3 ok_( feq(f[ ij2seq[(1,1,1,1,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(4,ne,1,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,1,ne,1,ngq)] ], ret, 15) ) ret = (ij2seq[(1,1,1,2,1)] + ij2seq[(5,1,ne,2,ngq)])/2 ok_( feq(f[ ij2seq[(1,1,1,2,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,1,ne,2,ngq)] ], ret, 15) ) ret = (ij2seq[(1,1,1,ngq,1)] + ij2seq[(5,1,ne,ngq,ngq)] + ij2seq[(5,2,ne,1,ngq)] + ij2seq[(1,2,1,1,1)])/4 ok_( feq(f[ ij2seq[(1,1,1,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,1,ne,ngq,ngq)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,2,ne,1,ngq)] ], ret, 15) ) ok_( feq(f[ ij2seq[(1,2,1,1,1)] ], ret, 15) ) ret = (ij2seq[(1,1,1,1,2)] + ij2seq[(4,ne,1,ngq,2)])/2 ok_( feq(f[ ij2seq[(1,1,1,1,2)] ], ret, 15) ) ok_( feq(f[ ij2seq[(4,ne,1,ngq,2)] ], ret, 15) ) ret = (ij2seq[(2,1,1,ngq,1)] + ij2seq[(5,ne,ne,ngq,1)] + ij2seq[(5,ne,ne-1,ngq,ngq)] + ij2seq[(2,2,1,1,1)])/4 ok_( feq(f[ ij2seq[(2,1,1,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,ne,ne,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,ne,ne-1,ngq,ngq)] ], ret, 15) ) ok_( feq(f[ ij2seq[(2,2,1,1,1)] ], ret, 15) ) ret = (ij2seq[(2,ne,1,ngq,1)] + ij2seq[(5,ne,1,ngq,1)] + ij2seq[(3,1,1,1,1)])/3 ok_( feq(f[ ij2seq[(2,ne,1,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(5,ne,1,ngq,1)] ], ret, 15) ) ok_( feq(f[ ij2seq[(3,1,1,1,1)] ], ret, 15) )
def test_se_sequential_3_4(): ''' SparseMatrixSE: Exact values on the MVP with sequential numbers (ne=3,ngq=4) ''' ne, ngq = 3, 4 spmat = SparseMatrixSE(ne, ngq) mvps = spmat.cs_ncf.variables['mvps'][:] f = np.arange(mvps.shape[0], dtype='f8') check_sparse_matrix_on_mvp(f, spmat.dsts, spmat.srcs, spmat.weights, mvps) ret = (0 + 467 + 684)/3 ok_( feq(f[0], ret, 15) ) ok_( feq(f[467], ret, 15) ) ok_( feq(f[684], ret, 15) ) ret = (1 + 685)/2 ok_( feq(f[1], ret, 15) ) ok_( feq(f[685], ret, 15) ) ret = (3 + 687 + 700 + 16)/4 ok_( feq(f[3], ret, 15) ) ok_( feq(f[687], ret, 15) ) ok_( feq(f[700], ret, 15) ) ok_( feq(f[16], ret, 15) ) ret = (4 + 471)/2 ok_( feq(f[4], ret, 15) ) ok_( feq(f[471], ret, 15) ) ret = (147 + 707 + 671 + 160)/4 ok_( feq(f[147], ret, 15) ) ok_( feq(f[707], ret, 15) ) ok_( feq(f[671], ret, 15) ) ok_( feq(f[160], ret, 15) ) ret = (179 + 611 + 288)/3 ok_( feq(f[179], ret, 15) ) ok_( feq(f[611], ret, 15) ) ok_( feq(f[288], ret, 15) )