def check_dot(gatype): if 0 == me: print '> Checking dot ...', np.random.seed(12345) # everyone has same seed g_a = create_global_array(gatype) g_b = create_global_array(gatype) a = create_local_a(gatype) b = np.random.random_sample((n,n)) if MIRROR: if 0 == iproc: ga.put(g_b, b) ga.put(g_a, a) else: if 0 == me: ga.put(g_b, b) ga.put(g_a, a) ga.sync() sum1 = np.sum(a*b) sum2 = ga.dot(g_a, g_b) if mismatch(sum1, sum2): ga.error('dot wrong %s != %s' % (sum1, sum2)) if 0 == me: print 'OK' ga.destroy(g_a) ga.destroy(g_b)
def check_scatter(gatype): nptype = ga.dtype(gatype) if 0 == me: print '> Checking scatter (might be slow)...', g_a = create_global_array(gatype) a = create_local_a(gatype) if 0 == me: ga.put(g_a, a) ga.sync() ijv = np.zeros((m,2), dtype=np.int64) v = np.zeros(m, dtype=nptype) random.seed(ga.nodeid()*51 + 1) # different seed for each proc for j in range(10): check = None if MIRROR: check = random.randint(0,lprocs-1) == iproc else: check = random.randint(0,nproc-1) == me if check: for loop in range(m): ijv[loop,:] = (random.randint(0,n-1),random.randint(0,n-1)) v[loop] = ijv[loop,0]+ijv[loop,1] ga.scatter(g_a, v, ijv) for loop in range(m): value = ga.get(g_a, ijv[loop], ijv[loop]+1).flatten() if not v[loop] == value: ga.error('scatter failed') if 0 == me: print 'OK' ga.destroy(g_a)
def check_gather(gatype): if 0 == me: print '> Checking gather (might be slow)...', g_a = create_global_array(gatype) a = create_local_a(gatype) if 0 == me: ga.put(g_a, a) ga.sync() ijv = np.zeros((m,2), dtype=np.int64) random.seed(ga.nodeid()*51 + 1) # different seed for each proc for j in range(10): itmp = None if MIRROR: itmp = random.randint(0,lprocs-1) else: itmp = random.randint(0,nproc-1) if itmp == me: for loop in range(m): ijv[loop,:] = (random.randint(0,n-1),random.randint(0,n-1)) #if ijv[loop,0] > ijv[loop,1]: # ijv[loop,:] = ijv[loop,::-1] # reverse result = ga.gather(g_a, ijv) for loop in range(m): value = ga.get(g_a, ijv[loop], ijv[loop]+1).flatten() if not result[loop] == value: ga.error('gather failed') if 0 == me: print 'OK' ga.destroy(g_a)
def check_get(gatype): """check nloop random gets from each node""" if 0 == me: print '> Checking random get (%d calls)...' % nloop g_a = create_global_array(gatype) a = create_local_a(gatype) if 0 == me: ga.put(g_a, a) ga.sync() nwords = 0 random.seed(ga.nodeid()*51+1) # different seed for each proc for loop in range(nloop): ilo,ihi = random.randint(0, nloop-1),random.randint(0, nloop-1) if ihi < ilo: ilo,ihi = ihi,ilo jlo,jhi = random.randint(0, nloop-1),random.randint(0, nloop-1) if jhi < jlo: jlo,jhi = jhi,jlo nwords += (ihi-ilo+1)*(jhi-jlo+1) ihi += 1 jhi += 1 result = ga.get(g_a, (ilo,jlo), (ihi,jhi)) if not np.all(result == a[ilo:ihi,jlo:jhi]): ga.error('random get failed') if 0 == me and loop % max(1,nloop/20) == 0: print ' call %d node %d checking get((%d,%d),(%d,%d)) total %f' % ( loop, me, ilo, ihi, jlo, jhi, nwords) if 0 == me: print 'OK' ga.destroy(g_a)
def check_put_disjoint(gatype): """each node fills in disjoint sections of the array""" if 0 == me: print '> Checking disjoint put ...', g_a = create_global_array(gatype) a = create_local_a(gatype) inc = (n-1)/20 + 1 ij = 0 for i in range(0,n,inc): for j in range(0,n,inc): check = False if MIRROR: check = ij % lprocs == iproc else: check = ij % nproc == me if check: lo = [i,j] hi = [min(i+inc,n), min(j+inc,n)] piece = a[ga.zip(lo,hi)] ga.put(g_a, piece, lo, hi) # the following check is not part of the original test.F result = ga.get(g_a, lo, hi) if not np.all(result == piece): ga.error("put followed by get failed", 1) ga.sync() ij += 1 ga.sync() # all nodes check all of a b = ga.get(g_a) if not np.all(a == b): ga.error('put failed, exiting') if 0 == me: print 'OK' ga.destroy(g_a)
def check_print_patch(gatype): g_a = create_global_array(gatype) a = create_local_a(gatype) if n > 7: if 0 == me: print '> Checking ga.print_patch --- should match' print a[2:5,2:7] ga.print_patch(g_a, (2,2), (5,7)) ga.destroy(g_a)
def check_zero(gatype): if 0 == me: print '> Checking zero ...', g_a = create_global_array(gatype) ga.zero(g_a) a = ga.get(g_a) if not np.all(a == 0): ga.error('ga.zero failed') if 0 == me: print 'OK' ga.destroy(g_a)
def verify_using_ga(g_a, g_b, g_c): g_v = ga.duplicate(g_c) ga.gemm(False,False,N,N,N,1,g_a,g_b,0,g_v) c = ga.access(g_c) v = ga.access(g_v) if c is not None: val = int(np.abs(np.sum(c-v))>0.0001) else: val = 0 val = ga.gop_add(val) ga.destroy(g_v) return val == 0
def verify_using_ga(g_a, g_b, g_c): g_v = ga.duplicate(g_c) ga.gemm(False, False, N, N, N, 1, g_a, g_b, 0, g_v) c = ga.access(g_c) v = ga.access(g_v) if c is not None: val = int(np.abs(np.sum(c - v)) > 0.0001) else: val = 0 val = ga.gop_add(val) ga.destroy(g_v) return val == 0
def check_copy(gatype): if 0 == me: print '> Checking copy ...', g_a = create_global_array(gatype) g_b = create_global_array(gatype) a = create_local_a(gatype) if 0 == me: ga.put(g_a, a) ga.copy(g_a, g_b) if not np.all(a == ga.get(g_b)): ga.error('copy failed') if 0 == me: print 'OK' ga.destroy(g_a) ga.destroy(g_b)
def check_scale(gatype): if 0 == me: print '> Checking scale ...', g_a = create_global_array(gatype) a = create_local_a(gatype) if 0 == me: ga.put(g_a, a) ga.sync() ga.scale(g_a, 0.123) a *= 0.123 if np.any(np.vectorize(mismatch)(a,ga.get(g_a))): ga.error('add failed') if 0 == me: print 'OK' ga.destroy(g_a)
def TRANSPOSE1D(): # Configure array dimensions. Force an unequal data distribution. dims = [nprocs*TOTALELEMS + nprocs/2] chunk = [TOTALELEMS] # minimum data on each process # create a global array g_a and duplicate it to get g_b g_a = ga.create(ga.C_INT, dims, "array A", chunk) if not g_a: ga.error("create failed: A") if not me: print "Created Array A" g_b = ga.duplicate(g_a, "array B") if not g_b: ga.error("duplicate failed") if not me: print "Created Array B" # initialize data in g_a if not me: print "Initializing matrix A" ga.put(g_a, np.arange(dims[0], dtype=np.int32)) # Synchronize all processors to guarantee that everyone has data # before proceeding to the next step. ga.sync() # Start initial phase of inversion by inverting the data held locally on # each processor. Start by finding out which data each processor owns. lo,hi = ga.distribution(g_a) # Get locally held data and copy it into local buffer a a = ga.get(g_a, lo, hi) # Invert data locally b = a[::-1] # Invert data globally by copying locally inverted blocks into # their inverted positions in the GA ga.put(g_b, b, dims[0]-hi[0], dims[0]-lo[0]) # Synchronize all processors to make sure inversion is complete ga.sync() # Check to see if inversion is correct if not me: verify(g_a, g_b) # Deallocate arrays ga.destroy(g_a) ga.destroy(g_b)
def TRANSPOSE1D(): # Configure array dimensions. Force an unequal data distribution. dims = [nprocs * TOTALELEMS + nprocs / 2] chunk = [TOTALELEMS] # minimum data on each process # create a global array g_a and duplicate it to get g_b g_a = ga.create(ga.C_INT, dims, "array A", chunk) if not g_a: ga.error("create failed: A") if not me: print "Created Array A" g_b = ga.duplicate(g_a, "array B") if not g_b: ga.error("duplicate failed") if not me: print "Created Array B" # initialize data in g_a if not me: print "Initializing matrix A" ga.put(g_a, np.arange(dims[0], dtype=np.int32)) # Synchronize all processors to guarantee that everyone has data # before proceeding to the next step. ga.sync() # Start initial phase of inversion by inverting the data held locally on # each processor. Start by finding out which data each processor owns. lo, hi = ga.distribution(g_a) # Get locally held data and copy it into local buffer a a = ga.get(g_a, lo, hi) # Invert data locally b = a[::-1] # Invert data globally by copying locally inverted blocks into # their inverted positions in the GA ga.put(g_b, b, dims[0] - hi[0], dims[0] - lo[0]) # Synchronize all processors to make sure inversion is complete ga.sync() # Check to see if inversion is correct if not me: verify(g_a, g_b) # Deallocate arrays ga.destroy(g_a) ga.destroy(g_b)
def verify(g_a, g_b, g_c): g_chk = ga.duplicate(g_a, "array check") if not g_chk: ga.error("duplicate failed") ga.sync() ga.gemm(False, False, TOTALELEMS, TOTALELEMS, TOTALELEMS, 1.0, g_a, g_b, 0.0, g_chk); ga.sync() ga.add(g_c, g_chk, g_chk, 1.0, -1.0) rchk = ga.dot(g_chk, g_chk) if not me: print "Normed difference in matrices: %12.4f" % rchk if not (-TOLERANCE < rchk < TOLERANCE): ga.error("Matrix multiply verify failed") else: print "Matrix Multiply OK" ga.destroy(g_chk)
def check_add(gatype): if 0 == me: print '> Checking add ...', g_a = create_global_array(gatype) g_b = create_global_array(gatype) a = create_local_a(gatype) b = create_local_b(gatype) alpha = None beta = None if 0 == me: ga.put(g_a, a) ga.sync(); np.random.seed(12345) # everyone has same seed if gatype in [ga.C_SCPL,ga.C_DCPL]: b_real = np.random.random_sample((n,n)) b_imag = np.random.random_sample((n,n)) b[:] = np.vectorize(complex)(b_real,b_imag) alpha = complex(0.1,-0.1) beta = complex(0.9,-0.9) else: b[:] = np.random.random_sample((n,n)) alpha = 0.1 beta = 0.9 a = alpha*a + beta*b if MIRROR: if 0 == iproc: ga.put(g_b, b) else: if 0 == me: ga.put(g_b, b) ga.sync() ga.add(g_a, g_b, g_b, alpha, beta) b = ga.get(g_b, buffer=b) if np.any(np.vectorize(mismatch)(b,a)): ga.error('add failed') if 0 == me: print 'OK' ga.destroy(g_a) ga.destroy(g_b)
def check_accumulate_overlap(gatype): if 0 == me: print '> Checking overlapping accumulate ...', g_a = create_global_array(gatype) ga.zero(g_a) ga.acc(g_a, [1], (n/2,n/2), (n/2+1,n/2+1), 1) ga.sync() if MIRROR: if 0 == iproc: x = abs(ga.get(g_a, (n/2,n/2), (n/2+1,n/2+1))[0,0] - lprocs) if not 0 == x: ga.error('overlapping accumulate failed -- expected %s got %s'%( x, lprocs)) else: if 0 == me: x = abs(ga.get(g_a, (n/2,n/2), (n/2+1,n/2+1))[0,0] - nproc) if not 0 == x: ga.error('overlapping accumulate failed -- expected %s got %s'%( x, nproc)) if 0 == me: print 'OK' ga.destroy(g_a)
def check_accumulate_disjoint(gatype): """Each node accumulates into disjoint sections of the array.""" if 0 == me: print '> Checking disjoint accumulate ...', g_a = create_global_array(gatype) a = create_local_a(gatype) b = np.fromfunction(lambda i,j: i+j+2, (n,n), dtype=ga.dtype(gatype)) if 0 == me: ga.put(g_a, a) ga.sync() inc = (n-1)/20 + 1 ij = 0 for i in range(0,n,inc): for j in range(0,n,inc): x = 10.0 lo = [i,j] hi = [min(i+inc,n), min(j+inc,n)] piece = b[ga.zip(lo,hi)] check = False if MIRROR: check = ij % lprocs == iproc else: check = ij % nproc == me if check: ga.acc(g_a, piece, lo, hi, x) ga.sync() ij += 1 # each process applies all updates to its local copy a[ga.zip(lo,hi)] += x * piece ga.sync() # all nodes check all of a if not np.all(ga.get(g_a) == a): ga.error('acc failed') if 0 == me: print 'OK' ga.destroy(g_a)
def matrix_multiply(): # Configure array dimensions. Force an unequal data distribution. dims = [TOTALELEMS]*NDIM chunk = [TOTALELEMS/nprocs-1]*NDIM # Create a global array g_a and duplicate it to get g_b and g_c. g_a = ga.create(ga.C_DBL, dims, "array A", chunk) if not g_a: ga.error("create failed: A") if not me: print "Created Array A" g_b = ga.duplicate(g_a, "array B") g_c = ga.duplicate(g_a, "array C") if not g_b or not g_c: ga.eror("duplicate failed") if not me: print "Created Arrays B and C" # Initialize data in matrices a and b. if not me: print "Initializing matrix A and B" a = np.random.rand(*dims)*29 b = np.random.rand(*dims)*37 # Copy data to global arrays g_a and g_b. if not me: ga.put(g_a, a) ga.put(g_b, b) # Synchronize all processors to make sure everyone has data. ga.sync() # Determine which block of data is locally owned. Note that # the same block is locally owned for all GAs. lo,hi = ga.distribution(g_c) # Get the blocks from g_a and g_b needed to compute this block in # g_c and copy them into the local buffers a and b. a = ga.get(g_a, (lo[0],0), (hi[0],dims[0])) b = ga.get(g_b, (0,lo[1]), (dims[1],hi[1])) # Do local matrix multiplication and store the result in local # buffer c. Start by evaluating the transpose of b. btrns = b.transpose() # Multiply a and b to get c. c = np.dot(a,b) # Copy c back to g_c. ga.put(g_c, c, lo, hi) verify(g_a, g_b, g_c) # Deallocate arrays. ga.destroy(g_a) ga.destroy(g_b) ga.destroy(g_c)
# put some fake data into input arrays A and B if me == 0: ga.put(g_a, np.random.random(N*N)) ga.put(g_b, np.random.random(N*N)) ga.sync() if me == 0: print "srumma...", srumma(g_a, g_b, g_c, CHUNK_SIZE, MULTIPLIER, g_counter) if me == 0: print "done" if me == 0: print "verifying using ga.gemm...", ok = verify_using_ga(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" if me == 0: print "verifying using np.dot...", ok = verify_using_np(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" ga.destroy(g_a) ga.destroy(g_b) ga.destroy(g_c) ga.destroy(g_counter)
nprocs = ga.nnodes() myrank = ga.nodeid() g_pi = ga.create(ga.C_DBL, [1]) one_time = False if len(sys.argv) == 2: n = int(sys.argv[1]) one_time = True while True: if not one_time: if myrank == 0: n = get_n() n = ga.brdcst(n) else: n = ga.brdcst(0) if n == 0: break ga.zero(g_pi) mypi = comp_pi(n, myrank, nprocs) ga.acc(g_pi, mypi) ga.sync() if myrank == 0: pi = ga.get(g_pi)[0] prn_pi(pi, PI) if one_time: break ga.destroy(g_pi)
g_c = ga.create(ga.C_DBL, [N,N]) # put some fake data into input arrays A and B if me == 0: ga.put(g_a, np.random.random(N*N)) ga.put(g_b, np.random.random(N*N)) ga.sync() if me == 0: print "srumma...", srumma(g_a, g_b, g_c, CHUNK_SIZE, MULTIPLIER) if me == 0: print "done" if me == 0: print "verifying using ga.gemm...", ok = verify_using_ga(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" if me == 0: print "verifying using np.dot...", ok = verify_using_np(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" ga.destroy(g_a) ga.destroy(g_b) ga.destroy(g_c)
data = None comm.Gather(np.array(out[1:], dtype=float), data, root=0) start7 = time.time() if rank == 0 and int(j) == 1: res = os.path.abspath( os.path.normpath(os.path.join(os.getcwd(), 'RMSD_{}.csv'.format(size)))) np.save(res, buf) os.remove('files/newtraj_{}_{}.xtc'.format(rank, j)) os.remove('files/.newtraj_{}_{}.xtc_offsets.npz'.format(rank, j)) ga.destroy(g_a) # Cost Calculation init_time = start2 - start1 comm_time1 = start3 - start2 comm_time2 = start5 - start4 comm_time3 = start7 - start6 comp_time = start4 - start3 access_time = start6 - start5 tot_time = comp_time + comm_time2 + access_time tot_time = comm.gather(tot_time, root=0) init_time = comm.gather(init_time, root=0) comm_time1 = comm.gather(comm_time1, root=0) comm_time2 = comm.gather(comm_time2, root=0) comm_time3 = comm.gather(comm_time3, root=0)
g_c = ga.create(ga.C_DBL, [N, N]) # put some fake data into input arrays A and B if me == 0: ga.put(g_a, np.random.random(N * N)) ga.put(g_b, np.random.random(N * N)) ga.sync() if me == 0: print "srumma...", srumma(g_a, g_b, g_c, CHUNK_SIZE, MULTIPLIER) if me == 0: print "done" if me == 0: print "verifying using ga.gemm...", ok = verify_using_ga(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" if me == 0: print "verifying using np.dot...", ok = verify_using_np(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" ga.destroy(g_a) ga.destroy(g_b) ga.destroy(g_c)
# put some fake data into input arrays A and B if me == 0: ga.put(g_a, np.random.random(N * N)) ga.put(g_b, np.random.random(N * N)) ga.sync() if me == 0: print "srumma...", srumma(g_a, g_b, g_c, CHUNK_SIZE, MULTIPLIER, g_counter) if me == 0: print "done" if me == 0: print "verifying using ga.gemm...", ok = verify_using_ga(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" if me == 0: print "verifying using np.dot...", ok = verify_using_np(g_a, g_b, g_c) if me == 0: if ok: print "OKAY" else: print "FAILED" ga.destroy(g_a) ga.destroy(g_b) ga.destroy(g_c) ga.destroy(g_counter)