print "Total force on floor=", O.forces.f(WALL_ID)[1] else: ####### MPI ###### #import yade's mpi module from yade import mpy as mp # customize mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID) mp.VERBOSE_OUTPUT = False mp.ERASE_REMOTE = True #erase bodies not interacting wit a given subdomain? mp.OPTIMIZE_COM = True #L1-optimization: pass a list of double instead of a list of states mp.USE_CPP_MPI = True and mp.OPTIMIZE_COM #L2-optimization: workaround python by passing a vector<double> at the c++ level if rank == 0: start = time.time() mp.mpirun(NSTEPS, merge) if rank == 0: end = time.time() print "num. bodies:", len([b for b in O.bodies]), len(O.bodies) if rank == 0: mp.mprint("Total force on floor=" + str(O.forces.f(WALL_ID)[1])) collectTimingMPI() else: mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1])) #if merge: mp.mergeScene() #if rank==0: O.save('mergedScene.yade') mp.MPI.Finalize() if rank == 0: wallTime = end - start created = os.path.isfile("wallTimes_2D_weakScalability-PREHACK-" + timeStr + ".dat") f = open('wallTimes_2D_weakScalability-PREHACK-' + timeStr + '.dat', 'a') if not created: f.write("numThreads mpi omp Nspheres wallTime merge\n") f.write("%g %s %s %g %g %g\n" % (numThreads, str(os.getenv('OMPI_COMM_WORLD_SIZE')),
timeStepper ) #remove the automatic timestepper. Very important: we don't want subdomains to use many different timesteps... O.engines = O.engines[0:tsIdx] + O.engines[tsIdx + 1:] O.dt = 0.001 #this very small timestep will make it possible to run 2000 iter without merging #O.dt=0.1*PWaveTimeStep() #very important, we don't want subdomains to use many different timesteps... # customize mpy mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID) mp.VERBOSE_OUTPUT = False mp.ERASE_REMOTE = False #erase bodies not interacting wit a given subdomain? mp.OPTIMIZE_COM = True #L1-optimization: pass a list of double instead of a list of states mp.USE_CPP_MPI = True and mp.OPTIMIZE_COM #L2-optimization: workaround python by passing a vector<double> at the c++ level mp.MERGE_W_INTERACTIONS = False mp.COPY_MIRROR_BODIES_WHEN_COLLIDE = True mp.VERBOSE_OUTPUT = False mp.YADE_TIMING = False mp.NO_OUTPUT = True mp.mpirun(NSTEPS, numThreads, True) mp.mprint("num. bodies:", len([b for b in O.bodies])) mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1])) Ek = 0 if mp.rank == 0: Ek = kineticEnergy() mp.mprint("got Ek=", Ek) refEk = 1120803.9955506378 if (abs(Ek - refEk) / refEk) > 1e-10: raise YadeCheckError("kinetic energy changed by" + str((Ek - refEk) / refEk))
sp_new = [] for i in range(0, len(sp)): sp_new.append(sp[indices[i]]) Nspheres = 122000 sp_new = sp_new[0:Nspheres] from yade import ymport facets = ymport.stl(fileName + '.stl', color=(0, 1, 0), material=Steel) fctIds = range(len(facets)) NSTEPS = 1000 if numMPIThreads > 1: mp.mprint("appending bodies, rank", mp.rank) if mp.rank == 0: O.bodies.append(facets) mp.mprint("master has", len(O.bodies), "facets") else: import numpy as np layers = np.array_split(sp_new, numMPIThreads - 1) mp.mprint("layers", [len(l) for l in layers]) layerNo = mp.rank - 1 #rank zero is for facets nextId = int(len(facets) + np.sum([len(x) for x in layers[:layerNo]])) mp.mprint("s", s, "startId", nextId) for s in layers[layerNo]: s.subdomain = mp.rank O.bodies.insertAtId(s, nextId) nextId += 1
print("num. bodies:", len([b for b in O.bodies]), len(O.bodies)) print("Total force on floor=", O.forces.f(WALL_ID)[1]) else: ####### MPI ###### #import yade's mpi module from yade import mpy as mp # customize mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID) mp.VERBOSE_OUTPUT = False mp.MAX_RANK_OUTPUT = 4 mp.mpirun( 1 ) #this is to eliminate initialization overhead in Cundall number and timings from yade import timing timing.reset() t1 = time.time() mp.mpirun(NSTEPS) t2 = time.time() mp.mprint("num. bodies:", len([b for b in O.bodies]), " ", len(O.bodies)) if rank == 0: mp.mprint("Total force on floor=" + str(O.forces.f(WALL_ID)[1])) mp.mprint("CPU wall time for ", NSTEPS, " iterations:", t2 - t1, "; Cundall number = ", len(O.bodies) * NSTEPS / (t2 - t1)) collectTiming() else: mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1])) #mp.mergeScene() #if rank==0: O.save('mergedScene.yade') mp.MPI.Finalize() exit()
O.dynDt = False #import yade's mpi module #from yade import mpy as mp # customize mp.VERBOSE_OUTPUT = False mp.DISTRIBUTED_INSERT = True mp.REALLOCATE_FREQUENCY = 4 mp.MAX_RANK_OUTPUT = 6 mp.mpirun( 1, numThreads, False ) #this is to eliminate initialization overhead in Cundall number and timings mp.YADE_TIMING = True t1 = time.time() mp.mpirun(NSTEPS, withMerge=False) t2 = time.time() mp.mprint("num. bodies:", len([b for b in O.bodies if b.subdomain == mp.rank]), "/", len(O.bodies)) def collectTiming(Cundall): created = os.path.isfile("collect3D.dat") f = open('collect3D.dat', 'a') if not created: f.write("CuNumber numThreads mpi omp Nspheres L M N runtime \n") from yade import timing f.write( str(Cundall) + " " + str(numThreads * int(os.getenv('OMP_NUM_THREADS'))) + " " + str(os.getenv('OMPI_COMM_WORLD_SIZE')) + " " + os.getenv('OMP_NUM_THREADS') + " " + str(N * M * L * (numThreads - 1)) + " " + str(L) + " " + str(M) + " " + str(N) + " " + str(timing.runtime()) + "\n")
O.engines=[ ForceResetter(), InsertionSortCollider([Bo1_Sphere_Aabb(), Bo1_Box_Aabb()], label='collider', allowBiggerThanPeriod=True), InteractionLoop( [Ig2_Sphere_Sphere_ScGeom(), Ig2_Box_Sphere_ScGeom()], [Ip2_FrictMat_FrictMat_FrictPhys()], [Law2_ScGeom_FrictPhys_CundallStrack()] ,label='InteractionLoop'), GlobalStiffnessTimeStepper(timestepSafetyCoefficient=0.7, timeStepUpdateInterval=200, parallelMode=True, label = "ts"), fluidCoupling, #to be called after timestepper NewtonIntegrator(damping = 0.0, label='newton', gravity = (0, 0.0, 0)), VTKRecorder(fileName='spheres/3d-vtk-', recorders=['all'], parallelMode=True, iterPeriod=1000) ] collider.verletDist =0.00075 mp.YADE_TIMING = False mp.FLUID_COUPLING = True mp.VERBOSE_OUTPUT= False mp.USE_CPP_INTERS = False mp.ERASE_REMOTE_MASTER = True mp.REALLOC_FREQUENCY = 12 mp.fluidBodies = sphereIDs mp.DOMAIN_DECOMPOSITION= True mp.mpirun(NSTEPS) mp.mprint("RUN FINISH") fluidCoupling.killMPI() exit() #mp.MPI.Finalize() #mp.mergeScene() #if mp.rank == 0: O.save('mergedScene.yade')
mp.MERGE_SPLIT = True mp.COPY_MIRROR_BODIES_WHEN_COLLIDE = False mp.WALL_ID = WALL_ID mp.mpirun(NSTEPS) print("num. bodies:", len([b for b in O.bodies]), len(O.bodies)) if rank == 0: fl = open('intrs_parallel.txt', 'w') for i in O.interactions: a = i.id1 b = i.id2 if a > b: a, b = b, a fl.write('%s %s\n' % (a, b)) fl.close() collectTiming() mp.mprint( "Total force on floor based on FORCES RECVD FROM WORKERS =" + str(O.forces.f(WALL_ID)[1])) else: mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1])) mp.mergeScene() if rank == 0: # just for saving and checking the state after merge. # O.save('mergedScene.yade') print("force recieved from workers = ", O.forces.f(WALL_ID)[1]) #O.forces.reset() #collider.__call__() ##print "num interactions = " , len(O.interactions) #O.step() #mp.mprint( "Total force on floor based on inters ="+str(O.forces.f(WALL_ID)[1]))
# 2018 © Bruno Chareyre <*****@*****.**> # Really a helloWorld (or almost, I couldn't refrain from adding a bit more). # Check other examples for concrete usage of mpy in DEM simulations from yade import mpy as mp mp.initialize(3) #mp.VERBOSE_OUTPUT = True # to see more of what happens behind the scene mp.mprint("I'm here") if mp.rank==0: # say hello: mp.sendCommand(executors=[1,2],command="mprint('Yes I am really here')",wait=False) # get retrun values if wait=True (blocking comm., think twice) print( mp.sendCommand(executors="all",command="len(O.bodies)",wait=True) ) ## exploit sendCommand to send data between yade instances directly with mpi4py (see mpi4py documentation) ## in the message we actually tell the worker to wait another message (nested comm), but the second one ## uses underlying mpi4py, and it handles pickable objects mp.sendCommand(executors=1,command="message=comm.recv(source=0); mprint('received: ',message)",wait=False) mp.comm.send("hello",dest=1) ## pickable objects ## pay attention to pointer adresses, they are different! (as expected) ## this is moving data around between independent parts of memory mp.sendCommand(executors=1,command="O.bodies.append(Body()); O.bodies[0].shape=Sphere(radius=0.456); comm.send(O.bodies[0],dest=0); mprint('sent a ',O.bodies[0].shape)",wait=False) bodyFrom1 = mp.comm.recv(source=1) mp.mprint("received a ",bodyFrom1.shape,"with radius=",bodyFrom1.shape.radius)