print "Total force on floor=", O.forces.f(WALL_ID)[1]
else: ####### MPI ######
#import yade's mpi module
from yade import mpy as mp
# customize
mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID)
mp.VERBOSE_OUTPUT = False
mp.ERASE_REMOTE = True #erase bodies not interacting wit a given subdomain?
mp.OPTIMIZE_COM = True #L1-optimization: pass a list of double instead of a list of states
mp.USE_CPP_MPI = True and mp.OPTIMIZE_COM #L2-optimization: workaround python by passing a vector<double> at the c++ level
if rank == 0: start = time.time()
mp.mpirun(NSTEPS, merge)
if rank == 0: end = time.time()
print "num. bodies:", len([b for b in O.bodies]), len(O.bodies)
if rank == 0:
mp.mprint("Total force on floor=" + str(O.forces.f(WALL_ID)[1]))
collectTimingMPI()
else:
mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1]))
#if merge: mp.mergeScene()
#if rank==0: O.save('mergedScene.yade')
mp.MPI.Finalize()
if rank == 0:
wallTime = end - start
created = os.path.isfile("wallTimes_2D_weakScalability-PREHACK-" +
timeStr + ".dat")
f = open('wallTimes_2D_weakScalability-PREHACK-' + timeStr + '.dat', 'a')
if not created: f.write("numThreads mpi omp Nspheres wallTime merge\n")
f.write("%g %s %s %g %g %g\n" %
(numThreads, str(os.getenv('OMPI_COMM_WORLD_SIZE')),
timeStepper
) #remove the automatic timestepper. Very important: we don't want subdomains to use many different timesteps...
O.engines = O.engines[0:tsIdx] + O.engines[tsIdx + 1:]
O.dt = 0.001 #this very small timestep will make it possible to run 2000 iter without merging
#O.dt=0.1*PWaveTimeStep() #very important, we don't want subdomains to use many different timesteps...
# customize mpy
mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID)
mp.VERBOSE_OUTPUT = False
mp.ERASE_REMOTE = False #erase bodies not interacting wit a given subdomain?
mp.OPTIMIZE_COM = True #L1-optimization: pass a list of double instead of a list of states
mp.USE_CPP_MPI = True and mp.OPTIMIZE_COM #L2-optimization: workaround python by passing a vector<double> at the c++ level
mp.MERGE_W_INTERACTIONS = False
mp.COPY_MIRROR_BODIES_WHEN_COLLIDE = True
mp.VERBOSE_OUTPUT = False
mp.YADE_TIMING = False
mp.NO_OUTPUT = True
mp.mpirun(NSTEPS, numThreads, True)
mp.mprint("num. bodies:", len([b for b in O.bodies]))
mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1]))
Ek = 0
if mp.rank == 0:
Ek = kineticEnergy()
mp.mprint("got Ek=", Ek)
refEk = 1120803.9955506378
if (abs(Ek - refEk) / refEk) > 1e-10:
raise YadeCheckError("kinetic energy changed by" +
str((Ek - refEk) / refEk))
sp_new = []
for i in range(0, len(sp)):
sp_new.append(sp[indices[i]])
Nspheres = 122000
sp_new = sp_new[0:Nspheres]
from yade import ymport
facets = ymport.stl(fileName + '.stl', color=(0, 1, 0), material=Steel)
fctIds = range(len(facets))
NSTEPS = 1000
if numMPIThreads > 1:
mp.mprint("appending bodies, rank", mp.rank)
if mp.rank == 0:
O.bodies.append(facets)
mp.mprint("master has", len(O.bodies), "facets")
else:
import numpy as np
layers = np.array_split(sp_new, numMPIThreads - 1)
mp.mprint("layers", [len(l) for l in layers])
layerNo = mp.rank - 1 #rank zero is for facets
nextId = int(len(facets) + np.sum([len(x) for x in layers[:layerNo]]))
mp.mprint("s", s, "startId", nextId)
for s in layers[layerNo]:
s.subdomain = mp.rank
O.bodies.insertAtId(s, nextId)
nextId += 1
print("num. bodies:", len([b for b in O.bodies]), len(O.bodies))
print("Total force on floor=", O.forces.f(WALL_ID)[1])
else: ####### MPI ######
#import yade's mpi module
from yade import mpy as mp
# customize
mp.ACCUMULATE_FORCES = True #trigger force summation on master's body (here WALL_ID)
mp.VERBOSE_OUTPUT = False
mp.MAX_RANK_OUTPUT = 4
mp.mpirun(
1
) #this is to eliminate initialization overhead in Cundall number and timings
from yade import timing
timing.reset()
t1 = time.time()
mp.mpirun(NSTEPS)
t2 = time.time()
mp.mprint("num. bodies:", len([b for b in O.bodies]), " ", len(O.bodies))
if rank == 0:
mp.mprint("Total force on floor=" + str(O.forces.f(WALL_ID)[1]))
mp.mprint("CPU wall time for ", NSTEPS, " iterations:", t2 - t1,
"; Cundall number = ",
len(O.bodies) * NSTEPS / (t2 - t1))
collectTiming()
else:
mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1]))
#mp.mergeScene()
#if rank==0: O.save('mergedScene.yade')
mp.MPI.Finalize()
exit()
O.dynDt = False
#import yade's mpi module
#from yade import mpy as mp
# customize
mp.VERBOSE_OUTPUT = False
mp.DISTRIBUTED_INSERT = True
mp.REALLOCATE_FREQUENCY = 4
mp.MAX_RANK_OUTPUT = 6
mp.mpirun(
1, numThreads, False
) #this is to eliminate initialization overhead in Cundall number and timings
mp.YADE_TIMING = True
t1 = time.time()
mp.mpirun(NSTEPS, withMerge=False)
t2 = time.time()
mp.mprint("num. bodies:", len([b for b in O.bodies if b.subdomain == mp.rank]),
"/", len(O.bodies))
def collectTiming(Cundall):
created = os.path.isfile("collect3D.dat")
f = open('collect3D.dat', 'a')
if not created:
f.write("CuNumber numThreads mpi omp Nspheres L M N runtime \n")
from yade import timing
f.write(
str(Cundall) + " " +
str(numThreads * int(os.getenv('OMP_NUM_THREADS'))) + " " +
str(os.getenv('OMPI_COMM_WORLD_SIZE')) + " " +
os.getenv('OMP_NUM_THREADS') + " " +
str(N * M * L * (numThreads - 1)) + " " + str(L) + " " + str(M) + " " +
str(N) + " " + str(timing.runtime()) + "\n")
O.engines=[
ForceResetter(),
InsertionSortCollider([Bo1_Sphere_Aabb(), Bo1_Box_Aabb()], label='collider', allowBiggerThanPeriod=True),
InteractionLoop(
[Ig2_Sphere_Sphere_ScGeom(), Ig2_Box_Sphere_ScGeom()],
[Ip2_FrictMat_FrictMat_FrictPhys()],
[Law2_ScGeom_FrictPhys_CundallStrack()]
,label='InteractionLoop'),
GlobalStiffnessTimeStepper(timestepSafetyCoefficient=0.7, timeStepUpdateInterval=200, parallelMode=True, label = "ts"),
fluidCoupling, #to be called after timestepper
NewtonIntegrator(damping = 0.0, label='newton', gravity = (0, 0.0, 0)),
VTKRecorder(fileName='spheres/3d-vtk-', recorders=['all'], parallelMode=True, iterPeriod=1000)
]
collider.verletDist =0.00075
mp.YADE_TIMING = False
mp.FLUID_COUPLING = True
mp.VERBOSE_OUTPUT= False
mp.USE_CPP_INTERS = False
mp.ERASE_REMOTE_MASTER = True
mp.REALLOC_FREQUENCY = 12
mp.fluidBodies = sphereIDs
mp.DOMAIN_DECOMPOSITION= True
mp.mpirun(NSTEPS)
mp.mprint("RUN FINISH")
fluidCoupling.killMPI()
exit()
#mp.MPI.Finalize()
#mp.mergeScene()
#if mp.rank == 0: O.save('mergedScene.yade')
mp.MERGE_SPLIT = True
mp.COPY_MIRROR_BODIES_WHEN_COLLIDE = False
mp.WALL_ID = WALL_ID
mp.mpirun(NSTEPS)
print("num. bodies:", len([b for b in O.bodies]), len(O.bodies))
if rank == 0:
fl = open('intrs_parallel.txt', 'w')
for i in O.interactions:
a = i.id1
b = i.id2
if a > b: a, b = b, a
fl.write('%s %s\n' % (a, b))
fl.close()
collectTiming()
mp.mprint(
"Total force on floor based on FORCES RECVD FROM WORKERS =" +
str(O.forces.f(WALL_ID)[1]))
else:
mp.mprint("Partial force on floor=" + str(O.forces.f(WALL_ID)[1]))
mp.mergeScene()
if rank == 0:
# just for saving and checking the state after merge.
#
O.save('mergedScene.yade')
print("force recieved from workers = ", O.forces.f(WALL_ID)[1])
#O.forces.reset()
#collider.__call__()
##print "num interactions = " , len(O.interactions)
#O.step()
#mp.mprint( "Total force on floor based on inters ="+str(O.forces.f(WALL_ID)[1]))
# 2018 © Bruno Chareyre <*****@*****.**>
# Really a helloWorld (or almost, I couldn't refrain from adding a bit more).
# Check other examples for concrete usage of mpy in DEM simulations
from yade import mpy as mp
mp.initialize(3)
#mp.VERBOSE_OUTPUT = True # to see more of what happens behind the scene
mp.mprint("I'm here")
if mp.rank==0:
# say hello:
mp.sendCommand(executors=[1,2],command="mprint('Yes I am really here')",wait=False)
# get retrun values if wait=True (blocking comm., think twice)
print( mp.sendCommand(executors="all",command="len(O.bodies)",wait=True) )
## exploit sendCommand to send data between yade instances directly with mpi4py (see mpi4py documentation)
## in the message we actually tell the worker to wait another message (nested comm), but the second one
## uses underlying mpi4py, and it handles pickable objects
mp.sendCommand(executors=1,command="message=comm.recv(source=0); mprint('received: ',message)",wait=False)
mp.comm.send("hello",dest=1)
## pickable objects
## pay attention to pointer adresses, they are different! (as expected)
## this is moving data around between independent parts of memory
mp.sendCommand(executors=1,command="O.bodies.append(Body()); O.bodies[0].shape=Sphere(radius=0.456); comm.send(O.bodies[0],dest=0); mprint('sent a ',O.bodies[0].shape)",wait=False)
bodyFrom1 = mp.comm.recv(source=1)
mp.mprint("received a ",bodyFrom1.shape,"with radius=",bodyFrom1.shape.radius)
