def setup():
#Import CPL library
from cplpy import CPL
#initialise MPI
from mpi4py import MPI
comm = MPI.COMM_WORLD
#Check run as part of a coupled run
comm.rank
# Parameters of the cpu topology (cartesian grid)
npxyz = np.array([1, 1, 1], order='F', dtype=np.int32)
xyzL = np.array([1., 1., 1.], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
#initialise CPL
CPL = CPL()
MD_COMM = CPL.init(CPL.MD_REALM)
CPL.setup_md(MD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]]), xyzL, xyz_orig)
recvbuf, sendbuf = CPL.get_arrays(recv_size=9, send_size=8)
#Analytical solution
dt = 0.05
U = 1.0
nu = 1.004e-2
Re = xyzL[1]/nu #Note Reynolds in independent of velocity in analytical fn
ncx = CPL.get("ncx")
ncy = CPL.get("ncy")
ncz = CPL.get("ncz")
CAObj = CA(Re=Re, U=U, Lmin=0., Lmax=xyzL[1], npoints=2*ncy+1, nmodes=100*ncy)
#Yield statement delineates end of setup and start of teardown
yield [CPL, MD_COMM, recvbuf, sendbuf, CAObj, dt, U, nu]
CPL.finalize()
MPI.Finalize()
xyzL = np.array([16.795961913825074, 45.349097, 16.795961913825074],
order='F',
dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
if (MD_COMM.Get_size() != NProcs):
print("Non-coherent number of processes in MD ", MD_COMM.Get_size(),
" no equal to ", npxyz[0], " X ", npxyz[1], " X ", npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
cart_comm = MD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_md(cart_comm, xyzL, xyz_orig)
#Get values from CPL library
ncy = CPL.get("ncy")
yL_md = CPL.get("yl_md")
yL_cfd = CPL.get("yl_cfd")
dy = CPL.get("dy")
jcmax_olap = CPL.get("jcmax_olap")
#Setup Coupled domain details
yL_cpl = yL_md + yL_cfd - dy * jcmax_olap
ncy_cpl = int(yL_cpl / dy)
#Create analytical solution object
nu = 1.7
Re = yL_cpl / nu
CAObj = CA(Re=Re, U=1., Lmin=0.0, Lmax=yL_cpl, npoints=2 * ncy_cpl + 2)
#Setup send and recv buffers
order='F',
dtype=np.float64)
else:
send_array = -5 * np.array(np.random.rand(4, velBCncx, velBCncy, velBCncz),
order='F',
dtype=np.float64)
if NPx % 2 != 0 and my_coords[0] == 0:
send_array = np.zeros((4, velBCncx, velBCncy, velBCncz),
order='F',
dtype=np.float64)
cpllib.recv(recv_array, cnstFRegion)
cpllib.send(send_array, velBCRegion)
nsteps = cpllib.get("nsteps_coupled")
for i in xrange(nsteps):
cpllib.recv(recv_array, cnstFRegion)
cpllib.send(send_array, velBCRegion)
cpllib.dump_region(velBCRegion,
send_array,
"md_vels.dat",
realm_comm,
components={
0: None,
1: None,
2: None
},
coords="other")
cpllib.finalize()
],
order='F',
dtype=np.int32)
BC_portion = CPL.my_proc_portion(BC_limits)
[BC_ncxl, BC_ncyl, BC_nczl] = CPL.get_no_cells(BC_portion)
#Plot output
fig, ax = plt.subplots(1, 1)
plt.ion()
plt.show()
#Analytical solution
U = 1.
nu = 1e-02
Re = U / nu
ncy = CPL.get("ncy")
CAObj = CA(Re=Re, U=U, Lmin=0., Lmax=xyzL[1], npoints=ncy)
#Porous region solution
pcell = 7
dy = CPL.get("dy")
yl_cfd = CPL.get("yl_cfd")
CApObj = CA(Re=Re,
U=U,
Lmin=yl_cfd - (pcell + 2.5) * dy,
Lmax=xyzL[1],
npoints=ncy)
#y_p = np.linspace(yl_cfd-(pcell+2)*dy,yl_cfd, ncy)
# Coupling parameters
xyzL = np.array([1., 1., 1.], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
#initialise CPL
CPL = CPL()
MD_COMM = CPL.init(CPL.MD_REALM)
CPL.setup_md(MD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]]), xyzL,
xyz_orig)
recvbuf, sendbuf = CPL.get_arrays(recv_size=9, send_size=8)
#Analytical solution
dt = 0.05
U = 1.
nu = 1e-02
Re = xyzL[1] * U / nu
ncx = CPL.get("ncx")
ncy = CPL.get("ncy")
ncz = CPL.get("ncz")
CAObj = CA(Re=Re,
U=U,
Lmin=0.,
Lmax=xyzL[1],
npoints=2 * ncy + 1,
nmodes=100 * ncy)
dx = xyzL[0] / float(ncx)
dy = xyzL[1] / float(ncy)
dz = xyzL[2] / float(ncz)
dV = dx * dy * dz
y = np.linspace(dy / 2., xyzL[1] - dy / 2., num=ncy)
#Main time loop
npxyz = np.array([1, 1, 1], order='F', dtype=np.int32)
NProcs = np.product(npxyz)
xyzL = np.array([6.70820393, 17.88854382, 1.0], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
if (nprocs_realm != NProcs):
print("Non-coherent number of processes in MD ", nprocs_realm,
" not equal to ", npxyz[0], " X ", npxyz[1], " X ", npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_md(cart_comm, xyzL, xyz_orig)
#Setup buffer to send CFD BC from MD
ncx = CPL.get("ncx")
dy = CPL.get("yl_cfd") / CPL.get("ncy")
ncy = np.floor(xyzL[1] / dy)
limits_CFD_BC = np.array([0, ncx, 0, 1, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_CFD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_send = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Setup buffer to recv constrained region
limits_MD_BC = np.array([0, ncx, 3, 4, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_MD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_recv = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
# Setup MD simulation object
md_cfd_dt_ratio = 50
NProcs = np.product(npxyz)
xyzL = np.array([6.70820393, 17.88854382, 1.0], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
ncxyz = np.array([8, 8, 1], order='F', dtype=np.int32)
if (nprocs_realm != NProcs):
print("Non-coherent number of processes in CFD ", nprocs_realm,
" no equal to ", npxyz[0], " X ", npxyz[1], " X ", npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_cfd(cart_comm, xyzL, xyz_orig, ncxyz )
#Setup buffer to get CFD BC from MD
ncx = CPL.get("ncx")
limits_CFD_BC = np.array([0, ncx, 0, 1, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_CFD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_recv = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Setup buffer to send constrained region
limits_MD_BC = np.array([0, ncx, 3, 4, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_MD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_send = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Set CFD simulation object
md_cfd_dt_ratio = 25
dt = 0.125; Nsteps = 100000/md_cfd_dt_ratio; tf = Nsteps*dt
time = np.arange(-dt,tf,dt)
#plt.subplots_adjust(right=0.9)
#axslider = plt.axes([0.25, 0.1, 0.65, 0.03])
freq = 0
#freq = 0.25; vmin = -2.0; vmax = 2.0
#sfreq = Slider(axslider, '$U_{wall}$', vmin, vmax, valinit=freq)
#def update(val):
# freq = sfreq.val
# global freq
# print("CHANGED", freq)
#sfreq.on_changed(update)
plt.ion()
plt.show()
## === Plot both grids ===
dx = CPL.get("xl_md") / float(CPL.get("ncx"))
dy = CPL.get("yl_md") / float(CPL.get("ncy"))
dz = CPL.get("zl_md") / float(CPL.get("ncz"))
dV = dx * dy * dz
#first_time = True
cnst_limits = CPL.get_cnst_limits()
cnst_portion = CPL.my_proc_portion(cnst_limits)
[cnst_ncxl, cnst_ncyl, cnst_nczl] = CPL.get_no_cells(cnst_portion)
olap_limits = CPL.get_olap_limits()
BC_limits = np.array([
olap_limits[0], olap_limits[1], olap_limits[2], olap_limits[3],
olap_limits[4], olap_limits[5]
],
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_cfd(cart_comm, xyzL, xyz_orig, ncxyz)
# recv data to plot
olap_limits = CPL.get_olap_limits()
portion = CPL.my_proc_portion(olap_limits)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
recv_array = np.zeros((1, ncxl, ncyl, nczl), order='F', dtype=np.float64)
recv_array, ierr = CPL.recv(recv_array, olap_limits)
#Plot output
fig, ax = plt.subplots(1,1)
# === Plot both grids ===
dx = CPL.get("xl_cfd")/float(CPL.get("ncx"))
dy = CPL.get("yl_cfd")/float(CPL.get("ncy"))
dz = CPL.get("zl_cfd")/float(CPL.get("ncz"))
ioverlap = (CPL.get("icmax_olap")-CPL.get("icmin_olap")+1)
joverlap = (CPL.get("jcmax_olap")-CPL.get("jcmin_olap")+1)
koverlap = (CPL.get("kcmax_olap")-CPL.get("kcmin_olap")+1)
xoverlap = ioverlap*dx
yoverlap = joverlap*dy
zoverlap = koverlap*dz
print(CPL.get("xl_cfd"))
#Plot CFD and coupler Grid
draw_grid(ax,
nx=CPL.get("ncx"),
ny=CPL.get("ncy"),
npxyz = np.array([1, 1, 1], order='F', dtype=np.int32)
NProcs = np.product(npxyz)
xyzL = np.array([6.70820393, 17.88854382, 1.0], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
if (nprocs_realm != NProcs):
print("Non-coherent number of processes in MD ", nprocs_realm,
" no equal to ", npxyz[0], " X ", npxyz[1], " X ", npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_md(cart_comm, xyzL, xyz_orig)
#Setup buffer to send CFD BC from MD
ncx = CPL.get("ncx"); dy = CPL.get("yl_cfd")/CPL.get("ncy")
ncy = np.floor(xyzL[1]/dy)
limits_CFD_BC = np.array([0, ncx, 0, 1, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_CFD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_send = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Setup buffer to recv constrained region
limits_MD_BC = np.array([0, ncx, 3, 4, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_MD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_recv = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
# Setup MD simulation object
md_cfd_dt_ratio = 25
dt = 0.005; Nsteps = 100000; tf = Nsteps*dt
sfreq = Slider(axslider, 'Freq', 0.1, 2.0, valinit=freq)
def update(val):
freq = sfreq.val
global freq
print("CHANGED", freq)
sfreq.on_changed(update)
plt.ion()
plt.show()
# === Plot both grids ===
dx = CPL.get("xl_cfd") / float(CPL.get("ncx"))
dy = CPL.get("yl_cfd") / float(CPL.get("ncy"))
dz = CPL.get("zl_cfd") / float(CPL.get("ncz"))
ioverlap = (CPL.get("icmax_olap") - CPL.get("icmin_olap") + 1)
joverlap = (CPL.get("jcmax_olap") - CPL.get("jcmin_olap") + 1)
koverlap = (CPL.get("kcmax_olap") - CPL.get("kcmin_olap") + 1)
xoverlap = ioverlap * dx
yoverlap = joverlap * dy
zoverlap = koverlap * dz
for time in range(100000):
# recv data to plot
olap_limits = CPL.get_olap_limits()
portion = CPL.my_proc_portion(olap_limits)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
NProcs = np.product(npxyz)
xyzL = np.array([6.70820393, 17.88854382, 1.0], order='F', dtype=np.float64)
xyz_orig = np.array([0.0, 0.0, 0.0], order='F', dtype=np.float64)
ncxyz = np.array([8, 8, 1], order='F', dtype=np.int32)
if (nprocs_realm != NProcs):
print("Non-coherent number of processes in CFD ", nprocs_realm,
" not equal to ", npxyz[0], " X ", npxyz[1], " X ", npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_cfd(cart_comm, xyzL, xyz_orig, ncxyz)
#Setup buffer to get CFD BC from MD
ncx = CPL.get("ncx")
limits_CFD_BC = np.array([0, ncx, 0, 1, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_CFD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_recv = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Setup buffer to send constrained region
limits_MD_BC = np.array([0, ncx, 3, 4, 0, 1], order='F', dtype=np.int32)
portion = CPL.my_proc_portion(limits_MD_BC)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
A_send = np.zeros((2, ncxl, ncyl, nczl), order='F', dtype=np.float64)
#Set CFD simulation object
md_cfd_dt_ratio = 50
dt = 0.125
Nsteps = 100000 / md_cfd_dt_ratio
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_cfd(cart_comm, xyzL, xyz_orig, ncxyz)
# recv data to plot
olap_limits = CPL.get_olap_limits()
portion = CPL.my_proc_portion(olap_limits)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
recv_array = np.zeros((1, ncxl, ncyl, nczl), order='F', dtype=np.float64)
recv_array, ierr = CPL.recv(recv_array, olap_limits)
#Plot output
fig, ax = plt.subplots(2,1)
#Plot x component on grid
for j in range(CPL.get("jcmin_olap"),CPL.get("jcmax_olap")+1):
ax[0].plot((recv_array[0,:,j,0]+1.+2*j), 's-')
ax[0].set_xlabel('$x$')
ax[0].set_ylabel('$y$')
#Plot xz of bottom cell
ax[1].pcolormesh(recv_array[0,:,0,:])
ax[1].set_xlabel('$x$')
ax[1].set_ylabel('$z$')
plt.show()
CPL.finalize()
MPI.Finalize()
class CFD():
def __init__(self, npxyz, xyzL, xyz_orig, ncxyz):
#initialise MPI and CPL
self.comm = MPI.COMM_WORLD
self.CPL = CPL()
self.CFD_COMM = self.CPL.init(CPL.CFD_REALM)
self.nprocs_realm = self.CFD_COMM.Get_size()
# Parameters of the cpu topology (cartesian grid)
self.npxyz = np.array(npxyz, order='F', dtype=np.int32)
self.NProcs = np.product(npxyz)
self.xyzL = np.array(xyzL, order='F', dtype=np.float64)
self.xyz_orig = np.array(xyz_orig, order='F', dtype=np.float64)
self.ncxyz = np.array(ncxyz, order='F', dtype=np.int32)
if (self.nprocs_realm != self.NProcs):
print("Non-coherent number of processes in CFD ",
self.nprocs_realm, " no equal to ", self.npxyz[0], " X ",
self.npxyz[1], " X ", self.npxyz[2])
MPI.Abort(errorcode=1)
#Setup coupled simulation
self.cart_comm = self.CFD_COMM.Create_cart(
[self.npxyz[0], self.npxyz[1], self.npxyz[2]])
self.CPL.setup_cfd(self.cart_comm, self.xyzL, self.xyz_orig,
self.ncxyz)
#Get limits of overlap region
self.olap_limits = self.CPL.get_olap_limits()
self.portion = self.CPL.my_proc_portion(self.olap_limits)
[self.ncxl, self.ncyl, self.nczl] = self.CPL.get_no_cells(self.portion)
self.dx = self.CPL.get("xl_cfd") / float(self.CPL.get("ncx"))
self.dy = self.CPL.get("yl_cfd") / float(self.CPL.get("ncy"))
self.dz = self.CPL.get("zl_cfd") / float(self.CPL.get("ncz"))
self.ioverlap = (self.CPL.get("icmax_olap") -
self.CPL.get("icmin_olap") + 1)
self.joverlap = (self.CPL.get("jcmax_olap") -
self.CPL.get("jcmin_olap") + 1)
self.koverlap = (self.CPL.get("kcmax_olap") -
self.CPL.get("kcmin_olap") + 1)
self.xoverlap = self.ioverlap * self.dx
self.yoverlap = self.joverlap * self.dy
self.zoverlap = self.koverlap * self.dz
def recv_CPL_data(self):
# recv data to plot
self.recv_array = np.zeros((1, self.ncxl, self.ncyl, self.nczl),
order='F',
dtype=np.float64)
self.recv_array, ierr = self.CPL.recv(self.recv_array,
self.olap_limits)
def plot_grid(self, ax):
#Plot CFD and coupler Grid
draw_grid(ax,
nx=self.CPL.get("ncx"),
ny=self.CPL.get("ncy"),
nz=self.CPL.get("ncz"),
px=self.CPL.get("npx_cfd"),
py=self.CPL.get("npy_cfd"),
pz=self.CPL.get("npz_cfd"),
xmin=self.CPL.get("x_orig_cfd"),
ymin=self.CPL.get("y_orig_cfd"),
zmin=self.CPL.get("z_orig_cfd"),
xmax=(self.CPL.get("icmax_olap") + 1) * self.dx,
ymax=self.CPL.get("yl_cfd"),
zmax=(self.CPL.get("kcmax_olap") + 1) * self.dz,
lc='r',
label='CFD')
#Plot MD domain
draw_grid(ax,
nx=1,
ny=1,
nz=1,
px=self.CPL.get("npx_md"),
py=self.CPL.get("npy_md"),
pz=self.CPL.get("npz_md"),
xmin=self.CPL.get("x_orig_md"),
ymin=-self.CPL.get("yl_md") + self.yoverlap,
zmin=self.CPL.get("z_orig_md"),
xmax=(self.CPL.get("icmax_olap") + 1) * self.dx,
ymax=self.yoverlap,
zmax=(self.CPL.get("kcmax_olap") + 1) * self.dz,
label='MD')
def plot_data(self, ax):
# === Plot both grids ===
#Plot x component on grid
x = np.linspace(
self.CPL.get("x_orig_cfd") + .5 * self.dx,
self.xoverlap - .5 * self.dx, self.ioverlap)
z = np.linspace(
self.CPL.get("z_orig_cfd") + .5 * self.dz,
self.zoverlap - .5 * self.dz, self.koverlap)
try:
for j in range(self.joverlap):
ax.plot(
x,
0.5 * self.dy * (self.recv_array[0, :, j, 0] + 1. + 2 * j),
's-')
except ValueError:
print("Arrays not equal:", x.shape, z.shape, self.recv_array.shape)
def finalise(self):
self.CPL.finalize()
MPI.Finalize()
#Setup coupled simulation
cart_comm = CFD_COMM.Create_cart([npxyz[0], npxyz[1], npxyz[2]])
CPL.setup_cfd(cart_comm, xyzL, xyz_orig, ncxyz)
# recv data to plot
olap_limits = CPL.get_olap_limits()
portion = CPL.my_proc_portion(olap_limits)
[ncxl, ncyl, nczl] = CPL.get_no_cells(portion)
recv_array = np.zeros((1, ncxl, ncyl, nczl), order='F', dtype=np.float64)
recv_array, ierr = CPL.recv(recv_array, olap_limits)
#Plot output
fig, ax = plt.subplots(2, 1)
#Plot x component on grid
for j in range(CPL.get("jcmin_olap"), CPL.get("jcmax_olap") + 1):
ax[0].plot((recv_array[0, :, j, 0] + 1. + 2 * j), 's-')
ax[0].set_xlabel('$x$')
ax[0].set_ylabel('$y$')
#Plot xz of bottom cell
ax[1].pcolormesh(recv_array[0, :, 0, :])
ax[1].set_xlabel('$x$')
ax[1].set_ylabel('$z$')
plt.show()
CPL.finalize()
MPI.Finalize()
# === Plot both grids ===
dx = CPL.get("xl_cfd") / float(CPL.get("ncx"))
