def __init__(self, grid, param):
for key in ['shape', 'incr', 'procs']:
setattr(self, key, grid[key])
for key in ['omega', 'ndeepest', 'myrank']:
setattr(self, key, param[key])
nh = 1
procs0 = [i*p for i, p in zip(self.incr, self.procs)]
# print("---------> procs0=", procs0, self.myrank)
loc = topo.rank2loc(self.myrank, procs0)
neighbours = topo.get_neighbours(loc, procs0, incr=self.incr)
size, domainindices = topo.get_variable_shape(
self.shape, neighbours, nh)
N = np.prod(size)
self.domainindices = domainindices
self.size = size
self.N = N
self.nh = nh
self.neighbours = neighbours
self.x = np.zeros((self.N,))
self.y = np.zeros((self.N,))
self.b = np.zeros((self.N,))
self.r = np.zeros((self.N,))
self.msk = np.zeros(self.size)
k0, k1, j0, j1, i0, i1 = self.domainindices
self.msk[k0:k1, j0:j1, i0:i1] = 1.
# self.xx = self.x
# self.xx.shape = self.size
# self.bb = self.b
# self.bb.shape = self.size
# self.rr = self.r
# self.rr.shape = self.size
# self.xx = self.x.reshape(self.size)
# self.bb = self.b.reshape(self.size)
# self.rr = self.r.reshape(self.size)
# self.xx = np.reshape(self.x, self.size)
# self.bb = np.reshape(self.b, self.size)
# self.rr = np.reshape(self.r, self.size)
# self.xx = np.zeros(self.size)
# self.bb = np.zeros(self.size)
# self.rr = np.zeros(self.size)
# self.x = self.xx.ravel()
# self.b = self.bb.ravel()
# self.r = self.rr.ravel()
self.halo = halo.Halo({"nh": nh, "size": size, "neighbours": neighbours,
"domainindices": domainindices, "shape": self.shape})
def print_subdomains(subdomains):
print('-' * 80)
for isub, subdom in enumerate(subdomains):
dom = subdom['dom']
procs = subdom['procs']
incr = subdom['incr']
tags = subdom['tags']
levs = subdom['levels']
glued = subdom['glued']
domloc = subdom['domloc']
ranks = np.arange(len(dom))
k, j, i = topo.rank2loc(ranks, procs)
if isub == 0:
k0 = k.copy()
j0 = j.copy()
i0 = i.copy()
procs0 = procs.copy()
print(' Partition %1i' % isub)
print('=============')
print('Subdomains are : ', dom)
print('corresponding to domain decomposition: ', procs)
print('increments in each directions: ', incr)
print('grid levels having this partition: ', levs)
print('Location of each subdomain')
print('k: ', k)
print('j: ', j)
print('i: ', i)
# for idx in gr.ranktoloc(ranks, procs):
# print('x: ', idx)
if 'gluing' in subdom.keys():
print('gluing was in directions ', subdom['gluing'])
for d in set(dom):
family = [r for r in ranks if dom[r] == d]
# in a family, all ranks have the same loc
# because they all compute the same subdomain
locs = [(k0[r], j0[r], i0[r]) for r in family]
# tags is defined from the previous domain decomposition
# glued = [[r for r in family if tags[r] == t]
# for t in set(tags[family])]
print(' - subdomain : ', set([d]))
print(' located at ', domloc[d]) # s[0])
print(' computed by ' % d, family)
if 'gluing' in subdom.keys():
print(' obtained by gluing ranks:', glued)
print(' neighbours:')
for r in family:
print('rank : %i / ' % r, subdom['allneighbours'][r])
def __init__(self, user_param):
# Check, freeze, and get user parameters
user_param.check()
user_param.freeze()
param = user_param.view_parameters()
self.param = param
npx = param["npx"]
npy = param["npy"]
npz = param["npz"]
# Add parameters that are automatically set
param["nx"] = param["global_nx"] // npx
param["ny"] = param["global_ny"] // npy
param["nz"] = param["global_nz"] // npz
# Set up MPI
topo.topology = param["geometry"]
procs = [npz, npy, npx]
myrank = mpitools.get_myrank(procs)
loc = topo.rank2loc(myrank, procs)
neighbours = topo.get_neighbours(loc, procs)
param["procs"] = procs
param["myrank"] = myrank
param["neighbours"] = neighbours
param["loc"] = loc
self.myrank = myrank
if self.myrank == 0:
print("-" * 80)
self.banner()
# Load the grid with the extended parameters
self.grid = grid.Grid(param)
# Load the IO; only the parameters modifiable by the user are saved
self.IO = nylesIO.NylesIO(param)
# redirect the x11 to output.txt
#sys.stdout = Logger(self.IO.output_directory+'/output.txt')
# save the param dictionary in a pkl file
if self.myrank == 0:
with open(self.IO.output_directory + "/param.pkl", "wb") as fid:
pickle.dump(param, fid)
# Initiate the model and needed variables
self.initiate(param)
def write_stats(self, path):
fid = open('%s/stats.pkl' % path, 'bw')
pickle.dump(self.stats, fid)
if __name__ == '__main__':
import numpy as np
from matplotlib import pyplot as plt
import topology as topo
procs = [1, 1, 1]
topo.topology = 'closed'
myrank = 0
nh = 2
loc = topo.rank2loc(myrank, procs)
neighbours = topo.get_neighbours(loc, procs)
nz, ny, nx = 32, 32, 128
Lx, Ly, Lz = 1.0, 1.0, 1.0
dx, dy, dz = Lx/nx, Ly/ny, Lz/nz
param = {
'nx': nx, 'ny': ny, 'nz': nz, 'nh': nh,
'Lx': Lx, 'Ly': Ly, 'Lz': Lz,
'neighbours': neighbours,
# Choose a timestepping method
'timestepping': 'LFAM3',
# 'timestepping': 'EF',
# Set the order of the upwind scheme
'orderA': 5,
# 'orderA': 3,
def test_111_domain():
topo.topology = 'perio_xyz'
extension = 26
nz = 1
ny = 1
nx = 1
nh = 1
loc = topo.rank2loc(myrank, procs)
neighbours = topo.get_neighbours(loc, procs, extension=extension)
param = {'nx': nx, 'ny': ny, 'nz': nz, 'nh': nh, 'neighbours': neighbours}
state = var.get_state(param)
b = state.get('b')
# we can define the halo toolbox
grid = {
'shape': [nz, ny, nx],
'size': b.size,
'nh': nh,
'neighbours': neighbours,
'domainindices': b.domainindices,
'extension': extension
}
halo = Halo(grid)
# for r, buf in halo.rbuf.items():
# if r in neighbours.keys():
# buf[:] = neighbours[r]*1.
# else:
# buf[:] = -1.
# set the field equal to myrank
# making clear that the halo is wrong
b.activeview = 'i'
# if myrank == 0:
# b.view('i')[:, :, :] = 999
# else:
# b.view('i')[:, :, :] = 0.
x = b.view('i')
x[:, :, :] = myrank
halo.fill(b)
# now in the halo we'd have the rank of the neighbour
y = np.asarray(x.copy(), dtype=int)
msg = 'pb withneighbours'
for direc, yourrank in neighbours.items():
dk, dj, di = direc
assert (y[1 + dk, 1 + dj, 1 + di] == yourrank), msg
if myrank == 0:
print('fill halo of Scalar is okay')
l = nz + 2 * nh
m = ny + 2 * nh
n = nx + 2 * nh
x = np.zeros((l, m, n))
grid = {
'shape': [nz, ny, nx],
'size': np.shape(x),
'nh': nh,
'neighbours': neighbours,
'domainindices': b.domainindices,
'extension': extension
}
halox = Halo(grid)
x[:] = myrank
halox.fill(x)
y = np.asarray(x.copy(), dtype=int)
for direc, yourrank in neighbours.items():
dk, dj, di = direc
#print(myrank, y[1+dk, 1+dj, 1+di] , yourrank)
assert (y[1 + dk, 1 + dj, 1 + di] == yourrank), msg
if myrank == 0:
print('fill halo of ndarray is okay')
def set_subdomains(allgrids):
grid = allgrids[0]
procs = grid['procs'].copy()
procs0 = procs.copy()
incr = [1, 1, 1]
nprocs = np.prod(procs)
ranks = np.arange(nprocs)
loc = topo.rank2loc(ranks, procs)
loc0 = loc.copy()
k0, j0, i0 = loc0
gathers = []
doms = []
first = True
tags = ranks * 0
subdom_partition = []
for lev, grid in enumerate(allgrids):
c = grid['coarsen']
keys = c.keys()
if any(['g' in k for k in keys]) or lev == 0:
g = []
levs = [lev]
for n, direc in enumerate('kji'):
if 'g' + direc in keys:
factor = 2 * c['g' + direc]
incr[n] *= factor
procs[n] //= factor
loc[n] //= factor
g += [direc]
if g != '':
gathers += [g]
else:
pass
k, j, i = loc
nz, ny, nx = procs0
incz, incy, incx = incr
dom = [0] * nprocs
for r in ranks:
l = [k[r], j[r], i[r]]
# d is the index of the subdomain
# rank with same d do the same computation
d = topo.myloc(l, incr, [0, 0, 0], procs0)
dom[r] = d
doms += [dom]
dd = set(dom)
neighbours = [{}] * len(ranks)
domloc = {}
glued = {}
for d in dd:
family = [r for r in ranks if dom[r] == d]
# in a family, all ranks have the same loc
# because they all compute the same subdomain
locs = [(k0[r], j0[r], i0[r]) for r in family]
# tags is defined from the previous domain decomposition
fam = family[0]
glued[d] = [[r for r in family if tags[r] == t]
for t in set(tags[family])]
if len(glued[d]) == 1:
glued[d] = glued[d][0]
# print(' - subdomain : ', set([d]))
# print(' located at ', locs[0])
# print(' computed by ' % d, family)
# if not(first):
# print(' obtained by gluing ranks:', glued)
# here is the list of neighbours
#
# the print has been discarded to lighten the output
# but clearly this information is very important
# note that the neighbours relationships for a given
# rank depends on the subdomain decomposition
# the neighbours get further and further as
# subdomains are glued. This is accounted by
# 'incr'. incr == 1 on the finest grid, then
# whenever there is gluing in one direction
# incr is doubled in that direction, until
# the point where the is only one subdomain is that
# direction. Each rank is then neighbour with itself
#
# print(' neighbours:')
for r in family:
l = [k0[r], j0[r], i0[r]]
nghbs = topo.get_neighbours(l,
procs0,
incr=incr,
extension=26)
ngbs = fixneighbours(nghbs, r, d)
# print('rank : %i / ' % r, ngbs)
neighbours[r] = ngbs
r0 = d # family[0]
domloc[d] = [loc0[0][r0], loc0[1][r0], loc0[2][r0]]
# print('domain: %i' % d, domloc[d])
localpartition = {
'procs': procs.copy(),
'incr': incr.copy(),
'gluing': g.copy(),
'glued': glued.copy(),
'domloc': domloc.copy(),
'dom': dom.copy(),
'levels': levs,
'allneighbours': neighbours.copy()
}
# print(levs, domloc)
# if not(first):
localpartition['tags'] = tags.copy()
subdom_partition += [localpartition]
#
# tags is used to identify which ranks are glued together
#
# glued ranks had identical tag in the previous
# subdomain decomposition
#
for d in dd:
family = [r for r in ranks if dom[r] == d]
for k, r in enumerate(family):
tags[r] = k
first = False
else:
levs += [lev]
return subdom_partition
