def old_distribute_cpus(parsize_domain, parsize_bands,
nspins, nibzkpts, comm=world,
idiotproof=True, mode='fd'):
"""Distribute k-points/spins to processors.
Construct communicators for parallelization over
k-points/spins and for parallelization using domain
decomposition."""
size = comm.size
rank = comm.rank
nsk = nspins * nibzkpts
if mode in ['fd', 'lcao']:
if parsize_bands is None:
parsize_bands = 1
if parsize_domain is not None:
if type(parsize_domain) is int:
ndomains = parsize_domain
else:
ndomains = (parsize_domain[0] *
parsize_domain[1] *
parsize_domain[2])
assert (size // parsize_bands) % ndomains == 0
else:
ntot = nsk * parsize_bands
ndomains = size // gcd(ntot, size)
else:
# Plane wave mode:
ndomains = 1
if parsize_bands is None:
parsize_bands = size // gcd(nsk, size)
assert size % parsize_bands == 0
# How many spin/k-point combinations do we get per node:
nu, x = divmod(nsk, size // parsize_bands // ndomains)
assert x == 0 or nu >= 2 or not idiotproof, 'load imbalance!'
r0 = (rank // ndomains) * ndomains
ranks = np.arange(r0, r0 + ndomains)
domain_comm = comm.new_communicator(ranks)
r0 = rank % (ndomains * parsize_bands)
ranks = np.arange(r0, r0 + size, ndomains * parsize_bands)
kpt_comm = comm.new_communicator(ranks)
r0 = rank % ndomains + kpt_comm.rank * (ndomains * parsize_bands)
ranks = np.arange(r0, r0 + (ndomains * parsize_bands), ndomains)
band_comm = comm.new_communicator(ranks)
assert size == domain_comm.size * kpt_comm.size * band_comm.size
return domain_comm, kpt_comm, band_comm
def distribute_cpus(parsize_domain, parsize_bands,
nspins, nibzkpts, comm=world,
idiotproof=True, mode='fd'):
"""Distribute k-points/spins to processors.
Construct communicators for parallelization over
k-points/spins and for parallelization using domain
decomposition."""
size = comm.size
rank = comm.rank
nsk = nspins * nibzkpts
if mode in ['fd', 'lcao']:
if parsize_bands is None:
parsize_bands = 1
if parsize_domain is not None:
if type(parsize_domain) is int:
ndomains = parsize_domain
else:
ndomains = (parsize_domain[0] *
parsize_domain[1] *
parsize_domain[2])
assert (size // parsize_bands) % ndomains == 0
else:
ntot = nsk * parsize_bands
ndomains = size // gcd(ntot, size)
else:
# Plane wave mode:
ndomains = 1
if parsize_bands is None:
parsize_bands = size // gcd(nsk, size)
assert size % parsize_bands == 0
# How many spin/k-point combinations do we get per node:
nu, x = divmod(nsk, size // parsize_bands // ndomains)
assert x == 0 or nu >= 2 or not idiotproof, 'load imbalance!'
r0 = (rank // ndomains) * ndomains
ranks = np.arange(r0, r0 + ndomains)
domain_comm = comm.new_communicator(ranks)
r0 = rank % (ndomains * parsize_bands)
ranks = np.arange(r0, r0 + size, ndomains * parsize_bands)
kpt_comm = comm.new_communicator(ranks)
r0 = rank % ndomains + kpt_comm.rank * (ndomains * parsize_bands)
ranks = np.arange(r0, r0 + (ndomains * parsize_bands), ndomains)
band_comm = comm.new_communicator(ranks)
assert size == domain_comm.size * kpt_comm.size * band_comm.size
return domain_comm, kpt_comm, band_comm
def create_parsize_maxbands(nbands, world_size):
"""Safely parse command line parallel arguments for band parallel case."""
# D: number of domains
# B: number of band groups
if parsize_bands is None:
if parsize_domain is None:
B = gcd(nbands, world_size) # largest possible
D = world_size // B
else:
D = parsize_domain
B = gcd(nbands, world_size // np.prod(D))
else:
B = parsize_bands
D = parsize_domain or world_size // B
return D, B
def create_parsize_maxbands(nbands, world_size):
"""Safely parse command line parallel arguments for band parallel case."""
# D: number of domains
# B: number of band groups
if parsize_bands is None:
if parsize_domain is None:
B = gcd(nbands, world_size) # largest possible
D = world_size // B
else:
D = parsize_domain
B = gcd(nbands, world_size // np.prod(D))
else:
B = parsize_bands
D = parsize_domain or world_size // B
return D, B
def get_parsizes(self): #XXX NO LONGER IN UT_HSOPS?!?
# Careful, overwriting imported GPAW params may cause amnesia in Python.
from gpaw import parsize_domain, parsize_bands
# Choose the largest possible parallelization over kpoint/spins
test_parsize_ks_pairs = gcd(self.nspins*self.nibzkpts, world.size)
remsize = world.size//test_parsize_ks_pairs
# If parsize_bands is not set, choose the largest possible
test_parsize_bands = parsize_bands or gcd(self.nbands, remsize)
# If parsize_bands is not set, choose as few domains as possible
test_parsize_domain = parsize_domain or (remsize//test_parsize_bands)
return test_parsize_domain, test_parsize_bands
def get_parsizes(self): #XXX NO LONGER IN UT_HSOPS?!?
# Careful, overwriting imported GPAW params may cause amnesia in Python.
from gpaw import parsize, parsize_bands
# Choose the largest possible parallelization over kpoint/spins
test_parsize_ks_pairs = gcd(self.nspins*self.nibzkpts, world.size)
remsize = world.size//test_parsize_ks_pairs
# If parsize_bands is not set, choose the largest possible
test_parsize_bands = parsize_bands or gcd(self.nbands, remsize)
# If parsize_bands is not set, choose as few domains as possible
test_parsize = parsize or (remsize//test_parsize_bands)
return test_parsize, test_parsize_bands
def setUp(self):
for virtvar in ['boundaries', 'celltype']:
assert getattr(self,virtvar) is not None, 'Virtual "%s"!' % virtvar
# Basic unit cell information:
pbc_c = {'zero' : (False,False,False), \
'periodic': (True,True,True), \
'mixed' : (True, False, True)}[self.boundaries]
a, b = self.a, 2**0.5*self.a
cell_cv = {'general' : np.array([[0,a,a],[a/2,0,a/2],[a/2,a/2,0]]),
'rotated' : np.array([[0,0,b],[b/2,0,0],[0,b/2,0]]),
'inverted' : np.array([[0,0,b],[0,b/2,0],[b/2,0,0]]),
'orthogonal': np.diag([b, b/2, b/2])}[self.celltype]
cell_cv = np.array([(4-3*pbc)*c_v for pbc,c_v in zip(pbc_c, cell_cv)])
# Decide how many kpoints to sample from the 1st Brillouin Zone
kpts_c = np.ceil((10/Bohr)/np.sum(cell_cv**2,axis=1)**0.5).astype(int)
kpts_c = tuple(kpts_c*pbc_c + 1-pbc_c)
bzk_kc = kpts2ndarray(kpts_c)
self.gamma = len(bzk_kc) == 1 and not bzk_kc[0].any()
#p = InputParameters()
#Z_a = self.atoms.get_atomic_numbers()
#xcfunc = XC(p.xc)
#setups = Setups(Z_a, p.setups, p.basis, p.lmax, xcfunc)
#symmetry, weight_k, self.ibzk_kc = reduce_kpoints(self.atoms, bzk_kc,
# setups, p.usesymm)
self.ibzk_kc = bzk_kc.copy() # don't use symmetry reduction of kpoints
self.nibzkpts = len(self.ibzk_kc)
self.ibzk_kv = kpoint_convert(cell_cv, skpts_kc=self.ibzk_kc)
# Parse parallelization parameters and create suitable communicators.
#parsize_domain, parsize_bands = create_parsize_minbands(self.nbands, world.size)
parsize_domain, parsize_bands = world.size//gcd(world.size, self.nibzkpts), 1
assert self.nbands % np.prod(parsize_bands) == 0
domain_comm, kpt_comm, band_comm = distribute_cpus(parsize_domain,
parsize_bands, self.nspins, self.nibzkpts)
# Set up band descriptor:
self.bd = BandDescriptor(self.nbands, band_comm)
# Set up grid descriptor:
N_c = np.round(np.sum(cell_cv**2, axis=1)**0.5 / self.h)
N_c += 4-N_c % 4 # makes domain decomposition easier
self.gd = GridDescriptor(N_c, cell_cv, pbc_c, domain_comm, parsize_domain)
self.assertEqual(self.gamma, np.all(~self.gd.pbc_c))
# What to do about kpoints?
self.kpt_comm = kpt_comm
if debug and world.rank == 0:
comm_sizes = tuple([comm.size for comm in [world, self.bd.comm, \
self.gd.comm, self.kpt_comm]])
print '%d world, %d band, %d domain, %d kpt' % comm_sizes
def setUp(self):
for virtvar in ['boundaries', 'celltype']:
assert getattr(self,virtvar) is not None, 'Virtual "%s"!' % virtvar
# Basic unit cell information:
pbc_c = {'zero' : (False,False,False), \
'periodic': (True,True,True), \
'mixed' : (True, False, True)}[self.boundaries]
a, b = self.a, 2**0.5*self.a
cell_cv = {'general' : np.array([[0,a,a],[a/2,0,a/2],[a/2,a/2,0]]),
'rotated' : np.array([[0,0,b],[b/2,0,0],[0,b/2,0]]),
'inverted' : np.array([[0,0,b],[0,b/2,0],[b/2,0,0]]),
'orthogonal': np.diag([b, b/2, b/2])}[self.celltype]
cell_cv = np.array([(4-3*pbc)*c_v for pbc,c_v in zip(pbc_c, cell_cv)])
# Decide how many kpoints to sample from the 1st Brillouin Zone
kpts_c = np.ceil((10/Bohr)/np.sum(cell_cv**2,axis=1)**0.5).astype(int)
kpts_c = tuple(kpts_c*pbc_c + 1-pbc_c)
bzk_kc = kpts2ndarray(kpts_c)
self.gamma = len(bzk_kc) == 1 and not bzk_kc[0].any()
#p = InputParameters()
#Z_a = self.atoms.get_atomic_numbers()
#xcfunc = XC(p.xc)
#setups = Setups(Z_a, p.setups, p.basis, p.lmax, xcfunc)
#symmetry, weight_k, self.ibzk_kc = reduce_kpoints(self.atoms, bzk_kc,
# setups, p.usesymm)
self.ibzk_kc = bzk_kc.copy() # don't use symmetry reduction of kpoints
self.nibzkpts = len(self.ibzk_kc)
self.ibzk_kv = kpoint_convert(cell_cv, skpts_kc=self.ibzk_kc)
# Parse parallelization parameters and create suitable communicators.
#parsize, parsize_bands = create_parsize_minbands(self.nbands, world.size)
parsize, parsize_bands = world.size//gcd(world.size, self.nibzkpts), 1
assert self.nbands % np.prod(parsize_bands) == 0
domain_comm, kpt_comm, band_comm = distribute_cpus(parsize,
parsize_bands, self.nspins, self.nibzkpts)
# Set up band descriptor:
self.bd = BandDescriptor(self.nbands, band_comm)
# Set up grid descriptor:
N_c = np.round(np.sum(cell_cv**2, axis=1)**0.5 / self.h)
N_c += 4-N_c % 4 # makes domain decomposition easier
self.gd = GridDescriptor(N_c, cell_cv, pbc_c, domain_comm, parsize)
self.assertEqual(self.gamma, np.all(~self.gd.pbc_c))
# What to do about kpoints?
self.kpt_comm = kpt_comm
if debug and world.rank == 0:
comm_sizes = tuple([comm.size for comm in [world, self.bd.comm, \
self.gd.comm, self.kpt_comm]])
print '%d world, %d band, %d domain, %d kpt' % comm_sizes
def frac(f, n=2 * 3 * 4 * 5):
if not isinstance(f, (int, float)):
return np.array([frac(a, n) for a in f]).T
if f == 0:
return 0, 1
x = n * f
if abs(x - round(x)) > 1e-6:
raise ValueError
x = int(round(x))
d = gcd(x, n)
return x // d, n // d
def frac(f, n=2 * 3 * 4 * 5):
if not isinstance(f, (int, float)):
return np.array([frac(a, n) for a in f]).T
if f == 0:
return 0, 1
x = n * f
if abs(x - round(x)) > 1e-6:
raise ValueError
x = int(round(x))
d = gcd(x, n)
return x // d, n // d
def distribute_cpus(parsize_domain, parsize_bands,
nspins, nibzkpts, comm=world,
idiotproof=True, mode='fd'):
nsk = nspins * nibzkpts
if mode in ['fd', 'lcao']:
if parsize_bands is None:
parsize_bands = 1
else:
# Plane wave mode:
if parsize_bands is None:
parsize_bands = comm.size // gcd(nsk, comm.size)
p = Parallelization(comm, nsk)
return p.build_communicators(domain=np.prod(parsize_domain),
band=parsize_bands)
N = 2000 # number of bands
repeats = 1
try:
N = int(sys.argv[1])
J = int(sys.argv[2])
except (IndexError, ValueError):
N = 6
J = 3
repeats = 3
# B: number of band groups
# D: number of domains
if parsize_bands is None:
if parsize_domain is None:
B = gcd(N, world.size)
D = world.size // B
else:
B = world.size // np.prod(parsize_domain)
D = parsize_domain
else:
B = parsize_bands
D = world.size // B
M = N // B # number of bands per group
assert M * B == N, 'M=%d, B=%d, N=%d' % (M,B,N)
h = 0.2 # grid spacing
a = h * G # side length of box
assert np.prod(D) * B == world.size, 'D=%s, B=%d, W=%d' % (D,B,world.size)
def prune_symmetries_atoms(self, spos_ac):
"""Remove symmetries that are not satisfied by the atoms."""
if len(spos_ac) == 0:
return
# Build lists of atom numbers for each type of atom - one
# list for each combination of atomic number, setup type,
# magnetic moment and basis set:
a_ij = {}
for a, id in enumerate(self.id_a):
if id in a_ij:
a_ij[id].append(a)
else:
a_ij[id] = [a]
a_j = a_ij[self.id_a[0]] # just pick the first species
# if supercell disable fractional translations:
if not self.symmorphic:
op_cc = np.identity(3, int)
ftrans_sc = spos_ac[a_j[1:]] - spos_ac[a_j[0]]
ftrans_sc -= np.rint(ftrans_sc)
for ft_c in ftrans_sc:
a_a = self.check_one_symmetry(spos_ac, op_cc, ft_c, a_ij)
if a_a is not None:
self.symmorphic = True
break
symmetries = []
ftsymmetries = []
# go through all possible symmetry operations
for op_cc in self.op_scc:
# first ignore fractional translations
a_a = self.check_one_symmetry(spos_ac, op_cc, [0, 0, 0], a_ij)
if a_a is not None:
symmetries.append((op_cc, [0, 0, 0], a_a))
elif not self.symmorphic:
# check fractional translations
sposrot_ac = np.dot(spos_ac, op_cc)
ftrans_jc = sposrot_ac[a_j] - spos_ac[a_j[0]]
ftrans_jc -= np.rint(ftrans_jc)
for ft_c in ftrans_jc:
try:
nom_c, denom_c = frac(ft_c)
except ValueError:
continue
ft_c = nom_c / denom_c
a_a = self.check_one_symmetry(spos_ac, op_cc, ft_c, a_ij)
if a_a is not None:
ftsymmetries.append((op_cc, ft_c, a_a))
for c, d in enumerate(denom_c):
self.gcd_c[c] = gcd(self.gcd_c[c] * d, d)
# Add symmetry operations with fractional translations at the end:
symmetries.extend(ftsymmetries)
self.op_scc = np.array([sym[0] for sym in symmetries])
self.ft_sc = np.array([sym[1] for sym in symmetries])
self.a_sa = np.array([sym[2] for sym in symmetries])
inv_cc = -np.eye(3, dtype=int)
self.has_inversion = (self.op_scc == inv_cc).all(2).all(1).any()
N = 2000 # number of bands
repeats = 20
try:
N = int(sys.argv[1])
K = int(sys.argv[2])
except (IndexError, ValueError):
N = 6
K = 3
repeats = 3
# B: number of band groups
# D: number of domains
if parsize_bands is None:
if parsize_domain is None:
B = gcd(N, world.size)
D = world.size // B
else:
B = world.size // np.prod(parsize_domain)
D = parsize_domain
else:
B = parsize_bands
D = world.size // B
M = N // B # number of bands per group
assert M * B == N, 'M=%d, B=%d, N=%d' % (M, B, N)
h = 0.2 # grid spacing
a = h * G # side length of box
assert np.prod(D) * B == world.size, 'D=%s, B=%d, W=%d' % (D, B, world.size)
def prune_symmetries_atoms(self, spos_ac):
"""Remove symmetries that are not satisfied by the atoms."""
if len(spos_ac) == 0:
return
# Build lists of atom numbers for each type of atom - one
# list for each combination of atomic number, setup type,
# magnetic moment and basis set:
a_ij = {}
for a, id in enumerate(self.id_a):
if id in a_ij:
a_ij[id].append(a)
else:
a_ij[id] = [a]
a_j = a_ij[self.id_a[0]] # just pick the first species
# if supercell disable fractional translations:
if not self.symmorphic:
op_cc = np.identity(3, int)
ftrans_sc = spos_ac[a_j[1:]] - spos_ac[a_j[0]]
ftrans_sc -= np.rint(ftrans_sc)
for ft_c in ftrans_sc:
a_a = self.check_one_symmetry(spos_ac, op_cc, ft_c, a_ij)
if a_a is not None:
self.symmorphic = True
break
symmetries = []
ftsymmetries = []
# go through all possible symmetry operations
for op_cc in self.op_scc:
# first ignore fractional translations
a_a = self.check_one_symmetry(spos_ac, op_cc, [0, 0, 0], a_ij)
if a_a is not None:
symmetries.append((op_cc, [0, 0, 0], a_a))
elif not self.symmorphic:
# check fractional translations
sposrot_ac = np.dot(spos_ac, op_cc)
ftrans_jc = sposrot_ac[a_j] - spos_ac[a_j[0]]
ftrans_jc -= np.rint(ftrans_jc)
for ft_c in ftrans_jc:
try:
nom_c, denom_c = frac(ft_c)
except ValueError:
continue
ft_c = nom_c / denom_c
a_a = self.check_one_symmetry(spos_ac, op_cc, ft_c, a_ij)
if a_a is not None:
ftsymmetries.append((op_cc, ft_c, a_a))
for c, d in enumerate(denom_c):
self.gcd_c[c] = gcd(self.gcd_c[c] * d, d)
# Add symmetry operations with fractional translations at the end:
symmetries.extend(ftsymmetries)
self.op_scc = np.array([sym[0] for sym in symmetries])
self.ft_sc = np.array([sym[1] for sym in symmetries])
self.a_sa = np.array([sym[2] for sym in symmetries])
inv_cc = -np.eye(3, dtype=int)
self.has_inversion = (self.op_scc == inv_cc).all(2).all(1).any()