def mk_eig(gf, job_tag, inv_type):
timer = q.Timer(f"py:mk_eig({job_tag},{inv_type})", True)
timer.start()
gpt_gf = g.convert(qg.gpt_from_qlat(gf), g.single)
parity = g.odd
params = get_lanc_params(job_tag, inv_type)
q.displayln_info(f"mk_eig: job_tag={job_tag} inv_type={inv_type}")
q.displayln_info(f"mk_eig: params={params}")
fermion_params = params["fermion_params"]
if "omega" in fermion_params:
qm = g.qcd.fermion.zmobius(gpt_gf, fermion_params)
else:
qm = g.qcd.fermion.mobius(gpt_gf, fermion_params)
w = g.qcd.fermion.preconditioner.eo2_ne(parity=parity)(qm)
def make_src(rng):
src = g.vspincolor(w.F_grid_eo)
# src[:] = g.vspincolor([[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]])
rng.cnormal(src)
src.checkerboard(parity)
return src
pit = g.algorithms.eigen.power_iteration(**params["pit_params"])
pit_ev, _, _ = pit(w.Mpc, make_src(g.random("lanc")))
q.displayln_info(f"mk_eig: pit_ev={pit_ev}")
#
cheby = g.algorithms.polynomial.chebyshev(params["cheby_params"])
irl = g.algorithms.eigen.irl(params["irl_params"])
evec, ev = irl(cheby(w.Mpc), make_src(g.random("lanc")))
evals = g.algorithms.eigen.evals(w.Mpc, evec, check_eps2=1e-6, real=True)
g.mem_report()
#
timer.stop()
return evec, evals
def perform(self, root):
global current_config, current_light_quark
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
if (current_light_quark is not None
and current_light_quark.evec_dir != self.evec_dir):
current_light_quark = None
if current_light_quark is None:
current_light_quark = light_quark(current_config, self.evec_dir)
prop_l = {
"sloppy": current_light_quark.prop_l_sloppy,
"exact": current_light_quark.prop_l_exact,
}[self.solver]
vcj = g.load(f"{root}/{self.conf}/pm_basis/basis")
c = g.coordinates(vcj[0])
c = c[c[:, 3] == self.t]
g.message(
f"t = {self.t}, ilist = {self.ilist}, basis size = {len(vcj)}, solver = {self.solver}"
)
root_job = f"{root}/{self.name}"
output = g.gpt_io.writer(f"{root_job}/propagators")
# create sources
srcD = [
g.vspincolor(current_config.l_exact.U_grid) for spin in range(4)
]
for i in self.ilist:
for spin in range(4):
srcD[spin][:] = 0
srcD[spin][c, spin, :] = vcj[i][c]
g.message("Norm of source:", g.norm2(srcD[spin]))
if i == 0:
g.message("Source at origin:", srcD[spin][0, 0, 0, 0])
g.message("Source at time-origin:", srcD[spin][0, 0, 0,
self.t])
prop = g.eval(prop_l * srcD)
g.mem_report(details=False)
for spin in range(4):
output.write(
{f"t{self.t}s{spin}c{i}_{self.solver}": prop[spin]})
output.flush()
maxiter=10,
)
light_sloppy_inverter = g.algorithms.inverter.defect_correcting(
g.algorithms.inverter.mixed_precision(light_innerL_inverter, g.single,
g.double),
eps=1e-8,
maxiter=2,
)
prop_l_low = l_sloppy.propagator(light_low_inverter)
prop_l_sloppy = l_exact.propagator(light_sloppy_inverter).grouped(6)
prop_l_exact = l_exact.propagator(light_exact_inverter).grouped(6)
# show available memory
g.mem_report(details=False)
# per job
for group, job, conf, jid, n in run_jobs:
g.message(f"""
Job {jid} / {n} : configuration {conf}, job tag {job}
""")
job_seed = job.split("_correlated")[0]
rng = g.random(f"hvp-conn-a2a-ensemble-{conf}-{job_seed}")
source_positions_low = [[
rng.uniform_int(min=0, max=L[i] - 1) for i in range(4)
] for j in range(jobs[job]["low"])]
"betastp": 0.0,
"maxiter": 20,
"Nminres": 7,
# "maxapply" : 100
})
# start vector
start = g.vspincolor(w.Mpc.vector_space[0].grid)
start[:] = g.vspincolor([[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]])
start.checkerboard(parity)
# generate eigenvectors
evec, ev = irl(c(w.Mpc), start) # , g.checkpointer("checkpoint")
# memory info
g.mem_report()
# print eigenvalues of NDagN as well
evals, eps2 = g.algorithms.eigen.evals(w.Mpc, evec, real=True)
assert all([e2 < 1e-11 for e2 in eps2])
# test low-mode approximation of inverse
inv = g.algorithms.inverter
lma = inv.deflate(evec, evals)(w.Mpc)
for i in range(len(evals)):
eps2 = g.norm2(evals[i] * lma * evec[i] - evec[i]) / g.norm2(
evec[i]) * evals[i]
g.message(f"Test low-mode approximation for evec[{i}]: {eps2}")
assert eps2 < 1e-11
# deflated solver
def step(self, mat, lmd, lme, evec, w, Nm, k):
assert k < Nm
verbose = g.default.is_verbose("irl")
ckpt = self.ckpt
alph = 0.0
beta = 0.0
evec_k = evec[k]
results = [w, alph, beta]
if ckpt.load(results):
w, alph, beta = results # use checkpoint
if verbose:
g.message("%-65s %-45s" %
("alpha[ %d ] = %s" % (k, alph), "beta[ %d ] = %s" %
(k, beta)))
else:
if self.params["mem_report"]:
g.mem_report(details=False)
# compute
t0 = g.time()
mat(w, evec_k)
t1 = g.time()
# allow to restrict maximal number of applications within run
self.napply += 1
if "maxapply" in self.params:
if self.napply == self.params["maxapply"]:
if verbose:
g.message(
"Maximal number of matrix applications reached")
sys.exit(0)
if k > 0:
w -= lme[k - 1] * evec[k - 1]
zalph = g.inner_product(evec_k, w)
alph = zalph.real
w -= alph * evec_k
beta = g.norm2(w)**0.5
w /= beta
t2 = g.time()
if k > 0:
g.orthogonalize(w, evec[0:k])
t3 = g.time()
ckpt.save([w, alph, beta])
if verbose:
g.message("%-65s %-45s %-50s" % (
"alpha[ %d ] = %s" % (k, zalph),
"beta[ %d ] = %s" % (k, beta),
" timing: %g s (matrix), %g s (ortho)" %
(t1 - t0, t3 - t2),
))
lmd[k] = alph
lme[k] = beta
if k < Nm - 1:
evec[k + 1] @= w
def __init__(self, config, evec_dir):
self.evec_dir = evec_dir
self.eig = g.load(evec_dir, grids=config.l_sloppy.F_grid_eo)
g.mem_report(details=False)
# pin coarse eigenvectors to GPU memory
self.pin = g.pin(self.eig[1], g.accelerator)
light_innerL_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.sequence(
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
fine_block=4,
linear_combination_block=32,
),
g.algorithms.inverter.split(
g.algorithms.inverter.cg({
"eps": 1e-8,
"maxiter": 200
}),
mpi_split=g.default.get_ivec("--mpi_split", None, 4),
),
),
)
light_innerH_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.sequence(
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
fine_block=4,
linear_combination_block=32,
),
g.algorithms.inverter.split(
g.algorithms.inverter.cg({
"eps": 1e-8,
"maxiter": 300
}),
mpi_split=g.default.get_ivec("--mpi_split", None, 4),
),
),
)
light_low_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
linear_combination_block=32,
fine_block=4,
),
)
light_exact_inverter = g.algorithms.inverter.defect_correcting(
g.algorithms.inverter.mixed_precision(light_innerH_inverter,
g.single, g.double),
eps=1e-8,
maxiter=10,
)
light_sloppy_inverter = g.algorithms.inverter.defect_correcting(
g.algorithms.inverter.mixed_precision(light_innerL_inverter,
g.single, g.double),
eps=1e-8,
maxiter=2,
)
self.prop_l_low = config.l_sloppy.propagator(light_low_inverter)
self.prop_l_sloppy = config.l_exact.propagator(
light_sloppy_inverter).grouped(2)
self.prop_l_exact = config.l_exact.propagator(
light_exact_inverter).grouped(2)
def perform(self, root):
global basis_size, sloppy_per_job, T, current_config
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
output_correlator = g.corr_io.writer(f"{root}/{self.name}/head.dat")
vcj = g.load(f"{root}/{self.conf}/pm_basis/basis")
for i0 in range(0, basis_size, sloppy_per_job):
half_peramb = {}
for l in g.load(
f"{root}/{self.conf}/pm_{self.solver}_t{self.t}_i{i0}/propagators"
):
for x in l:
half_peramb[x] = l[x]
g.mem_report(details=False)
vc = g.vcolor(vcj[0].grid)
c = g.coordinates(vc)
prec = {"sloppy": 0, "exact": 1}[self.solver]
for spin_prime in range(4):
plan = None
for spin in range(4):
for i in range(i0, i0 + sloppy_per_job):
hp = half_peramb[f"t{self.t}s{spin}c{i}_{self.solver}"]
if plan is None:
plan = g.copy_plan(vc, hp)
plan.destination += vc.view[c]
plan.source += hp.view[c, spin_prime, :]
plan = plan()
plan(vc, hp)
t0 = g.time()
slc_j = [
g(g.adj(vcj[j]) * vc) for j in range(basis_size)
]
t1 = g.time()
slc = g.slice(slc_j, 3)
t2 = g.time()
for j in range(basis_size):
output_correlator.write(
f"output/peramb_prec{prec}/n_{j}_{i}_s_{spin_prime}_{spin}_t_{self.t}",
slc[j],
)
t3 = g.time()
if i % 50 == 0:
g.message(spin_prime, spin, i, "Timing", t1 - t0,
t2 - t1, t3 - t2)
output_correlator.close()
def blockStep(self, mat, lmd, lme, evec, w, w_copy, Nm, b, Nu):
assert (b+1)*Nu <= Nm
verbose = g.default.is_verbose("irl")
ckpt = self.ckpt
alph = 0.0
beta = 0.0
L= b*Nu
R= (b+1)*Nu
for k in range (L,R):
if self.params["mem_report"]:
g.mem_report(details=False)
# compute
t0 = g.time()
if not ckpt.load(w[k-L]):
mat(w[k-L], evec[k])
# mat(v, B)
ckpt.save(w[k-L])
t1 = g.time()
# allow to restrict maximal number of applications within run
self.napply += 1
if "maxapply" in self.params:
if self.napply == self.params["maxapply"]:
if verbose:
g.message("Maximal number of matrix applications reached")
sys.exit(0)
for u in range (Nu):
for k in range (u,Nu):
ip=g.innerProduct(evec[L+k],evec[L+u])
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],evec[%d])=%e %e"% (L+k,L+u,ip.real,ip.imag))
if b > 0:
for u in range (Nu):
for k in range (L-Nu+u,L):
w[u] -= np.conjugate(lme[u,k]) * evec[k]
for k in range (L-Nu+u,L):
ip=g.innerProduct(evec[k],w[u])
# if g.norm2(ip)>1e-6:
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],w[%d])=%e %e"% (k,u,ip.real,ip.imag))
else:
for u in range (Nu):
g.message("norm(evec[%d])=%e"%(u,g.norm2(evec[u])))
for u in range (Nu):
for k in range (L+u,R):
lmd[u][k] = g.innerProduct(evec[k],w[u])
lmd[k-L][L+u]=np.conjugate(lmd[u][k])
lmd[u][L+u]=np.real(lmd[u][L+u])
for u in range (Nu):
for k in range (L,R):
w[u] -= lmd[u][k]*evec[k]
for k in range (L,R):
ip=g.innerProduct(evec[k],w[u])
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],w[%d])=%e %e"% (k,u,ip.real,ip.imag))
w_copy[u] = g.copy(w[u]);
for u in range (Nu):
for k in range (L,R):
lme[u][k]=0.;
for u in range (Nu):
g.orthogonalize(w[u],evec[0:R])
w[u] *= 1.0 / g.norm2(w[u]) ** 0.5
for k in range (R):
ip=g.innerProduct(evec[k],w[u])
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],w[%d])=%e %e"% (k,u,ip.real,ip.imag))
for u in range (Nu):
g.orthogonalize(w[u],evec[0:R])
w[u] *= 1.0 / g.norm2(w[u]) ** 0.5
for k in range (R):
ip=g.innerProduct(evec[k],w[u])
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],w[%d])=%e %e"% (k,u,ip.real,ip.imag))
for u in range (0,Nu):
if u >0:
g.orthogonalize(w[u],w[0:u])
w[u] *= 1.0 / g.norm2(w[u]) ** 0.5
for k in range (u):
ip=g.innerProduct(w[k],w[u])
if np.abs(ip) >1e-6:
g.message("inner(w[%d],w[%d])=%e %e"% (k,u,ip.real,ip.imag))
ip=g.innerProduct(w[u],w[u])
g.message("inner(w[%d],w[%d])=%e %e"% (u,u,ip.real,ip.imag))
for u in range (Nu):
for v in range (u,Nu):
lme[u][L+v] = g.innerProduct(w[u],w_copy[v])
lme[u][L+u] = np.real(lme[u][L+u])
t3 = g.time()
for u in range (Nu):
for k in range (L+u,R):
g.message(
" In block %d, beta[%d][%d]=%e %e"
%( b, u, k-b*Nu,lme[u][k].real,lme[u][k].imag )
)
# ckpt.save([w, alph, beta])
if b < (Nm/Nu - 1):
for u in range (Nu):
evec[R+u] = g.copy(w[u])
ip=g.innerProduct(evec[R+u],evec[R+u])
if np.abs(ip) >1e-6:
g.message("inner(evec[%d],evec[%d])=%e %e"% (R+u,R+u,ip.real,ip.imag))
def mk_ceig(gf, job_tag, inv_type):
timer = q.Timer(f"py:mk_ceig({job_tag},{inv_type})", True)
timer.start()
gpt_gf = g.convert(qg.gpt_from_qlat(gf), g.single)
parity = g.odd
params = get_lanc_params(job_tag, inv_type)
q.displayln_info(f"mk_ceig: job_tag={job_tag} inv_type={inv_type}")
q.displayln_info(f"mk_ceig: params={params}")
fermion_params = params["fermion_params"]
if "omega" in fermion_params:
qm = g.qcd.fermion.zmobius(gpt_gf, fermion_params)
else:
qm = g.qcd.fermion.mobius(gpt_gf, fermion_params)
w = g.qcd.fermion.preconditioner.eo2_ne(parity=parity)(qm)
def make_src(rng):
src = g.vspincolor(w.F_grid_eo)
# src[:] = g.vspincolor([[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]])
rng.cnormal(src)
src.checkerboard(parity)
return src
pit = g.algorithms.eigen.power_iteration(**params["pit_params"])
pit_ev, _, _ = pit(w.Mpc, make_src(g.random("lanc")))
q.displayln_info(f"mk_ceig: pit_ev={pit_ev}")
#
cheby = g.algorithms.polynomial.chebyshev(params["cheby_params"])
irl = g.algorithms.eigen.irl(params["irl_params"])
evec, ev = irl(cheby(w.Mpc), make_src(g.random("lanc")))
evals = g.algorithms.eigen.evals(w.Mpc, evec, check_eps2=1e-6, real=True)
g.mem_report()
#
inv = g.algorithms.inverter
#
cparams = get_clanc_params(job_tag, inv_type)
q.displayln_info(f"mk_ceig: cparams={cparams}")
#
grid_coarse = g.block.grid(w.F_grid_eo,
[get_ls_from_fermion_params(fermion_params)] +
cparams["block"])
nbasis = cparams["nbasis"]
basis = evec[0:nbasis]
b = g.block.map(grid_coarse, basis)
for i in range(2):
b.orthonormalize()
del evec
gc.collect()
#
ccheby = g.algorithms.polynomial.chebyshev(cparams["cheby_params"])
cop = b.coarse_operator(ccheby(w.Mpc))
#
cstart = g.vcomplex(grid_coarse, nbasis)
cstart[:] = g.vcomplex([1] * nbasis, nbasis)
eps2 = g.norm2(cop * cstart - b.project * ccheby(w.Mpc) * b.promote *
cstart) / g.norm2(cstart)
g.message(f"Test coarse-grid promote/project cycle: {eps2}")
cirl = g.algorithms.eigen.irl(cparams["irl_params"])
cevec, cev = cirl(cop, cstart)
#
smoother = inv.cg(cparams["smoother_params"])(w.Mpc)
smoothed_evals = []
tmpf = g.lattice(basis[0])
for i, cv in enumerate(cevec):
tmpf @= smoother * b.promote * cv
smoothed_evals = smoothed_evals + g.algorithms.eigen.evals(
w.Mpc, [tmpf], check_eps2=10, real=True)
g.mem_report()
#
timer.stop()
return basis, cevec, smoothed_evals
