def perform(self, root):
global basis_size, sloppy_per_job, T, current_config
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
output_correlator = g.corr_io.writer(f"{root}/{self.name}/head.dat")
# <np,sp| D^{-1} Gamma D^{-1} |n,s> = < (D^{-1})^\dagger |np,sp> | Gamma | D^{-1} |n,s > >
# = < Gamma5 D^{-1} Gamma5 |np,sp> | Gamma | D^{-1} |n,s > >
# = < D^{-1} |np,sp> | Gamma5 Gamma | D^{-1} |n,s > > gamma5_sign[sp]
gamma5_sign = [1.0, 1.0, -1.0, -1.0]
indices = [0, 1, 2, 5]
prec = {"sloppy": 0, "exact": 1}[self.solver]
for i0 in range(0, basis_size, sloppy_per_job):
half_peramb_i = {}
for l in g.load(
f"{root}/{self.conf}/pm_{self.solver}_t{self.t}_i{i0}/propagators"
):
for x in l:
half_peramb_i[x] = l[x]
for j0 in range(0, basis_size, sloppy_per_job):
if j0 == i0:
half_peramb_j = half_peramb_i
else:
half_peramb_j = {}
for l in g.load(
f"{root}/{self.conf}/pm_{self.solver}_t{self.t}_i{j0}/propagators"
):
for x in l:
half_peramb_j[x] = l[x]
for i in range(i0, i0 + sloppy_per_job):
for spin in range(4):
g.message(i, spin)
hp_i = half_peramb_i[
f"t{self.t}s{spin}c{i}_{self.solver}"]
for mu in indices:
hp_i_gamma = g(g.gamma[5] * g.gamma[mu] * hp_i)
for spin_prime in range(4):
slc_j = [
g(gamma5_sign[spin_prime] *
g.adj(half_peramb_j[
f"t{self.t}s{spin_prime}c{j}_{self.solver}"]
) * hp_i_gamma)
for j in range(j0, j0 + sloppy_per_job)
]
slc = g.slice(slc_j, 3)
for j in range(j0, j0 + sloppy_per_job):
output_correlator.write(
f"output/G{mu}_prec{prec}/n_{j}_{i}_s_{spin_prime}_{spin}_t_{self.t}",
slc[j - j0],
)
output_correlator.close()
def perform(self, root):
global basis_size, T, current_config
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
c = None
vcj = [
g.vcolor(current_config.l_exact.U_grid) for jr in range(basis_size)
]
for vcjj in vcj:
vcjj[:] = 0
for tprime in range(T):
basis_evec, basis_evals = g.load(self.basis_fmt %
(self.conf, tprime))
plan = g.copy_plan(vcj[0],
basis_evec[0],
embed_in_communicator=vcj[0].grid)
c = g.coordinates(basis_evec[0])
plan.destination += vcj[0].view[np.hstack(
(c, np.ones((len(c), 1), dtype=np.int32) * tprime))]
plan.source += basis_evec[0].view[c]
plan = plan()
for l in range(basis_size):
plan(vcj[l], basis_evec[l])
for l in range(basis_size):
g.message("Check norm:", l, g.norm2(vcj[l]))
g.save(f"{root}/{self.name}/basis", vcj)
def perform(self, root):
global current_config, current_light_quark
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
if (current_light_quark is not None
and current_light_quark.evec_dir != self.evec_dir):
current_light_quark = None
if current_light_quark is None:
current_light_quark = light_quark(current_config, self.evec_dir)
prop_l = {
"sloppy": current_light_quark.prop_l_sloppy,
"exact": current_light_quark.prop_l_exact,
}[self.solver]
vcj = g.load(f"{root}/{self.conf}/pm_basis/basis")
c = g.coordinates(vcj[0])
c = c[c[:, 3] == self.t]
g.message(
f"t = {self.t}, ilist = {self.ilist}, basis size = {len(vcj)}, solver = {self.solver}"
)
root_job = f"{root}/{self.name}"
output = g.gpt_io.writer(f"{root_job}/propagators")
# create sources
srcD = [
g.vspincolor(current_config.l_exact.U_grid) for spin in range(4)
]
for i in self.ilist:
for spin in range(4):
srcD[spin][:] = 0
srcD[spin][c, spin, :] = vcj[i][c]
g.message("Norm of source:", g.norm2(srcD[spin]))
if i == 0:
g.message("Source at origin:", srcD[spin][0, 0, 0, 0])
g.message("Source at time-origin:", srcD[spin][0, 0, 0,
self.t])
prop = g.eval(prop_l * srcD)
g.mem_report(details=False)
for spin in range(4):
output.write(
{f"t{self.t}s{spin}c{i}_{self.solver}": prop[spin]})
output.flush()
def perform(self, root):
global basis_size, sloppy_per_job, T, current_config, compress_ratio
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
U = current_config.U
reduced_mpi = [x for x in U[0].grid.mpi]
for i in range(len(reduced_mpi)):
if reduced_mpi[i] % 2 == 0:
reduced_mpi[i] //= 2
# create random selection of points with same spatial sites on each sink time slice
# use different spatial sites for each source time-slice
# this should be optimal for the local operator insertions
rng = g.random(f"sparse2_{self.conf}_{self.t}")
grid = U[0].grid
t0 = grid.ldimensions[3] * grid.processor_coor[3]
t1 = t0 + grid.ldimensions[3]
spatial_sites = int(compress_ratio * np.prod(grid.ldimensions[0:3]))
spatial_coordinates = rng.choice(g.coordinates(U[0]), spatial_sites)
local_coordinates = np.repeat(spatial_coordinates, t1 - t0, axis=0)
for t in range(t0, t1):
local_coordinates[t - t0::t1 - t0, 3] = t
sdomain = g.domain.sparse(current_config.l_exact.U_grid,
local_coordinates)
half_peramb = {"sparse_domain": sdomain}
for i0 in range(0, basis_size, sloppy_per_job):
for l in g.load(
f"{root}/{self.conf}/pm_{self.solver}_t{self.t}_i{i0}/propagators"
):
for x in l:
S = sdomain.lattice(l[x].otype)
sdomain.project(S, l[x])
half_peramb[x] = S
g.message(x)
g.save(
f"{root}/{self.name}/propagators",
half_peramb,
g.format.gpt({"mpi": reduced_mpi}),
)
def __init__(self, conf_file):
self.conf_file = conf_file
self.U = g.load(conf_file)
self.L = self.U[0].grid.fdimensions
self.l_exact = g.qcd.fermion.mobius(
self.U,
{
"mass": 0.00054,
"M5": 1.8,
"b": 1.5,
"c": 0.5,
"Ls": 12,
"boundary_phases": [1.0, 1.0, 1.0, -1.0],
},
)
self.l_sloppy = self.l_exact.converted(g.single)
def perform(self, root):
global basis_size, T
output_correlator = g.corr_io.writer(f"{root}/{self.name}/head.dat")
vcj = g.load(f"{root}/{self.conf}/pm_basis/basis")
for m in self.mom:
mom_str = "_".join([str(x) for x in m])
p = np.array([
2.0 * np.pi / vcj[0].grid.gdimensions[i] * m[i]
for i in range(3)
] + [0])
phase = g.complex(vcj[0].grid)
phase[:] = 1
phase @= g.exp_ixp(p) * phase
g.message("L = ", vcj[0].grid.gdimensions)
g.message("Momentum", p, m)
for n in range(basis_size):
t0 = g.time()
vc_n = g(phase * vcj[n])
t1 = g.time()
slc_nprime = [
g(g.adj(vcj[nprime]) * vc_n)
for nprime in range(basis_size)
]
t2 = g.time()
slc = g.slice(slc_nprime, 3)
t3 = g.time()
for nprime in range(basis_size):
output_correlator.write(
f"output/mom/{mom_str}_n_{nprime}_{n}", slc[nprime])
t4 = g.time()
if n % 10 == 0:
g.message(n, "Timing", t1 - t0, t2 - t1, t3 - t2, t4 - t3)
output_correlator.close()
g.mem_report()
# fermion
q = params["fmatrix"](U)
# load basis vectors
nbasis = params["nbasis"]
# fg_basis,fg_cevec,fg_feval = g.load(params["basis"],{
# "grids" : q.F_grid_eo, "nmax" : nbasis,
# "advise_basis" : g.infrequent_use,
# "advise_cevec" : g.infrequent_use
# })
rng = g.random("test")
try:
fg_basis = g.load("basis", {"grids": q.F_grid_eo})[0]
except g.LoadError:
fg_basis = g.advise(
[g.vspincolor(q.F_grid_eo) for i in range(nbasis)], g.infrequent_use
)
rng.zn(fg_basis)
g.save("basis", [fg_basis])
# g.mem_report()
# g.prefetch( fg_basis, g.to_accelerator)
# g.mem_report()
# w=fg_basis[-1]
# g.orthogonalize(w,fg_basis[0:1])
# g.orthogonalize(w,fg_basis[0:15])
import numpy as np
import sys
conf = g.default.get_single("--conf", None)
g.message(f"Fixing {conf}")
evec_in = (
"/gpfs/alpine/phy138/proj-shared/phy138flavor/lehner/runs/summit-96I-" +
conf + "-256/lanczos.output")
evec_out = (
"/gpfs/alpine/phy138/proj-shared/phy138flavor/lehner/runs/summit-96I-" +
conf + "-256/lanczos.output.fixed")
fmt = g.format.cevec({"nsingle": 100, "max_read_blocks": 16})
U = g.convert(
g.load(
"/gpfs/alpine/phy138/proj-shared/phy138flavor/chulwoo/evols/96I2.8Gev/evol0/configurations/ckpoint_lat."
+ conf),
g.single,
)
eps_norm = 1e-4
eps2_evec = 1e-5
eps_eval = 1e-2
nskip = 1
load_from_alternative_scheme = True
qz = g.qcd.fermion.mobius(
U,
{
"mass": 0.00054,
"M5": 1.8,
"b": 1.5,
def __init__(self, config, evec_dir):
self.evec_dir = evec_dir
self.eig = g.load(evec_dir, grids=config.l_sloppy.F_grid_eo)
g.mem_report(details=False)
# pin coarse eigenvectors to GPU memory
self.pin = g.pin(self.eig[1], g.accelerator)
light_innerL_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.sequence(
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
fine_block=4,
linear_combination_block=32,
),
g.algorithms.inverter.split(
g.algorithms.inverter.cg({
"eps": 1e-8,
"maxiter": 200
}),
mpi_split=g.default.get_ivec("--mpi_split", None, 4),
),
),
)
light_innerH_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.sequence(
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
fine_block=4,
linear_combination_block=32,
),
g.algorithms.inverter.split(
g.algorithms.inverter.cg({
"eps": 1e-8,
"maxiter": 300
}),
mpi_split=g.default.get_ivec("--mpi_split", None, 4),
),
),
)
light_low_inverter = g.algorithms.inverter.preconditioned(
g.qcd.fermion.preconditioner.eo1_ne(parity=g.odd),
g.algorithms.inverter.coarse_deflate(
self.eig[1],
self.eig[0],
self.eig[2],
block=400,
linear_combination_block=32,
fine_block=4,
),
)
light_exact_inverter = g.algorithms.inverter.defect_correcting(
g.algorithms.inverter.mixed_precision(light_innerH_inverter,
g.single, g.double),
eps=1e-8,
maxiter=10,
)
light_sloppy_inverter = g.algorithms.inverter.defect_correcting(
g.algorithms.inverter.mixed_precision(light_innerL_inverter,
g.single, g.double),
eps=1e-8,
maxiter=2,
)
self.prop_l_low = config.l_sloppy.propagator(light_low_inverter)
self.prop_l_sloppy = config.l_exact.propagator(
light_sloppy_inverter).grouped(2)
self.prop_l_exact = config.l_exact.propagator(
light_exact_inverter).grouped(2)
"Nm": 250,
"resid": 1e-14,
"betastp": 0.0,
"maxiter": 20,
"Nminres": 0,
})
c = g.algorithms.polynomial.chebyshev({"low": 0.009, "high": 2.3, "order": 40})
# create destination directory
if not os.path.exists(destination):
os.makedirs(destination, exist_ok=True)
# load gauge field
g.message(f"Loading {config}")
U = g.load(config)
# smear gauge links in ultra-local manner in time but heavily in space
Nt = U[0].grid.gdimensions[3]
g.message("Plaquette before", g.qcd.gauge.plaquette(U))
config_smeared = f"{destination}/smeared_lat.{cnr}"
try:
U = g.load(config_smeared)
except g.LoadError:
U0 = g.copy(U)
for t in range(Nt):
g.message("Time slice", t)
U_temp = [g.lattice(u) for u in U]
================================================================================
HVP connected run on summit ; this run will attempt:
================================================================================
""")
for group, job, conf, jid, n in run_jobs:
g.message(f"""
Job {jid} / {n} : configuration {conf}, job tag {job}
""")
# configuration needs to be the same for all jobs, so load eigenvectors and configuration
conf = run_jobs[0][2]
group = run_jobs[0][0]
U = g.load(groups[group]["conf_fmt"] % conf)
L = U[0].grid.fdimensions
l_exact = g.qcd.fermion.mobius(
U,
{
"mass": 0.00054,
"M5": 1.8,
"b": 1.5,
"c": 0.5,
"Ls": 12,
"boundary_phases": [1.0, 1.0, 1.0, -1.0],
},
)
l_sloppy = l_exact.converted(g.single)
for i in range(2):
b.orthonormalize()
# save in fixed layout
g.save(
f"{work_dir}/cevec",
[basis, cevec, feval],
g.format.cevec({
"nsingle": nsingle,
"max_read_blocks": 16,
"mpi": [1, 2, 2, 2, 2]
}),
)
# and load again to verify
basis2, cevec2, feval2 = g.load(f"{work_dir}/cevec", {"grids": fgrid})
assert len(basis) == len(basis2)
assert len(cevec) == len(cevec2)
assert len(feval) == len(feval2)
for i in range(len(basis)):
eps2 = g.norm2(basis[i] - basis2[i]) / g.norm2(basis[i])
g.message(f"basis {i} resid {eps2}")
if i < nsingle:
assert eps2 == 0.0
else:
assert eps2 < 1e-9
for i in range(len(cevec)):
eps2 = g.norm2(cevec[i] - cevec2[i]) / g.norm2(cevec[i])
rng = g.random("test")
# cold start
U = g.qcd.gauge.unit(g.grid([48, 48, 48, 192], g.double))
latest_it = None
it0 = 0
dst = g.default.get("--root", None)
N = 4000
for it in range(N):
if os.path.exists(f"{dst}/ckpoint_lat.{it}"):
latest_it = it
if latest_it is not None:
g.copy(U, g.load(f"{dst}/ckpoint_lat.{latest_it}"))
rng = g.random(f"test{dst}{latest_it}", "vectorized_ranlux24_24_64")
it0 = latest_it + 1
# gauge field obc
def project_open_bc(f):
f[3][:, :, :, f[3].grid.gdimensions[3] - 1] = 0
return f
project_open_bc(U)
pc = g.qcd.fermion.preconditioner
inv = g.algorithms.inverter
eofa_ratio = g.qcd.pseudofermion.action.exact_one_flavor_ratio
#!/usr/bin/env python3
import gpt as g
# parameters
config = g.default.get("--config", None)
evec_light = g.default.get("--evec_light", None)
# abbreviations
pc = g.qcd.fermion.preconditioner
inv = g.algorithms.inverter
# load config
U = g.load(config)
# sloppy strange quark
strange_sloppy = g.qcd.fermion.zmobius(
g.convert(U, g.single),
{
"mass":
0.0850,
"M5":
1.8,
"b":
1.0,
"c":
0.0,
"omega": [
1.0903256131299373,
0.9570283702230611,
0.7048886040934104,
0.48979921782791747,
def perform(self, root):
global basis_size, sloppy_per_job, T, current_config
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
output_correlator = g.corr_io.writer(f"{root}/{self.name}/head.dat")
vcj = g.load(f"{root}/{self.conf}/pm_basis/basis")
for i0 in range(0, basis_size, sloppy_per_job):
half_peramb = {}
for l in g.load(
f"{root}/{self.conf}/pm_{self.solver}_t{self.t}_i{i0}/propagators"
):
for x in l:
half_peramb[x] = l[x]
g.mem_report(details=False)
vc = g.vcolor(vcj[0].grid)
c = g.coordinates(vc)
prec = {"sloppy": 0, "exact": 1}[self.solver]
for spin_prime in range(4):
plan = None
for spin in range(4):
for i in range(i0, i0 + sloppy_per_job):
hp = half_peramb[f"t{self.t}s{spin}c{i}_{self.solver}"]
if plan is None:
plan = g.copy_plan(vc, hp)
plan.destination += vc.view[c]
plan.source += hp.view[c, spin_prime, :]
plan = plan()
plan(vc, hp)
t0 = g.time()
slc_j = [
g(g.adj(vcj[j]) * vc) for j in range(basis_size)
]
t1 = g.time()
slc = g.slice(slc_j, 3)
t2 = g.time()
for j in range(basis_size):
output_correlator.write(
f"output/peramb_prec{prec}/n_{j}_{i}_s_{spin_prime}_{spin}_t_{self.t}",
slc[j],
)
t3 = g.time()
if i % 50 == 0:
g.message(spin_prime, spin, i, "Timing", t1 - t0,
t2 - t1, t3 - t2)
output_correlator.close()
f"gauge.bc_{bc}.bin",
f"src.bc_{bc}.bin",
f"dst_Dw_dble.bc_{bc}.bin",
f"dst_Dwee_dble.bc_{bc}.bin",
f"dst_Dweo_dble.bc_{bc}.bin",
f"dst_Dwoe_dble.bc_{bc}.bin",
f"dst_Dwoo_dble.bc_{bc}.bin",
f"dst_Dwdag_dble.bc_{bc}.bin",
]
# request test files
for f in request_files:
g.repository.load(f"{work_dir}/{f}", f"gpt://tests/qcd/openqcd_dslash/{f}")
# load fields from files
U = g.load(f"{work_dir}/gauge.bc_{bc}.bin")
src = read_openqcd_fermion(f"{work_dir}/src.bc_{bc}.bin")
dst_oqcd_M = read_openqcd_fermion(f"{work_dir}/dst_Dw_dble.bc_{bc}.bin")
dst_oqcd_Moo = read_openqcd_fermion(f"{work_dir}/dst_Dwoo_dble.bc_{bc}.bin", g.odd)
dst_oqcd_Mee = read_openqcd_fermion(f"{work_dir}/dst_Dwee_dble.bc_{bc}.bin", g.even)
dst_oqcd_Moe = read_openqcd_fermion(f"{work_dir}/dst_Dwoe_dble.bc_{bc}.bin", g.odd)
dst_oqcd_Meo = read_openqcd_fermion(f"{work_dir}/dst_Dweo_dble.bc_{bc}.bin", g.even)
dst_oqcd_Mdag = read_openqcd_fermion(f"{work_dir}/dst_Dwdag_dble.bc_{bc}.bin")
# construct methods of our dirac operators from the read in fields
dst_oqcd_Dhop = g.lattice(dst_oqcd_M)
dst_oqcd_Mdiag = g.lattice(dst_oqcd_M)
g.set_checkerboard(dst_oqcd_Dhop, dst_oqcd_Moe)
g.set_checkerboard(dst_oqcd_Dhop, dst_oqcd_Meo)
g.set_checkerboard(dst_oqcd_Mdiag, dst_oqcd_Mee)
g.set_checkerboard(dst_oqcd_Mdiag, dst_oqcd_Moo)
# parameters
fn = g.default.get("--params", "params.txt")
params = g.params(fn, verbose=True)
# load configuration
U = params["config"]
# show available memory
g.mem_report()
# fermion
q = params["fmatrix"](U)
# load basis vectors
basis, feval = g.load(params["basis"])
nbasis = len(basis)
# memory info
g.mem_report()
# norms
for i in range(nbasis):
g.message("Norm2 of basis[%d] = %g" % (i, g.norm2(basis[i])))
g.mem_report()
# prepare and test basis
for i in range(nbasis):
g.message(i)
g.algorithms.eigen.evals(q.Mpc, [basis[i]], check_eps2=1e-4, real=True)
"mpi": [
2,
2,
2,
1,
] # save fields in 2 x 2 x 1 x 1 processor grid instead of --mpi grid
}),
)
#
# load function
#
# - g.load(fn) loads everything in fn and creates new grids as needed
# - g.load(fn,{ "grids" : ..., "paths" : ... }) both grids and paths are optional parameters and may be lists,
# grids are re-used when loading, paths restricts which items to load (allows for glob.glob syntax /U/*)
res = g.load(f"{work_dir}/out")
for i in range(4):
eps2 = g.norm2(res["U"][i] - U[i])
g.message("Test first restore of U[%d]:" % i, eps2)
assert eps2 < 1e-25
res = g.load(f"{work_dir}/out2", {"paths": "/U/*"})
for i in range(4):
eps2 = g.norm2(res["U"][i] - U[i])
g.message("Test second restore of U[%d]:" % i, eps2)
assert eps2 < 1e-25
# checkpointer save
ckpt = g.checkpointer(f"{work_dir}/ckpt")
alpha = 0.125
import gpt
import numpy
# workdir
if "WORK_DIR" in os.environ:
work_dir = os.environ["WORK_DIR"]
else:
work_dir = "."
# request test files
files = ["psrc-prop-0.field", "pion-corr.txt"]
for f in files:
gpt.repository.load(f"{work_dir}/{f}", f"gpt://tests/io/qlat/{f}")
# load field
prop = gpt.load(f"{work_dir}/psrc-prop-0.field")
gpt.message("Grid from qlat propagator =", prop.grid)
# calculate correlator
corr_pion = gpt.slice(gpt.trace(gpt.adj(prop) * prop), 3)
# load reference
with open(f"{work_dir}/pion-corr.txt", "r") as f:
txt = f.readlines()
# read lines corresponding to real part of time slices and
# check difference w.r.t. what we have loaded above
for i in range(8):
ref = float(txt[1 + i * 2].split(" ")[-1][:-1])
diff = abs(ref - corr_pion[i].real)
assert diff < 1e-7 # propagator was computed in single precision
#!/usr/bin/env python3
#
# Authors: Christoph Lehner 2020
#
import gpt as g
import numpy as np
# load configuration
U = g.load("/hpcgpfs01/work/clehner/configs/openQCD/A250t000n54")
# do everything in single-precision
U = g.convert(U, g.single)
# use the gauge configuration grid
grid = U[0].grid
L = np.array(grid.fdimensions)
# quark
w = g.qcd.fermion.wilson_clover(
U,
{
"kappa": 0.137,
"csw_r": 0,
"csw_t": 0,
"xi_0": 1,
"nu": 1,
"isAnisotropic": False,
"boundary_phases": [1.0, 1.0, 1.0, -1.0],
},
)
#!/usr/bin/env python3
#
# Authors: Christoph Lehner 2020
#
# Desc.: Illustrate core concepts and features
#
import gpt as g
import numpy as np
import sys
import time
# load configuration
U = g.load("/hpcgpfs01/work/clehner/configs/openQCD/A250t000n54")
# do everything in single-precision
U = g.convert(U, g.single)
# use the gauge configuration grid
grid = U[0].grid
# quark
w = g.qcd.fermion.wilson_clover(
U,
{
"kappa": 0.13565,
"csw_r": 2.0171,
"csw_t": 2.0171,
"xi_0": 1,
"nu": 1,
"isAnisotropic": False,
"boundary_phases": [1.0, 1.0, 1.0, 1.0],
#
# Authors: Daniel Richtmann 2020
# Christoph Lehner 2020
#
# Desc.: Exercise linear solvers
#
import gpt as g
import numpy as np
import sys
import time
import os.path
# load configuration
#homedir = os.path.expanduser("~")
#U = g.load(homedir + "/configs/openqcd/test_16x8_pbcn6")
U = g.load("/hpcgpfs01/work/clehner/configs/32IDfine/ckpoint_lat.200") # TODO: add parallel RNG so we can do tests from random gauge configs
# do everything in single-precision
U = g.convert(U, g.single)
# use the gauge configuration grid
grid=U[0].grid
# quark
w=g.qcd.fermion.wilson_clover(U,{
"kappa" : 0.13565,
"csw_r" : 2.0171 / 2., # for now test with very heavy quark
"csw_t" : 2.0171 / 2.,
"xi_0" : 1,
"nu" : 1,
"isAnisotropic" : False,
#!/usr/bin/env python3
#
# Authors: Christoph Lehner 2020
#
# Desc.: Illustrate core concepts and features
#
import gpt as g
import numpy as np
import sys
import random
import cgpt
# load configuration
U = g.load("/hpcgpfs01/work/clehner/configs/32IDfine/ckpoint_lat.200")
# Show metadata of field
g.message("Metadata", U[0].metadata)
# to single precision
#U = g.convert(U, g.single)
# save in default gpt format
g.save(
"out",
{
"va\nl": [
0, 1, 3, "tes\n\0t", 3.123456789123456789, 1.123456789123456789e-7,
1 + 3.1231251251234123413j
], # fundamental data types
"np":
g.coordinates(U[0].grid), # write numpy array from root node
""")
if p_source is None:
g.message("Need to provide source file")
sys.exit(1)
if p_mpi_split is None:
g.message("Need to provide mpi_split")
sys.exit(1)
# create rng if needed
rng = None if p_rng_seed is None else g.random(p_rng_seed)
# load source
U = g.load(p_source)
# split in time
Nt = U[0].grid.gdimensions[3]
g.message(f"Separate {Nt} time slices")
Usep = [g.separate(u, 3) for u in U[0:3]]
Vt = [g.mcolor(Usep[0][0].grid) for t in range(Nt)]
cache = {}
split_grid = Usep[0][0].grid.split(p_mpi_split, Usep[0][0].grid.fdimensions)
g.message("Split grid")
Usep_split = [g.split(Usep[mu], split_grid, cache) for mu in range(3)]
Vt_split = g.split(Vt, split_grid, cache)
# optimizer
opt = g.algorithms.optimize
#
# Production code to generate fine-grid basis vectorscoarse-grid eigenvectors using existing
#
import gpt as g
# parameters
fn = g.default.get("--params", "params.txt")
params = g.params(fn, verbose=True)
# load configuration
U = params["config"]
# matrix to use
fmatrix = params["fmatrix"](U)
op = params["op"](fmatrix)
grid = op.grid[0]
# implicitly restarted lanczos
irl = params["method_evec"]
# run
start = g.vspincolor(grid)
start[:] = g.vspincolor([[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]])
start.checkerboard(g.odd) # traditionally, calculate odd-site vectors
try:
basis, ev = g.load("checkpoint", grids=grid)
except g.LoadError:
basis, ev = irl(op, start, params["checkpointer"])
g.save("checkpoint", (basis, ev))
def perform(self, root):
global basis_size, sloppy_per_job, T, current_config
if current_config is not None and current_config.conf_file != self.conf_file:
current_config = None
if current_config is None:
current_config = config(self.conf_file)
output_correlator = g.corr_io.writer(f"{root}/{self.name}/head.dat")
# <np,sp| D^{-1} Gamma D^{-1} |n,s> = < (D^{-1})^\dagger |np,sp> | Gamma | D^{-1} |n,s > >
# = < Gamma5 D^{-1} Gamma5 |np,sp> | Gamma | D^{-1} |n,s > >
# = < D^{-1} |np,sp> | Gamma5 Gamma | D^{-1} |n,s > > gamma5_sign[sp]
indices = [0, 1, 2, 5]
prec = {"sloppy": 0, "exact": 1}[self.solver]
half_peramb = g.load(
f"{root}/{self.conf}/pm_compressed_half_peramb_{self.solver}_t{self.t}/propagators"
)
sdomain = half_peramb["sparse_domain"]
scale = (sdomain.grid.gsites / sdomain.grid.Nprocessors /
len(sdomain.local_coordinates))
g.message("scale =", scale)
gamma5_sign = [1.0 * scale, 1.0 * scale, -1.0 * scale, -1.0 * scale]
for i0 in range(0, basis_size, sloppy_per_job):
half_peramb_i = {}
for i in range(i0, i0 + sloppy_per_job):
for spin in range(4):
f = g.vspincolor(sdomain.grid)
f[:] = 0
sdomain.promote(
f, half_peramb[f"t{self.t}s{spin}c{i}_{self.solver}"])
half_peramb_i[f"t{self.t}s{spin}c{i}_{self.solver}"] = f
for j0 in range(0, basis_size, sloppy_per_job):
if j0 == i0:
half_peramb_j = half_peramb_i
else:
half_peramb_j = {}
for j in range(j0, j0 + sloppy_per_job):
for spin in range(4):
f = g.vspincolor(sdomain.grid)
f[:] = 0
sdomain.promote(
f, half_peramb[
f"t{self.t}s{spin}c{j}_{self.solver}"])
half_peramb_j[
f"t{self.t}s{spin}c{j}_{self.solver}"] = f
for i in range(i0, i0 + sloppy_per_job):
for spin in range(4):
g.message(i, spin)
hp_i = g(sdomain.weight() * half_peramb_i[
f"t{self.t}s{spin}c{i}_{self.solver}"])
for mu in indices:
hp_i_gamma = g(g.gamma[5] * g.gamma[mu] * hp_i)
for spin_prime in range(4):
slc_j = [
g(gamma5_sign[spin_prime] *
g.adj(half_peramb_j[
f"t{self.t}s{spin_prime}c{j}_{self.solver}"]
) * hp_i_gamma)
for j in range(j0, j0 + sloppy_per_job)
]
slc = g.slice(slc_j, 3)
for j in range(j0, j0 + sloppy_per_job):
output_correlator.write(
f"output/G{mu}_prec{prec}/n_{j}_{i}_s_{spin_prime}_{spin}_t_{self.t}",
slc[j - j0],
)
output_correlator.close()
for mpi_layout in [[1, 1, 1, 1, 1], [1, 2, 2, 2, 2]]:
# save in fixed layout
g.save(
f"{work_dir}/cevec",
[basis, cevec, feval],
g.format.cevec({
"nsingle": nsingle,
"max_read_blocks": 8,
"mpi": mpi_layout
}),
)
# and load again to verify
basis2, cevec2, feval2 = g.load(f"{work_dir}/cevec", grids=fgrid)
assert len(basis) == len(basis2)
assert len(cevec) == len(cevec2)
assert len(feval) == len(feval2)
for i in range(len(basis)):
eps2 = g.norm2(basis[i] - basis2[i]) / g.norm2(basis[i])
g.message(f"basis {i} resid {eps2}")
if i < nsingle:
assert eps2 == 0.0
else:
assert eps2 < 1e-9
for i in range(len(cevec)):
eps2 = g.norm2(cevec[i] - cevec2[i]) / g.norm2(cevec[i])
# load configuration
U = params["config"]
# show available memory
g.mem_report()
# fermion
q = params["fmatrix"](U)
# load basis vectors
nbasis = params["nbasis"]
fg_basis, fg_cevec, fg_feval = g.load(
params["basis"],
{
"grids": q.F_grid_eo,
"nmax": nbasis,
"advise_basis": g.infrequent_use,
"advise_cevec": g.infrequent_use,
},
)
# memory info
g.mem_report()
# norms
for i in range(nbasis):
g.message("Norm2 of basis[%d] = %g" % (i, g.norm2(fg_basis[i])))
for i in range(nbasis):
g.message("Norm2 of cevec[%d] = %g" % (i, g.norm2(fg_cevec[i])))
2,
2,
2,
1,
] # save fields in 2 x 2 x 1 x 1 processor grid instead of --mpi grid
}
),
)
#
# load function
#
# - g.load(fn) loads everything in fn and creates new grids as needed
# - g.load(fn,{ "grids" : ..., "paths" : ... }) both grids and paths are optional parameters and may be lists,
# grids are re-used when loading, paths restricts which items to load (allows for glob.glob syntax /U/*)
res = g.load(f"{work_dir}/out")
for i in range(4):
eps2 = g.norm2(res["U"][i] - U[i])
g.message("Test first restore of U[%d]:" % i, eps2)
assert eps2 < 1e-25
eps2 = g.norm2(res["S"] - S)
g.message("Test sparse field restore:", eps2)
assert eps2 < 1e-25
# check load sparse lattice
U0prime2 = g.lattice(U[0])
U0prime2[:] = 0
res["sdomain"].promote(U0prime2, res["S"])
eps2 = g.norm2(U0prime - U0prime2)
def load_eig():
assert isinstance(eig, list)
assert len(eig) == 1
if eig[0] is None:
eig[0] = g.load(path_load, grids=grids)
return eig[0]
