def classifiers(args):
"""
Trains a binary classifier using the features
generated in the 'features' step.
Parameters
----------
args : dict
Configuration dict, typically loaded from YAML file
"""
if 'classifiers' in args.keys():
if 'rfc' in args['classifiers']:
with timer("Training Random Forest"):
train.randomForestClassifier(args['working'], args['working'])
with timer("Testing Random Forest"):
clf = os.path.join(args['working'], 'classifiers',
'rfc.joblib')
test.test_classifier(args['working'], args['working'],
args['test_data'], clf, args['darpa'],
args['options']['threads'])
if 'gnb' in args['classifiers']:
with timer("Training Gaussian Naive Bayes"):
train.naiveBayesClassifier(args['working'], args['working'])
with timer("Testing Gaussian Naive Bayes"):
clf = os.path.join(args['working'], 'classifiers',
'gnb.joblib')
test.test_classifier(args['working'], args['working'],
args['test_data'], clf, args['darpa'],
args['options']['threads'])
def main():
## Parameters ##
data = cmn.dataset(xSet = "dist_days_time_dayOfWeek")
data.load()#N_points = 1000)
futureMask = makeFutureMask(data.tScope, data.tTest)
# Try many values of k
vals = np.ceil(2 ** (np.arange(15) / 1.5))
rmse = np.zeros(shape=(len(vals)), dtype=np.float)
for i in range(len(vals)):
k = vals[i]
timer = cmn.timer()
yHat = manyNearestNeighborsVector(data.xScope, data.yScope, data.xTest, k, futureMask)
print "k = {}\tRuntime = {:.2f}".format(k, timer.dur())
rmse[i] = cmn.rmse(data.yTest, yHat)
print "\tRMSE = {:.2f}".format(rmse[i])
data.saveYHat(yHat, model = "{}NN".format(k))
# Visualize and save the images for the model
data.visualize(yHat, "{}NN".format(k))
# Plot the historical RMSE
clf()
plot(vals, rmse)
xlabel("Number of nearest points, k in kNN")
ylabel("Root Mean Squared Error (seconds)")
title("kNN Model, RMSE for different ks")
savefig("{}/{}_{}_k-rmse.png".format(data.figPath, data.serviceName, data.routeName))
def test_generate_WKB(ctx_factory, grid_shape, proc_shape, dtype, random,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
L = (10,)*3
volume = np.product(L)
dk = tuple(2 * np.pi / Li for Li in L)
modes = ps.RayleighGenerator(ctx, fft, dk, volume)
# only checking that this call is successful
fk, dfk = modes.generate_WKB(queue, random=random)
if timing:
ntime = 10
from common import timer
t = timer(lambda: modes.generate_WKB(queue, random=random), ntime=ntime)
print(f"{random=} set_modes took {t:.3f} ms for {grid_shape=}")
def main():
## Parameters ##
data = cmn.dataset(xSet = "dist_days_time_dayOfWeek")
data.load(Nparts = 10)#N_points = 1000)
futureMask = makeFutureMask(data.tScope, data.tTest)
# Try many values of k
vals = np.ceil(2 ** np.arange(10))
rmse = np.zeros(shape=(len(vals)), dtype=np.float)
for i in range(len(vals)):
k = vals[i]
timer = cmn.timer()
yHat = regress(data.xScope, data.yScope, data.xTest, k, futureMask)
print "rho = {}\tRuntime = {:.2f}".format(k, timer.dur())
rmse[i] = cmn.rmse(data.yTest, yHat)
print "\tRMSE = {:.2f}".format(rmse[i])
data.saveYHat(yHat, model = "kernel_{}rho".format(k))
# Visualize and save the images for the model
data.visualize(yHat, "kernel_{}rho".format(k))
# Plot the historical RMSE
clf()
plot(vals, rmse)
xlabel("rho Paramater")
ylabel("Root Mean Squared Error (seconds)")
title("Kernel Regression Model, RMSE for different rhos")
savefig("{}/{}_{}_kernel_rho-rmse.png".format(data.figPath, data.serviceName, data.routeName))
def test_reduction(ctx_factory,
grid_shape,
proc_shape,
dtype,
op,
_grid_shape,
pass_grid_dims,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
from pymbolic import var
from pystella import Field
tmp_insns = [(var("x"), Field("f") / 2 + .31)]
reducers = {}
reducers["avg"] = [(var("x"), op)]
if pass_grid_dims:
reducer = ps.Reduction(mpi,
reducers,
rank_shape=rank_shape,
tmp_instructions=tmp_insns,
grid_size=np.product(grid_shape))
else:
reducer = ps.Reduction(mpi, reducers, tmp_instructions=tmp_insns)
f = clr.rand(queue, rank_shape, dtype=dtype)
import pyopencl.tools as clt
pool = clt.MemoryPool(clt.ImmediateAllocator(queue))
result = reducer(queue, f=f, allocator=pool)
avg = result["avg"]
avg_test = reducer.reduce_array(f / 2 + .31, op)
if op == "avg":
avg_test /= np.product(grid_shape)
rtol = 5e-14 if dtype == np.float64 else 1e-5
assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \
f"{op} reduction innaccurate for {grid_shape=}, {proc_shape=}"
if timing:
from common import timer
t = timer(lambda: reducer(queue, f=f, allocator=pool), ntime=1000)
if mpi.rank == 0:
print(
f"reduction took {t:.3f} ms for {grid_shape=}, {proc_shape=}")
bandwidth = f.nbytes / 1024**3 / t * 1000
print(f"Bandwidth = {bandwidth:.1f} GB/s")
def test_field_statistics(ctx_factory,
grid_shape,
proc_shape,
dtype,
_grid_shape,
pass_grid_dims,
timing=False):
ctx = ctx_factory()
queue = cl.CommandQueue(ctx)
h = 1
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
# make select parameters local for convenience
h = 2
f = clr.rand(queue, (2, 1) + tuple(ni + 2 * h for ni in rank_shape),
dtype=dtype)
if pass_grid_dims:
statistics = ps.FieldStatistics(mpi,
h,
rank_shape=rank_shape,
grid_size=np.product(grid_shape))
else:
statistics = ps.FieldStatistics(mpi, h)
import pyopencl.tools as clt
pool = clt.MemoryPool(clt.ImmediateAllocator(queue))
stats = statistics(f, allocator=pool)
avg = stats["mean"]
var = stats["variance"]
f_h = f.get()
rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h], axis=(-3, -2, -1))
avg_test = mpi.allreduce(rank_sum) / np.product(grid_shape)
rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h]**2, axis=(-3, -2, -1))
var_test = mpi.allreduce(rank_sum) / np.product(grid_shape) - avg_test**2
rtol = 5e-14 if dtype == np.float64 else 1e-5
assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \
f"average innaccurate for {grid_shape=}, {proc_shape=}"
assert np.allclose(var, var_test, rtol=rtol, atol=0), \
f"variance innaccurate for {grid_shape=}, {proc_shape=}"
if timing:
from common import timer
t = timer(lambda: statistics(f, allocator=pool))
if mpi.rank == 0:
print(
f"field stats took {t:.3f} ms "
f"for outer shape {f.shape[:-3]}, {grid_shape=}, {proc_shape=}"
)
def test_histogram(ctx_factory, grid_shape, proc_shape, dtype, num_bins,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")):
max_rtol, avg_rtol = 1e-10, 1e-11
else:
max_rtol, avg_rtol = 5e-4, 5e-5
from pymbolic import var
_fx = ps.Field("fx")
histograms = {
"count": (var("abs")(_fx) * num_bins, 1),
"squared": (var("abs")(_fx) * num_bins, _fx**2),
}
hist = ps.Histogrammer(mpi, histograms, num_bins, dtype, rank_shape=rank_shape)
rng = clr.ThreefryGenerator(ctx, seed=12321)
fx = rng.uniform(queue, rank_shape, dtype)
fx_h = fx.get()
result = hist(queue, fx=fx)
res = result["count"]
assert np.sum(res.astype("int64")) == np.product(grid_shape), \
f"Count histogram doesn't sum to grid_size ({np.sum(res)})"
bins = np.linspace(0, 1, num_bins+1).astype(dtype)
weights = np.ones_like(fx_h)
np_res = np.histogram(fx_h, bins=bins, weights=weights)[0]
np_res = mpi.allreduce(np_res)
max_err, avg_err = get_errs(res, np_res)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"Histogrammer inaccurate for grid_shape={grid_shape}" \
f": {max_err=}, {avg_err=}"
res = result["squared"]
np_res = np.histogram(fx_h, bins=bins, weights=fx_h**2)[0]
np_res = mpi.allreduce(np_res)
max_err, avg_err = get_errs(res, np_res)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"Histogrammer with weights inaccurate for grid_shape={grid_shape}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: hist(queue, fx=fx))
print(f"histogram took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_generate(ctx_factory, grid_shape, proc_shape, dtype, random, timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
num_bins = int(sum(Ni**2 for Ni in grid_shape)**.5 / 2 + .5) + 1
L = (10,)*3
volume = np.product(L)
dk = tuple(2 * np.pi / Li for Li in L)
spectra = ps.PowerSpectra(mpi, fft, dk, volume)
modes = ps.RayleighGenerator(ctx, fft, dk, volume, seed=5123)
kbins = min(dk) * np.arange(0, num_bins)
test_norm = 1 / 2 / np.pi**2 / np.product(grid_shape)**2
for exp in [-1, -2, -3]:
def power(k):
return k**exp
fk = modes.generate(queue, random=random, norm=1, field_ps=power)
spectrum = spectra.norm * spectra.bin_power(fk, queue=queue, k_power=3)[1:-1]
true_spectrum = test_norm * kbins[1:-1]**3 * power(kbins[1:-1])
err = np.abs(1 - spectrum / true_spectrum)
tol = .1 if num_bins < 64 else .3
assert (np.max(err[num_bins//3:-num_bins//3]) < tol
and np.average(err[1:]) < tol), \
f"init power spectrum incorrect for {random=}, k**{exp}"
if random:
fx = fft.idft(cla.to_device(queue, fk)).real
if isinstance(fx, cla.Array):
fx = fx.get()
grid_size = np.product(grid_shape)
avg = mpi.allreduce(np.sum(fx)) / grid_size
var = mpi.allreduce(np.sum(fx**2)) / grid_size - avg**2
skew = mpi.allreduce(np.sum(fx**3)) / grid_size - 3 * avg * var - avg**3
skew /= var**1.5
assert skew < tol, \
f"init power spectrum has large skewness for k**{exp}"
if timing:
ntime = 10
from common import timer
t = timer(lambda: modes.generate(queue, random=random), ntime=ntime)
print(f"{random=} set_modes took {t:.3f} ms for {grid_shape=}")
def features(args):
"""
Applys the trained Word2Vec model to translate
the token vectors into feature vectors.
Parameters
----------
args : dict
Configuration dict, typically loaded from YAML file
"""
with timer("Generating Feature Vectors"):
pf.finalize_feature_vectors(args['working'], args['working'],
args['darpa'])
def tokens(args):
"""
Generate token vectors from raw pcap files.
Parameters
----------
args : dict
Configuration dict, typically loaded from YAML file
"""
with timer("Generating Tokens and Dictionary"):
pp.main(args['train_data'],
args['working'],
num_threads=args['options']['threads'],
ngram=[args['hyperparameters']['ngram']],
vocab_size=args['hyperparameters']['vocab_size'])
def main():
## Parameters ##
data = cmn.dataset(xSet = "dist_days_time_dayOfWeek")
data.load(Nparts = 10)#N_points = 1000)
futureMask = makeFutureMask(data.tScope, data.tTest)
rmse = np.zeros(shape=(len(vals)), dtype=np.float)
timer = cmn.timer()
yHat = regress(data.xScope, data.yScope, data.xTest, 1, futureMask)
print "onTime\tRuntime = {:.2f}".format(timer.dur())
rmse[i] = cmn.rmse(data.yTest, yHat)
print "\tRMSE = {:.2f}".format(rmse[i])
data.saveYHat(yHat, model = "onTime".format(k))
# Visualize and save the images for the model
data.visualize(yHat, "onTime".format(k))
def embeddings(args):
"""
Train a Word2Vec model using Tensorflow with
the token vectors generated in the 'tokens'
step.
Parameters
----------
args : dict
Configuration dict, typically loaded from YAML file
"""
with timer("Training Word2Vec Model"):
if 'embeddings' not in args.keys() or args['embeddings'] == None:
te.create(args['working'], args['working'],
args['hyperparameters']['vocab_size'])
else:
te.update(args['working'], args['embeddings'], args['working'],
args['hyperparameters']['vocab_size'])
def main():
## Parameters ##
data = cmn.dataset(xSet = "dist_days_time_dayOfWeek")
data.load()#N_points = 1000)
k = 10
futureMask = makeFutureMask(data.tScope, data.tTest)
timer = cmn.timer()
yHat = manyNearestNeighborsVector(data.xScope, data.yScope, data.xTest, k, futureMask)
print "Vectorized Runtime = {:.2f}".format(timer.dur())
print "RMSE = {:.2f}".format(cmn.rmse(data.yTest, yHat))
data.saveYHat(yHat, model = "{}NN".format(k))
#timer.reset()
#yHat2 = manyNearestNeighbors(data.xTrain, data.yTrain, data.xTest, k)
#print "Iterative Runtime = {:.2f}".format(timer.dur())
#print "RMSE = {}".format(cmn.rmse(data.yTest, yHat))
# Visualize and save the images for the model
data.visualize(yHat, "{}NN".format(k))
def main():
## Parameters ##
#data = cmn.dataset(xSet = "traj")
#data.load()#N_points = 1000)
#futureMask = makeFutureMask(data.tScope, data.tTest)
dataPath = "/projects/onebusaway/BakerNiedMLProject/data/routefeatures"
resPath = "/projects/onebusaway/BakerNiedMLProject/data/modelPredictions"
figPath = "/projects/onebusaway/BakerNiedMLProject/figures/predictions"
serviceName = "intercitytransit"
routeName = "route13"
xSet = "traj"
ySet = "dev"
x = np.loadtxt("{}/{}_{}_{}.txt".format(dataPath, serviceName, routeName, xSet), dtype=np.float)
# Try many values of k
vals = np.ceil(2 ** (np.arange(15) / 1.5))
rmse = np.zeros(shape=(len(vals)), dtype=np.float)
minK = 0
minRMSE = 0
sel = np.random.permutation(range(len(x)));
split = len(x)/4;
xTrain = x[sel[:split*2]];
xVal = x[sel[split*2:3*split]];
xTest = x[sel[3*split:]];
yTest = np.zeros(len(xVal));
yHat = np.zeros(len(xVal));
data_norm = np.empty(shape = x.shape)
theMean = x[:,:].mean()
theStdDev = x[:,:].std()
data_norm = (x - theMean)/ theStdDev
xTrainNorm = data_norm[sel[:split*2]];
xValNorm = data_norm[sel[split*2:3*split]];
xTestNorm = data_norm[sel[3*split:]];
for i in range(len(vals)):
k = vals[i]
model = "{}NN".format(k);
timer = cmn.timer()
print xTrain.shape;
print xTest.shape;
print xVal.shape;
for j in range(len(xVal)):
v = len(xVal[0])-15
t = np.random.randint(10,v);
yTest[j] = xVal[j][t+10];
#print xTrain[:,:t].shape;
#print xTrain[:,t+10].shape;
#print xVal[j,:t].shape;
#print t;
yHat[i] = manyNearestNeighborsVector(xTrainNorm[:,:t], xTrain[:,t+10], xValNorm[j,:t].reshape(1,t), k, weights=np.ones(t))
print "k = {}\tRuntime = {:.2f}".format(k, timer.dur())
rmse[i] = cmn.rmse(yTest, yHat)
if i == 0 or rmse[i]<minRMSE:
minRMSE = rmse[i]
minK = vals[i]
print "\tRMSE = {:.2f}".format(rmse[i])
np.savetxt("{}/{}_{}_{}_{}_val.txt".format(resPath, serviceName, routeName, model, xSet), cmn.cmb(xVal, yTest, yHat))
k = minK
model = "{}NN".format(k);
yTest = np.zeros(len(xTest));
yHat = np.zeros(len(xTest));
for i in range(len(xTest)):
v = len(xVal[0])-15
t = np.random.randint(10,v);
yTest[i] = xTest[i][t+10];
yHat[i] = manyNearestNeighborsVector(xTrain[:,:t], xTrain[:,t+10], xTest[i,:t].reshape(1,t), k, weights=np.ones(t))
# Visualize and save the images for the model
#data.visualize(yHat, "{}NN".format(k))
np.savetxt("{}/{}_{}_{}_{}_test.txt".format(resPath, serviceName, routeName, model, xSet), cmn.cmb(xTest, yTest, yHat))
# Plot the historical RMSE
clf()
plot(vals, rmse)
xlabel("Number of nearest points, k in kNN")
ylabel("Root Mean Squared Error (seconds)")
title("kNN Model, RMSE for different ks")
savefig("{}/{}_{}_k-rmse.png".format(figPath, serviceName, routeName))
def test_elementwise(ctx_factory, grid_shape, proc_shape, dtype, timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
rank_shape = tuple(Ni // pi for Ni, pi in zip(grid_shape, proc_shape))
from pymbolic import var
a = var("a")
b = var("b")
from pystella.field import Field
x = Field("x")
y = Field("y")
z = Field("z")
tmp_dict = {a[0]: x + 2, a[1]: 2 + x * y, b: x + y / 2}
map_dict = {x: a[0] * y**2 * x + a[1] * b, z: z + a[1] * b}
single_insn = {x: y + z}
ew_map = ps.ElementWiseMap(map_dict, tmp_instructions=tmp_dict)
x = clr.rand(queue, rank_shape, dtype=dtype)
y = clr.rand(queue, rank_shape, dtype=dtype)
z = clr.rand(queue, rank_shape, dtype=dtype)
a0 = x + 2
a1 = 2 + x * y
b = x + y / 2
x_true = a0 * y**2 * x + a1 * b
z_true = z + a1 * b
ew_map(queue, x=x, y=y, z=z)
max_rtol = 5e-14 if dtype == np.float64 else 1e-5
avg_rtol = 5e-14 if dtype == np.float64 else 1e-5
max_err, avg_err = get_errs(x_true.get(), x.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"x innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(z_true.get(), z.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"z innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}"
# test success of single instruction
ew_map_single = ps.ElementWiseMap(single_insn)
ew_map_single(queue, x=x, y=y, z=z)
x_true = y + z
max_err, avg_err = get_errs(x_true.get(), x.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"x innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: ew_map(queue, x=x, y=y, z=z)[0])
print(
f"elementwise map took {t:.3f} ms for {grid_shape=}, {proc_shape=}"
)
bandwidth = 5 * x.nbytes / 1024**3 / t * 1000
print(f"Bandwidth = {bandwidth:.1f} GB/s")
agent = PongAgent(state_size, n_actions)
elif mode == "evo":
from evo import PongAgent, training_loop
agent = PongAgent(state_size, n_actions)
callbacks = [
TrainingLogger(config.training_log),
ModelSaver(agent, config.agent),
]
agent = training_loop(agent, env, config, callbacks)
return agent
# ----------------------------------------------------------------------
def main():
config.read_args()
initial_seed(config.seed)
display_versions()
agent = train_agent("dqn") # dqn, evo, pg
agent.save(config.agent)
print("All OK")
# ----------------------------------------------------------------------
if __name__ == "__main__":
with timer("Pong RL"):
main()
def check_outp(self):
log = self.path_for('log')
outp = self.path_for('outp')
dat = self.path_for('persist')
if not log:
raise Warning("check_outp: Job %s missing log file" % self.name)
if not outp:
raise Warning("check_outp: Job %s missing outp file" % self.name)
with common.timer('rawfine'):
raw_dat = Datum.from_tuples(rawfine.read2csv(log, dat, meta=False))
with common.timer('refine'):
csv_dat = []
with open(outp, 'r') as fh:
for lineno, line in enumerate(fh):
line = line.strip()
if not line:
continue
if line.startswith(COMMENT_LINE):
continue
mean, err, its = line.split(',')
datum = Datum(float(mean), float(err), int(its))
datum._line_no = lineno
fin = math.isfinite
if not (fin(datum.mean) and fin(datum.stderr)):
continue
csv_dat.append(datum)
# lcs = common.LCS(csv_dat, raw_dat)
with common.timer('lcs'):
lcs = common.LCSGreedyApprox(raw_dat, csv_dat)
lcs.invert()
com_dat = lcs.common()
diff = lcs.diff()[0]
if diff:
fix = "#mean,stderr,weight\n"
for d, k, x in diff:
try:
x, _ = x
except TypeError:
pass
fix += str(x) + "\n"
warn = "\n"
warn += "check_outp: Job %s:\n" % self.name
warn += " Inconsistent outp(%d, %s) log(%d, %s) outputs!\n" % (
len(csv_dat), EXTS['outp'], len(raw_dat), EXTS['log'])
overs = set()
notcsv = 0
warn2 = " in outp but not log (*):\n"
warn2 += " #indx(line): mean,stderr,n\n"
for d, k, x in diff:
if d < 0:
if k in self.overrides['outp']:
overs.add(k)
else:
ln = getattr(x, '_line_no', None)
ln = '????' if ln is None else '%4d' % ln
warn2 += " %4d(%s): %s\n" % (k, ln, x)
notcsv += 1
if notcsv:
warn2 += " * may indicate incorrect weighting...\n"
warn += warn2
excess = self.overrides['outp'].difference(overs)
if excess:
warn += " overriden unnecessarily: %r\n" % excess
notlog = 0
warn2 = " in log but not outp:\n"
for d, k, x in diff:
if d > 0:
warn2 += " %4d: %s (mean~%f)\n" % (k, x, 1 / x.mean)
notlog += 1
if notlog:
warn += warn2
warn += " A candidate fix has been written to %s" % EXTS[
'outp_fix']
if notcsv or notlog or excess:
with open(self.path_for('outp_fix', True), 'w') as f:
f.write(fix)
raise Warning(warn)
max_flux = data.groupby(['object_id',
'passband'])['flux'].max().reset_index()
max_flux = pd.merge(max_flux,
data[['object_id', 'passband', 'flux', 'mjd']],
on=['object_id', 'passband', 'flux'],
how='left')
max_flux.columns = [
'object_id', 'passband', 'max(flux)', 'time(max(flux))'
]
max_flux.drop_duplicates(subset=['object_id', 'passband'], inplace=True)
max_flux.reset_index(drop=True).to_feather(config.DATA_DIR +
'passband_meta.f')
if __name__ == "__main__":
with timer("Make directory"):
mkdir(config.DATA_DIR)
mkdir(config.DEBUG_CSV_SAVE_DIR)
mkdir(config.FEATURE_LOAD_DIR)
mkdir(config.FEATURE_SAVE_DIR)
mkdir(config.MODEL_DIR)
mkdir(config.SHARE_DIR)
mkdir(config.SUBMIT_DIR)
with timer("Convert metadata"):
concat_to_feather(config.DATA_DIR + "training_set_metadata.csv",
config.DATA_DIR + "test_set_metadata.csv",
config.DATA_DIR + "meta.f",
column_order=[
'object_id', 'ra', 'decl', 'gal_l', 'gal_b',
'ddf', 'hostgal_specz', 'hostgal_photoz',
def test_dft(ctx_factory,
grid_shape,
proc_shape,
dtype,
use_fftw,
timing=False):
if not use_fftw and np.product(proc_shape) > 1:
pytest.skip("Must use mpi4py-fft on more than one rank.")
ctx = ctx_factory()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
mpi0 = ps.DomainDecomposition(proc_shape, 0, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype, use_fftw=use_fftw)
grid_size = np.product(grid_shape)
rdtype = fft.rdtype
if fft.is_real:
np_dft = np.fft.rfftn
np_idft = np.fft.irfftn
else:
np_dft = np.fft.fftn
np_idft = np.fft.ifftn
rtol = 1e-11 if dtype in ("float64", "complex128") else 2e-3
rng = clr.ThreefryGenerator(ctx, seed=12321 * (mpi.rank + 1))
fx = rng.uniform(queue, rank_shape, rdtype) + 1e-2
if not fft.is_real:
fx = fx + 1j * rng.uniform(queue, rank_shape, rdtype)
fx = fx.get()
fk = fft.dft(fx)
if isinstance(fk, cla.Array):
fk = fk.get()
fk, _fk = fk.copy(), fk # hang on to one that fftw won't overwrite
fx2 = fft.idft(_fk)
if isinstance(fx2, cla.Array):
fx2 = fx2.get()
fx_glb = np.empty(shape=grid_shape, dtype=dtype)
for root in range(mpi.nranks):
mpi0.gather_array(queue, fx, fx_glb, root=root)
fk_glb_np = np.ascontiguousarray(np_dft(fx_glb))
fx2_glb_np = np.ascontiguousarray(np_idft(fk_glb_np))
if use_fftw:
fk_np = fk_glb_np[fft.fft.local_slice(True)]
fx2_np = fx2_glb_np[fft.fft.local_slice(False)]
else:
fk_np = fk_glb_np
fx2_np = fx2_glb_np
max_err, avg_err = get_errs(fx, fx2 / grid_size)
assert max_err < rtol, \
f"IDFT(DFT(f)) != f for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(fk_np, fk)
assert max_err < rtol, \
f"DFT disagrees with numpy for {grid_shape=}, {proc_shape=}:"\
f" {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(fx2_np, fx2 / grid_size)
assert max_err < rtol, \
f"IDFT disagrees with numpy for {grid_shape=}, {proc_shape=}:"\
f" {max_err=}, {avg_err=}"
fx_cl = cla.empty(queue, rank_shape, dtype)
pencil_shape = tuple(ni + 2 * h for ni in rank_shape)
fx_cl_halo = cla.empty(queue, pencil_shape, dtype)
fx_np = np.empty(rank_shape, dtype)
fx_np_halo = np.empty(pencil_shape, dtype)
fk_cl = cla.empty(queue, fft.shape(True), fft.fk.dtype)
fk_np = np.empty(fft.shape(True), fft.fk.dtype)
# FIXME: check that these actually produce the correct result
fx_types = {
"cl": fx_cl,
"cl halo": fx_cl_halo,
"np": fx_np,
"np halo": fx_np_halo,
"None": None
}
fk_types = {"cl": fk_cl, "np": fk_np, "None": None}
# run all of these to ensure no runtime errors even if no timing
ntime = 20 if timing else 1
from common import timer
if mpi.rank == 0:
print(f"N = {grid_shape}, ",
"complex" if np.dtype(dtype).kind == "c" else "real")
from itertools import product
for (a, input_), (b, output) in product(fx_types.items(),
fk_types.items()):
t = timer(lambda: fft.dft(input_, output), ntime=ntime)
if mpi.rank == 0:
print(f"dft({a}, {b}) took {t:.3f} ms")
for (a, input_), (b, output) in product(fk_types.items(),
fx_types.items()):
t = timer(lambda: fft.idft(input_, output), ntime=ntime)
if mpi.rank == 0:
print(f"idft({a}, {b}) took {t:.3f} ms")
def extract_features(meta: pd.DataFrame,
lc: pd.DataFrame,
source: str,
normalize: bool = False,
snr: int = 3,
zbounds: str = 'estimated',
skip: int = 0,
end: int = -1,
clip_bounds: bool = False,
t_bounds: bool = False,
columns: List[str] = None):
if normalize:
try:
for band in ['g', 'i', 'r', 'u', 'y', 'z']:
b = read_bandpass('lsst/total_{}.dat'.format(band),
wave_unit=u.nm,
trim_level=0.001,
name='lsst{}_n'.format(band),
normalize=True)
sncosmo.register(b, 'lsst{}_n'.format(band))
except:
raise
pass
with timer('dropping meta'):
meta = meta[meta.object_id.isin(lc.object_id)].reset_index(drop=True)
print('shape(meta): {}'.format(meta.shape))
if 'object_id' in meta:
meta.set_index('object_id', inplace=True)
if normalize:
passbands = [
'lsstu_n', 'lsstg_n', 'lsstr_n', 'lssti_n', 'lsstz_n', 'lssty_n'
]
else:
passbands = ['lsstu', 'lsstg', 'lsstr', 'lssti', 'lsstz', 'lssty']
with timer('prep'):
lc['band'] = lc['passband'].apply(lambda x: passbands[x])
lc['zpsys'] = 'ab'
lc['zp'] = 25.0
# create a model
model = Model(source=source)
params = model.param_names
if columns:
ret = pd.DataFrame(columns=columns)
else:
columns = ['chisq', 'ncall'] + [source + '_' + c for c in params] + [
source + '_' + c + '_err' for c in params
]
ret = pd.DataFrame(columns=columns)
prefix = source
if zbounds == 'fixed':
prefix += '_f_'
else:
prefix += '_p_'
prefix += 'sn{}_'.format(snr)
if normalize:
prefix += 'n_'
ret.columns = [prefix + c for c in ret.columns]
n_errors = 0
n_loop = len(meta)
if end > 0:
n_loop = end
for i in tqdm(range(skip, n_loop)):
object_id = meta.index[i]
try:
ret.loc[object_id] = fit_lc(model, meta, lc, object_id, zbounds,
clip_bounds, t_bounds, snr)
except:
n_errors += 1
if i == 30 and n_errors == 31:
print('All 30 first attempts were failed. stopped')
raise
print('total {} data processed. {} data was skipped'.format(
len(meta), n_errors))
ret.reset_index(inplace=True)
ret.rename(columns={'index': 'object_id'}, inplace=True)
return ret
def generate_index(args):
jobdir = args.jobdir
cachef = None
jobs = {}
cache = {}
if args.cache:
cachef = os.path.join(jobdir, args.cache)
try:
if not args.replan:
with open(cachef, 'r') as f:
cache = json.load(f)
except FileNotFoundError:
pass
with common.timer(1):
for root, _, files in os.walk(jobdir):
for file in files:
path = os.path.join(root, file)
rpath = pathlib.PurePath(path).relative_to(jobdir)
jobn = str(rpath.with_suffix(''))
if rpath.suffix in EXTS_rev:
jobs.setdefault(jobn, Job(jobdir, jobn))
jobs[jobn].add_data(rpath.suffix)
index = Index(args)
with common.timer(2):
warnings = False
prog_bar = False
for jobn, job in jobs.items():
if jobn in cache:
params = Parameters.from_dict(cache[jobn])
job.set_parameters(params)
else:
error('.', end='', flush=True)
prog_bar = True
with common.timer((3, jobn)):
try:
with common.timer((3, jobn, 'overrides')):
job.get_overrides()
with common.timer((3, jobn, 'outp')):
job.check_outp()
with common.timer((3, jobn, 'params')):
params = job.get_parameters()
cache[jobn] = params.to_dict()
except Warning as w:
error(w)
warnings = True
continue
index[params].add(job)
if prog_bar: error('')
if cachef:
with open(cachef, 'w') as f:
json.dump(cache, f, sort_keys=True, indent=4)
if warnings:
error("\n*** Please fix the above warnings before continuing")
error(
"* For inconsistent outp/log, manually edit relevant files to resolve inconsistencies or use override*"
)
error(
"* For missing outp files, use rawfine.py cautiously to reconstruct"
)
error(
"* For missing log files, first touch the log file, then override* the relevant warnings."
)
error(
"* For uninferred parameters, you may have an empty log file - add params to the log file..."
)
error("** .override files: see README.md")
exit(1)
return False
with common.timer(4):
warnings = False
for p, xs in index.all():
try:
xs.resolve()
except Warning as w:
error(w)
warnings = True
continue
if warnings:
error("\n*** Please fix the above warnings before continuing")
error("* Staged plans need manual review before use;")
error(" if the plan looks right, remove the staged key")
exit(1)
return False
return index
def main():
## Parameters ##
xSet = "dist_days_time_dayOfWeek_normalized"
# Load the Data
data = cmn.dataset(xSet = xSet)
data.load(Nparts = 10)#, N_points = 12000)
# Set X, Y, xTest and get sizes
Y = data.yScope;
(N_train, N_feat) = data.xScope.shape
N_test = data.xTest.shape[0]
# Precompute the future mask
futureMask = makeFutureMask(data.tScope, data.tTest, futureTime = -30 * 60)
# Compute the distance matrix (since they all use the same)
weights = np.array([3, 0.5, 2, 1])
timer = cmn.timer()
traintile = np.tile(np.reshape(data.xScope, (N_train, N_feat, 1)), (1, 1, N_test));
testtile = np.tile(np.transpose(np.reshape(data.xTest, (N_test, N_feat, 1)), (2, 1, 0)), (N_train, 1, 1));
weightstile = np.tile(weights.reshape(1, N_feat, 1), (N_train, 1, N_test))
print "Dist runtime: {:.2f}".format(timer.dur())
dist = np.sum(((traintile - testtile) ** 2) * weightstile, axis=1)
## Try out some models with rhos and ks
filename = "{}/{}_{}_tests_Ntrain{}.txt".format(data.resPath, data.serviceName, data.routeName, N_train)
f = open(filename, 'w')
str = "Model\t{}\t{}\t{}\n".format("Param", "RunTime", "RMSE")
print str[:len(str)-1]
f.write(str)
# Make a list of parameters to test
rhos = 2 ** (np.arange(-10, 10) / 2.0)
ks = np.ceil( 2 ** (np.arange(18) / 1.5))
# Allocate data containers
N_onTime = 1
N_kernel = len(rhos)
N_kNN = len(ks)
N_attempts = N_onTime + N_kernel + N_kNN
attempts = range(0, N_attempts)
rmses = np.zeros(shape=(N_attempts), dtype=np.float)
times = np.zeros(shape=(N_attempts), dtype=np.float)
models = [""] * N_attempts
for i in attempts:
timer = cmn.timer()
if(i < N_onTime):
yHat = onTime(dist.shape[1])
models[i] = "onTime\t"
elif(i < (N_kernel + N_onTime)):
rho = rhos[i - N_onTime]
yHat = kernel(dist, futureMask, Y, rho)
models[i] = "kernel\t{: 3.2f}".format(rho)
elif(i < (N_kNN + N_kernel + N_onTime)):
k = ks[i - N_onTime - N_kernel]
yHat = kNN(dist, futureMask, Y, k)
models[i] = "kNN\t{: 5.0f}".format(k)
times[i] = timer.dur()
rmses[i] = cmn.rmse(data.yTest, yHat)
str = "{}\t{:.2f}\t{:.2f}\n".format(models[i], times[i], rmses[i])
print str[:len(str)-1]
f.write(str)
f.close()
def test_share_halos(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
_grid_shape,
pass_grid_shape,
timing=False):
ctx = ctx_factory()
if isinstance(h, int):
h = (h, ) * 3
queue = cl.CommandQueue(ctx)
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(
proc_shape, h, grid_shape=(grid_shape if pass_grid_shape else None))
rank_shape, substart = mpi.get_rank_shape_start(grid_shape)
# data will be same on each rank
rng = clr.ThreefryGenerator(ctx, seed=12321)
data = rng.uniform(queue,
tuple(Ni + 2 * hi for Ni, hi in zip(grid_shape, h)),
dtype).get()
if h[0] > 0:
data[:h[0], :, :] = data[-2 * h[0]:-h[0], :, :]
data[-h[0]:, :, :] = data[h[0]:2 * h[0], :, :]
if h[1] > 0:
data[:, :h[1], :] = data[:, -2 * h[1]:-h[1], :]
data[:, -h[1]:, :] = data[:, h[1]:2 * h[1], :]
if h[2] > 0:
data[:, :, :h[2]] = data[:, :, -2 * h[2]:-h[2]]
data[:, :, -h[2]:] = data[:, :, h[2]:2 * h[2]]
subdata = np.empty(tuple(ni + 2 * hi for ni, hi in zip(rank_shape, h)),
dtype)
rank_slice = tuple(
slice(si + hi, si + ni + hi)
for ni, si, hi in zip(rank_shape, substart, h))
unpadded_slc = tuple(slice(hi, -hi) if hi > 0 else slice(None) for hi in h)
subdata[unpadded_slc] = data[rank_slice]
subdata_device = cla.to_device(queue, subdata)
mpi.share_halos(queue, subdata_device)
subdata2 = subdata_device.get()
pencil_slice = tuple(
slice(si, si + ri + 2 * hi)
for ri, si, hi in zip(rank_shape, substart, h))
assert (subdata2 == data[pencil_slice]).all(), \
f"rank {mpi.rank} {mpi.rank_tuple} has incorrect halo data"
# test that can call with different-shaped input
if not pass_grid_shape:
subdata_device_new = clr.rand(
queue, tuple(ni // 2 + 2 * hi for ni, hi in zip(rank_shape, h)),
dtype)
mpi.share_halos(queue, subdata_device_new)
if timing:
from common import timer
t = timer(lambda: mpi.share_halos(queue, fx=subdata_device))
if mpi.rank == 0:
print(f"share_halos took {t:.3f} ms for "
f"{grid_shape=}, {h=}, {proc_shape=}")
def test_field_histogram(ctx_factory, grid_shape, proc_shape, dtype, timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
pencil_shape = tuple(Ni + 2 * h for Ni in rank_shape)
num_bins = 432
if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")):
max_rtol, avg_rtol = 1e-10, 1e-11
else:
max_rtol, avg_rtol = 5e-4, 5e-5
hist = ps.FieldHistogrammer(mpi, num_bins, dtype,
rank_shape=rank_shape, halo_shape=h)
rng = clr.ThreefryGenerator(ctx, seed=12321)
fx = rng.uniform(queue, (2, 2)+pencil_shape, dtype, a=-1.2, b=3.)
fx_h = fx.get()[..., h:-h, h:-h, h:-h]
result = hist(fx)
outer_shape = fx.shape[:-3]
from itertools import product
slices = list(product(*[range(n) for n in outer_shape]))
for slc in slices:
res = result["linear"][slc]
np_res = np.histogram(fx_h[slc], bins=result["linear_bins"][slc])[0]
np_res = mpi.allreduce(np_res)
max_err, avg_err = get_errs(res, np_res)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"linear Histogrammer inaccurate for grid_shape={grid_shape}" \
f": {max_err=}, {avg_err=}"
res = result["log"][slc]
bins = result["log_bins"][slc]
# avoid FPA comparison issues
# numpy sometimes doesn't count the actual maximum/minimum
eps = 1e-14 if np.dtype(dtype) == np.dtype("float64") else 1e-4
bins[0] *= (1 - eps)
bins[-1] *= (1 + eps)
np_res = np.histogram(np.abs(fx_h[slc]), bins=bins)[0]
np_res = mpi.allreduce(np_res)
norm = np.maximum(np.abs(res), np.abs(np_res))
norm[norm == 0.] = 1.
max_err, avg_err = get_errs(res, np_res)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"log Histogrammer inaccurate for grid_shape={grid_shape}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: hist(fx[0, 0]))
print(f"field histogram took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_spectral_poisson(ctx_factory, grid_shape, proc_shape, h, dtype,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
L = (3, 5, 7)
dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape))
dk = tuple(2 * np.pi / Li for Li in L)
if h == 0:
def get_evals_2(k, dx):
return - k**2
derivs = ps.SpectralCollocator(fft, dk)
else:
from pystella.derivs import SecondCenteredDifference
get_evals_2 = SecondCenteredDifference(h).get_eigenvalues
derivs = ps.FiniteDifferencer(mpi, h, dx, stream=False)
solver = ps.SpectralPoissonSolver(fft, dk, dx, get_evals_2)
pencil_shape = tuple(ni + 2*h for ni in rank_shape)
statistics = ps.FieldStatistics(mpi, 0, rank_shape=rank_shape,
grid_size=np.product(grid_shape))
fx = cla.empty(queue, pencil_shape, dtype)
rho = clr.rand(queue, rank_shape, dtype)
rho -= statistics(rho)["mean"]
lap = cla.empty(queue, rank_shape, dtype)
rho_h = rho.get()
for m_squared in (0, 1.2, 19.2):
solver(queue, fx, rho, m_squared=m_squared)
fx_h = fx.get()
if h > 0:
fx_h = fx_h[h:-h, h:-h, h:-h]
derivs(queue, fx=fx, lap=lap)
diff = np.fabs(lap.get() - rho_h - m_squared * fx_h)
max_err = np.max(diff) / cla.max(clm.fabs(rho))
avg_err = np.sum(diff) / cla.sum(clm.fabs(rho))
max_rtol = 1e-12 if dtype == np.float64 else 1e-4
avg_rtol = 1e-13 if dtype == np.float64 else 1e-5
assert max_err < max_rtol and avg_err < avg_rtol, \
f"solution inaccurate for {h=}, {grid_shape=}, {proc_shape=}"
if timing:
from common import timer
time = timer(lambda: solver(queue, fx, rho, m_squared=m_squared), ntime=10)
if mpi.rank == 0:
print(f"poisson took {time:.3f} ms for {grid_shape=}, {proc_shape=}")
def test_spectra(ctx_factory, grid_shape, proc_shape, dtype, L, timing=False):
ctx = ctx_factory()
queue = cl.CommandQueue(ctx)
h = 1
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
L = L or (3, 5, 7)
dk = tuple(2 * np.pi / Li for Li in L)
cdtype = fft.cdtype
spec = ps.PowerSpectra(mpi,
fft,
dk,
np.product(L),
bin_width=min(dk) + .001)
# FIXME: bin_width=min(dk) sometimes disagrees to O(.1%) with numpy...
assert int(np.sum(spec.bin_counts)) == np.product(grid_shape), \
"bin counts don't sum to total number of points/modes"
k_power = 2.
fk = make_data(*fft.shape(True)).astype(cdtype)
fk_d = cla.to_device(queue, fk)
spectrum = spec.bin_power(fk_d, k_power=k_power)
bins = np.arange(-.5, spec.num_bins + .5) * spec.bin_width
sub_k = list(x.get() for x in fft.sub_k.values())
kvecs = np.meshgrid(*sub_k, indexing="ij", sparse=False)
kmags = np.sqrt(sum((dki * ki)**2 for dki, ki in zip(dk, kvecs)))
if fft.is_real:
counts = 2. * np.ones_like(kmags)
counts[kvecs[2] == 0] = 1
counts[kvecs[2] == grid_shape[-1] // 2] = 1
else:
counts = 1. * np.ones_like(kmags)
if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")):
max_rtol = 1e-8
avg_rtol = 1e-11
else:
max_rtol = 2e-2
avg_rtol = 2e-4
bin_counts2 = spec.bin_power(np.ones_like(fk), queue=queue, k_power=0)
max_err, avg_err = get_errs(bin_counts2, np.ones_like(bin_counts2))
assert max_err < max_rtol and avg_err < avg_rtol, \
f"bin counting disagrees between PowerSpectra and np.histogram" \
f" for {grid_shape=}: {max_err=}, {avg_err=}"
hist = np.histogram(kmags,
bins=bins,
weights=np.abs(fk)**2 * counts * kmags**k_power)[0]
hist = mpi.allreduce(hist) / spec.bin_counts
# skip the Nyquist mode and the zero mode
max_err, avg_err = get_errs(spectrum[1:-2], hist[1:-2])
assert max_err < max_rtol and avg_err < avg_rtol, \
f"power spectrum inaccurate for {grid_shape=}: {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: spec.bin_power(fk_d, k_power=k_power))
print(f"power spectrum took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_tensor_projector(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
L = (10, 8, 11.5)
dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape))
dk = tuple(2 * np.pi / Li for Li in L)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
cdtype = fft.cdtype
if h > 0:
stencil = FirstCenteredDifference(h)
project = ps.Projector(fft, stencil.get_eigenvalues, dk, dx)
derivs = ps.FiniteDifferencer(mpi, h, dx)
else:
project = ps.Projector(fft, lambda k, dx: k, dk, dx)
derivs = ps.SpectralCollocator(fft, dk)
vector_x = cla.empty(queue, (3, ) + tuple(ni + 2 * h for ni in rank_shape),
dtype)
div = cla.empty(queue, rank_shape, dtype)
pdx = cla.empty(queue, (3, ) + rank_shape, dtype)
def get_divergence_errors(hij):
max_errors = []
avg_errors = []
for i in range(1, 4):
for mu in range(3):
fft.idft(hij[tensor_id(i, mu + 1)], vector_x[mu])
derivs.divergence(queue, vector_x, div)
derivs(queue, fx=vector_x[0], pdx=pdx[0])
derivs(queue, fx=vector_x[1], pdy=pdx[1])
derivs(queue, fx=vector_x[2], pdz=pdx[2])
norm = sum([clm.fabs(pdx[mu]) for mu in range(3)])
max_errors.append(cla.max(clm.fabs(div)) / cla.max(norm))
avg_errors.append(cla.sum(clm.fabs(div)) / cla.sum(norm))
return np.array(max_errors), np.array(avg_errors)
max_rtol = 1e-11 if dtype == np.float64 else 1e-4
avg_rtol = 1e-13 if dtype == np.float64 else 1e-5
def get_trace_errors(hij_h):
trace = sum([hij_h[tensor_id(i, i)] for i in range(1, 4)])
norm = np.sqrt(
sum(np.abs(hij_h[tensor_id(i, i)])**2 for i in range(1, 4)))
trace = np.abs(trace[norm != 0]) / norm[norm != 0]
trace = trace[trace < .9]
return np.max(trace), np.sum(trace) / trace.size
k_shape = fft.shape(True)
hij = cla.empty(queue, shape=(6, ) + k_shape, dtype=cdtype)
for mu in range(6):
hij[mu] = make_data(queue, fft).astype(cdtype)
project.transverse_traceless(queue, hij)
hij_h = hij.get()
if isinstance(fft, gDFT):
assert all(is_hermitian(hij_h[i]) for i in range(6)), \
f"TT projection is non-hermitian for {grid_shape=}, {h=}"
max_err, avg_err = get_divergence_errors(hij)
assert all(max_err < max_rtol) and all(avg_err < avg_rtol), \
f"TT projection not transverse for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
max_err, avg_err = get_trace_errors(hij_h)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"TT projected tensor isn't traceless for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
plus = make_data(queue, fft).astype(cdtype)
minus = make_data(queue, fft).astype(cdtype)
project.pol_to_tensor(queue, plus, minus, hij)
if isinstance(fft, gDFT):
assert all(is_hermitian(hij[i]) for i in range(6)), \
f"pol->tensor is non-hermitian for {grid_shape=}, {h=}"
max_err, avg_err = get_divergence_errors(hij)
assert all(max_err < max_rtol) and all(avg_err < avg_rtol), \
f"pol->tensor not transverse for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
hij_h = hij.get()
max_err, avg_err = get_trace_errors(hij_h)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->tensor isn't traceless for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
hij_2 = cla.zeros_like(hij)
project.transverse_traceless(queue, hij, hij_2)
hij_h_2 = hij_2.get()
max_err, avg_err = get_errs(hij_h, hij_h_2)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->tensor != its own TT projection for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
plus1 = cla.zeros_like(plus)
minus1 = cla.zeros_like(minus)
project.tensor_to_pol(queue, plus1, minus1, hij)
if isinstance(fft, gDFT):
assert is_hermitian(plus1) and is_hermitian(minus1), \
f"polarizations aren't hermitian for {grid_shape=}, {h=}"
max_err, avg_err = get_errs(plus1.get(), plus.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->tensor->pol (plus) is not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(minus1.get(), minus.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->tensor->pol (minus) is not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
project.tensor_to_pol(queue, hij[0], hij[1], hij)
max_err, avg_err = get_errs(plus1.get(), hij[0].get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"in-place pol->tensor->pol (plus) not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(minus1.get(), hij[1].get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"in-place pol->tensor->pol (minus) not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
ntime = 10
t = timer(lambda: project.transverse_traceless(queue, hij),
ntime=ntime)
print(f"TT projection took {t:.3f} ms for {grid_shape=}")
t = timer(lambda: project.pol_to_tensor(queue, plus, minus, hij),
ntime=ntime)
print(f"pol->tensor took {t:.3f} ms for {grid_shape=}")
t = timer(lambda: project.tensor_to_pol(queue, plus, minus, hij),
ntime=ntime)
print(f"tensor->pol took {t:.3f} ms for {grid_shape=}")
def test_scalar_energy(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
grid_size = np.product(grid_shape)
nscalars = 2
def potential(f):
phi, chi = f[0], f[1]
return 1 / 2 * phi**2 + 1 / 2 * chi**2 + 1 / 2 * phi**2 * chi**2
scalar_sector = ps.ScalarSector(nscalars, potential=potential)
scalar_energy = ps.Reduction(mpi,
scalar_sector,
rank_shape=rank_shape,
grid_size=grid_size,
halo_shape=h)
pencil_shape = tuple(ni + 2 * h for ni in rank_shape)
f = clr.rand(queue, (nscalars, ) + pencil_shape, dtype)
dfdt = clr.rand(queue, (nscalars, ) + pencil_shape, dtype)
lap = clr.rand(queue, (nscalars, ) + rank_shape, dtype)
energy = scalar_energy(queue, f=f, dfdt=dfdt, lap_f=lap, a=np.array(1.))
kin_test = []
grad_test = []
for fld in range(nscalars):
df_h = dfdt[fld].get()
rank_sum = np.sum(df_h[h:-h, h:-h, h:-h]**2)
kin_test.append(1 / 2 * mpi.allreduce(rank_sum) / grid_size)
f_h = f[fld].get()
lap_h = lap[fld].get()
rank_sum = np.sum(-f_h[h:-h, h:-h, h:-h] * lap_h)
grad_test.append(1 / 2 * mpi.allreduce(rank_sum) / grid_size)
energy_test = {}
energy_test["kinetic"] = np.array(kin_test)
energy_test["gradient"] = np.array(grad_test)
phi = f[0].get()[h:-h, h:-h, h:-h]
chi = f[1].get()[h:-h, h:-h, h:-h]
pot_rank = np.sum(potential([phi, chi]))
energy_test["potential"] = np.array(mpi.allreduce(pot_rank) / grid_size)
max_rtol = 1e-14 if dtype == np.float64 else 1e-5
avg_rtol = 1e-14 if dtype == np.float64 else 1e-5
for key, value in energy.items():
max_err, avg_err = get_errs(value, energy_test[key])
assert max_err < max_rtol and avg_err < avg_rtol, \
f"{key} inaccurate for {nscalars=}, {grid_shape=}, {proc_shape=}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: scalar_energy(
queue, a=np.array(1.), f=f, dfdt=dfdt, lap_f=lap))
if mpi.rank == 0:
print(f"scalar energy took {t:.3f} "
f"ms for {nscalars=}, {grid_shape=}, {proc_shape=}")
import pandas as pd
import numpy as np
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion, Pipeline, make_union, make_pipeline
from common import timer, read_csv, ItemSelector, TextStats
with timer("Load data"):
df_protein_train = read_csv("df_protein_train.csv")
df_protein_test = read_csv("df_protein_test.csv")
df_protein = pd.concat([df_protein_train, df_protein_test])
df_protein.Sequence = df_protein.Sequence.apply(lambda x: x.upper())
feature_union = make_union(
make_pipeline(ItemSelector(key="Sequence"),
CountVectorizer(analyzer='char', ngram_range=(1, 1))),
make_pipeline(
ItemSelector(key="Sequence"),
TfidfVectorizer(analyzer='char', ngram_range=(1, 1), use_idf=False)),
make_pipeline(ItemSelector(key="Sequence"), TextStats(), DictVectorizer()))
with timer("Fit feature_union"):
feat = feature_union.fit_transform(df_protein)
out_col = [f'protein_stat_{i}' for i in range(feat.shape[1])]
output_file = "./input/temp/df_protein_stat.csv"
with timer(f"Save file to {output_file}"):
def test_gather_scatter(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
_grid_shape,
pass_grid_shape,
timing=False):
ctx = ctx_factory()
if isinstance(h, int):
h = (h, ) * 3
queue = cl.CommandQueue(ctx)
grid_shape = _grid_shape or grid_shape
mpi = ps.DomainDecomposition(proc_shape, h)
rank_shape, substart = mpi.get_rank_shape_start(grid_shape)
rank_slice = tuple(
slice(si, si + ri) for ri, si, hi in zip(rank_shape, substart, h))
pencil_shape = tuple(ni + 2 * hi for ni, hi in zip(rank_shape, h))
unpadded_slc = tuple(slice(hi, -hi) if hi > 0 else slice(None) for hi in h)
# create random data with same seed on all ranks
rng = clr.ThreefryGenerator(ctx, seed=12321)
data = rng.uniform(queue, grid_shape, dtype)
# cl.Array -> cl.Array
subdata = cla.zeros(queue, pencil_shape, dtype)
mpi.scatter_array(queue, data if mpi.rank == 0 else None, subdata, 0)
sub_h = subdata.get()
data_h = data.get()
assert (sub_h[unpadded_slc] == data_h[rank_slice]).all()
data_test = cla.zeros_like(data)
mpi.gather_array(queue, subdata, data_test if mpi.rank == 0 else None, 0)
data_test_h = data_test.get()
if mpi.rank == 0:
assert (data_test_h == data_h).all()
# np.ndarray -> np.ndarray
mpi.scatter_array(queue, data_h if mpi.rank == 0 else None, sub_h, 0)
assert (sub_h[unpadded_slc] == data_h[rank_slice]).all()
mpi.gather_array(queue, sub_h, data_test_h if mpi.rank == 0 else None, 0)
if mpi.rank == 0:
assert (data_test_h == data_h).all()
# scatter cl.Array -> np.ndarray
sub_h[:] = 0
mpi.scatter_array(queue, data if mpi.rank == 0 else None, sub_h, 0)
assert (sub_h[unpadded_slc] == data_h[rank_slice]).all()
# gather np.ndarray -> cl.Array
data_test[:] = 0
mpi.gather_array(queue, sub_h, data_test if mpi.rank == 0 else None, 0)
data_test_h = data_test.get()
if mpi.rank == 0:
assert (data_test_h == data_h).all()
# scatter np.ndarray -> cl.Array
subdata[:] = 0
mpi.scatter_array(queue, data_h if mpi.rank == 0 else None, subdata, 0)
sub_h = subdata.get()
assert (sub_h[unpadded_slc] == data_h[rank_slice]).all()
# gather cl.Array -> np.ndarray
data_test_h[:] = 0
mpi.gather_array(queue, subdata, data_test_h if mpi.rank == 0 else None, 0)
if mpi.rank == 0:
assert (data_test_h == data_h).all()
if timing:
from common import timer
ntime = 25
times = {}
times["scatter cl.Array -> cl.Array"] = \
timer(lambda: mpi.scatter_array(queue, data, subdata, 0), ntime=ntime)
times["scatter cl.Array -> np.ndarray"] = \
timer(lambda: mpi.scatter_array(queue, data, sub_h, 0), ntime=ntime)
times["scatter np.ndarray -> cl.Array"] = \
timer(lambda: mpi.scatter_array(queue, data_h, subdata, 0), ntime=ntime)
times["scatter np.ndarray -> np.ndarray"] = \
timer(lambda: mpi.scatter_array(queue, data_h, sub_h, 0), ntime=ntime)
times["gather cl.Array -> cl.Array"] = \
timer(lambda: mpi.gather_array(queue, subdata, data, 0), ntime=ntime)
times["gather cl.Array -> np.ndarray"] = \
timer(lambda: mpi.gather_array(queue, subdata, data_h, 0), ntime=ntime)
times["gather np.ndarray -> cl.Array"] = \
timer(lambda: mpi.gather_array(queue, sub_h, data, 0), ntime=ntime)
times["gather np.ndarray -> np.ndarray"] = \
timer(lambda: mpi.gather_array(queue, sub_h, data_h, 0), ntime=ntime)
if mpi.rank == 0:
print(f"{grid_shape=}, {h=}, {proc_shape=}")
for key, val in times.items():
print(f"{key} took {val:.3f} ms")
import pandas as pd
import numpy as np
from common import timer, read_csv
fp_col = [f'fp_{i}' for i in range(167)]
out_col = ["molecule_count"] + fp_col
output_file = "./input/temp/df_molecule_stat.csv"
with timer("Load data"):
df_molecule = read_csv("df_molecule.csv")
df_aff_train = read_csv("df_affinity_train.csv")
df_aff_test = read_csv("df_affinity_test_toBePredicted.csv")
df_aff = pd.concat([df_aff_train, df_aff_test])
with timer("Make molecule count feature"):
df_molecule_count = df_aff.groupby("Molecule_ID", as_index=False).Ki.agg(
{"molecule_count": "count"})
df_molecule = df_molecule.merge(df_molecule_count, on=["Molecule_ID"])
with timer("Parse fingerprint"):
fingerprint = df_molecule.Fingerprint.apply(
lambda x: np.array(x.split(', '))).values
fingerprint = np.vstack(fingerprint).astype(np.uint8)
df_fingerprint = pd.DataFrame(fingerprint, columns=fp_col, dtype=np.uint8)
df_molecule = pd.concat([df_molecule, df_fingerprint], axis=1)
del df_fingerprint, fingerprint
with timer(f"Save file to {output_file}"):
df_molecule[['Molecule_ID'] + out_col].to_csv(output_file, index=False)
warnings.filterwarnings('ignore')
import pandas as pd
import numpy as np
import gc
from sklearn.linear_model import Ridge
from sklearn.model_selection import KFold
from common import timer, read_csv, ItemSelector
fp_col = [f'fp_{i}' for i in range(167)]
w2v_col = [f'w2v_{i}' for i in range(128)]
with timer("Load data"):
df_affinity_train = read_csv("df_affinity_train.csv")
df_molecule = read_csv("df_molecule_stat.csv", "./input/temp/")
df_protein_w2v = read_csv("df_w2v_ws3.csv", "./input/temp/")
df_affinity_test = read_csv("df_affinity_test_toBePredicted.csv")
df_train = df_affinity_train.merge(df_molecule)
df_train = df_train.merge(df_protein_w2v)
df_test = df_affinity_test.merge(df_molecule)
df_test = df_test.merge(df_protein_w2v)
test = df_test
all_protein_id = df_train.Protein_ID.unique()
kfold = KFold(n_splits=5, shuffle=True, random_state=2018)
def test_gradient_laplacian(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
stream,
timing=False):
if h == 0 and stream is True:
pytest.skip("no streaming spectral")
ctx = ctx_factory()
queue = cl.CommandQueue(ctx)
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, start = mpi.get_rank_shape_start(grid_shape)
L = (3, 5, 7)
dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape))
dk = tuple(2 * np.pi / Li for Li in L)
if h == 0:
def get_evals_1(k, dx):
return k
def get_evals_2(k, dx):
return -k**2
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
derivs = ps.SpectralCollocator(fft, dk)
else:
from pystella.derivs import FirstCenteredDifference, SecondCenteredDifference
get_evals_1 = FirstCenteredDifference(h).get_eigenvalues
get_evals_2 = SecondCenteredDifference(h).get_eigenvalues
if stream:
try:
derivs = ps.FiniteDifferencer(mpi,
h,
dx,
rank_shape=rank_shape,
stream=stream)
except: # noqa
pytest.skip("StreamingStencil unavailable")
else:
derivs = ps.FiniteDifferencer(mpi, h, dx, rank_shape=rank_shape)
pencil_shape = tuple(ni + 2 * h for ni in rank_shape)
# set up test data
fx_h = np.empty(pencil_shape, dtype)
kvec = np.array(dk) * np.array([-5, 4, -3]).astype(dtype)
xvec = np.meshgrid(*[
dxi * np.arange(si, si + ni)
for dxi, si, ni in zip(dx, start, rank_shape)
],
indexing="ij")
phases = sum(ki * xi for ki, xi in zip(kvec, xvec))
if h > 0:
fx_h[h:-h, h:-h, h:-h] = np.sin(phases)
else:
fx_h[:] = np.sin(phases)
fx_cos = np.cos(phases)
fx = cla.to_device(queue, fx_h)
lap = cla.empty(queue, rank_shape, dtype)
grd = cla.empty(queue, (3, ) + rank_shape, dtype)
derivs(queue, fx=fx, lap=lap, grd=grd)
eff_kmag_sq = sum(
get_evals_2(kvec_i, dxi) for dxi, kvec_i in zip(dx, kvec))
lap_true = eff_kmag_sq * np.sin(phases)
max_rtol = 1e-9 if dtype == np.float64 else 3e-4
avg_rtol = 1e-11 if dtype == np.float64 else 5e-5
# filter small values dominated by round-off error
mask = np.abs(lap_true) > 1e-11
max_err, avg_err = get_errs(lap_true[mask], lap.get()[mask])
assert max_err < max_rtol and avg_err < avg_rtol, \
f"lap inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \
f" {max_err=}, {avg_err=}"
for i in range(3):
eff_k = get_evals_1(kvec[i], dx[i])
pdi_true = eff_k * fx_cos
# filter small values dominated by round-off error
mask = np.abs(pdi_true) > 1e-11
max_err, avg_err = get_errs(pdi_true[mask], grd[i].get()[mask])
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pd{i} inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \
f" {max_err=}, {avg_err=}"
vec = cla.empty(queue, (3, ) + pencil_shape, dtype)
for mu in range(3):
vec[mu] = fx
div = cla.empty(queue, rank_shape, dtype)
derivs.divergence(queue, vec, div)
div_true = sum(grd[i] for i in range(3)).get()
# filter small values dominated by round-off error
mask = np.abs(div_true) > 1e-11
max_err, avg_err = get_errs(div_true[mask], div.get()[mask])
assert max_err < max_rtol and avg_err < avg_rtol, \
f"div inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \
f" {max_err=}, {avg_err=}"
if timing:
from common import timer
base_args = dict(queue=queue, fx=fx)
div_args = dict(queue=queue, vec=vec, div=div)
if h == 0:
import pyopencl.tools as clt
pool = clt.MemoryPool(clt.ImmediateAllocator(queue))
base_args["allocator"] = pool
div_args["allocator"] = pool
times = {}
times["gradient and laplacian"] = timer(
lambda: derivs(lap=lap, grd=grd, **base_args))
times["gradient"] = timer(lambda: derivs(grd=grd, **base_args))
times["laplacian"] = timer(lambda: derivs(lap=lap, **base_args))
times["pdx"] = timer(lambda: derivs(pdx=grd[0], **base_args))
times["pdy"] = timer(lambda: derivs(pdy=grd[1], **base_args))
times["pdz"] = timer(lambda: derivs(pdz=grd[2], **base_args))
times["divergence"] = timer(lambda: derivs.divergence(**div_args))
if mpi.rank == 0:
print(f"{grid_shape=}, {h=}, {proc_shape=}")
for key, val in times.items():
print(f"{key} took {val:.3f} ms")
def test_stencil(ctx_factory,
grid_shape,
proc_shape,
dtype,
stream,
h=1,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
rank_shape = tuple(Ni // pi for Ni, pi in zip(grid_shape, proc_shape))
from pymbolic import var
x = var("x")
y = var("y")
i, j, k = var("i"), var("j"), var("k")
map_dict = {}
map_dict[y[i, j,
k]] = (x[i + h + h, j + h, k + h] + x[i + h, j + h + h, k + h] +
x[i + h, j + h, k + h + h] + x[i - h + h, j + h, k + h] +
x[i + h, j - h + h, k + h] + x[i + h, j + h, k - h + h])
if stream:
try:
stencil_map = ps.StreamingStencil(map_dict,
prefetch_args=["x"],
halo_shape=h)
except: # noqa
pytest.skip("StreamingStencil unavailable")
else:
stencil_map = ps.Stencil(map_dict, h, prefetch_args=["x"])
x = clr.rand(queue, tuple(ni + 2 * h for ni in rank_shape), dtype)
y = clr.rand(queue, rank_shape, dtype)
x_h = x.get()
y_true = (x_h[2 * h:, h:-h, h:-h] + x_h[h:-h, 2 * h:, h:-h] +
x_h[h:-h, h:-h, 2 * h:] + x_h[:-2 * h, h:-h, h:-h] +
x_h[h:-h, :-2 * h, h:-h] + x_h[h:-h, h:-h, :-2 * h])
stencil_map(queue, x=x, y=y)
max_rtol = 5e-14 if dtype == np.float64 else 1e-5
avg_rtol = 5e-14 if dtype == np.float64 else 1e-5
max_err, avg_err = get_errs(y_true, y.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"y innaccurate for {grid_shape=}, {h=}, {proc_shape=}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
t = timer(lambda: stencil_map(queue, x=x, y=y)[0])
print(
f"stencil took {t:.3f} ms for {grid_shape=}, {h=}, {proc_shape=}")
bandwidth = (x.nbytes + y.nbytes) / 1024**3 / t * 1000
print(f"Bandwidth = {bandwidth} GB/s")
----------
args : dict
Configuration dict, typically loaded from YAML file
"""
tokens(args)
embeddings(args)
features(args)
classifiers(args)
if __name__ == '__main__':
# Parse arguments and call appropriate method
docargs = docopt(__doc__)
# Program modes
modes = {
'run': run,
'tokens': tokens,
'embeddings': embeddings,
'features': features,
'classifiers': classifiers
}
f = modes[[mode for mode in modes if docargs[mode]][0]]
c = docargs['--config']
with open(c, 'r') as yml:
args = yaml.safe_load(yml)
with timer("Packet2Vec"):
f(args)
def test_vector_projector(ctx_factory,
grid_shape,
proc_shape,
h,
dtype,
timing=False):
if ctx_factory:
ctx = ctx_factory()
else:
ctx = ps.choose_device_and_make_context()
queue = cl.CommandQueue(ctx)
mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape)
rank_shape, _ = mpi.get_rank_shape_start(grid_shape)
pencil_shape = tuple(ni + 2 * h for ni in rank_shape)
L = (10, 8, 11.5)
dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape))
dk = tuple(2 * np.pi / Li for Li in L)
fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype)
cdtype = fft.cdtype
if h > 0:
stencil = FirstCenteredDifference(h)
project = ps.Projector(fft, stencil.get_eigenvalues, dk, dx)
derivs = ps.FiniteDifferencer(mpi, h, dx)
else:
project = ps.Projector(fft, lambda k, dx: k, dk, dx)
derivs = ps.SpectralCollocator(fft, dk)
vector_x = cla.empty(queue, (3, ) + pencil_shape, dtype)
div = cla.empty(queue, rank_shape, dtype)
pdx = cla.empty(queue, (3, ) + rank_shape, dtype)
def get_divergence_error(vector):
for mu in range(3):
fft.idft(vector[mu], vector_x[mu])
derivs.divergence(queue, vector_x, div)
derivs(queue, fx=vector_x[0], pdx=pdx[0])
derivs(queue, fx=vector_x[1], pdy=pdx[1])
derivs(queue, fx=vector_x[2], pdz=pdx[2])
norm = sum([clm.fabs(pdx[mu]) for mu in range(3)])
max_err = cla.max(clm.fabs(div)) / cla.max(norm)
avg_err = cla.sum(clm.fabs(div)) / cla.sum(norm)
return max_err, avg_err
max_rtol = 1e-11 if dtype == np.float64 else 1e-4
avg_rtol = 1e-13 if dtype == np.float64 else 1e-5
k_shape = fft.shape(True)
vector = cla.empty(queue, (3, ) + k_shape, cdtype)
for mu in range(3):
vector[mu] = make_data(queue, fft).astype(cdtype)
project.transversify(queue, vector)
max_err, avg_err = get_divergence_error(vector)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"transversify failed for {grid_shape=}, {h=}: {max_err=}, {avg_err=}"
plus = make_data(queue, fft).astype(cdtype)
minus = make_data(queue, fft).astype(cdtype)
project.pol_to_vec(queue, plus, minus, vector)
if isinstance(fft, gDFT):
assert all(is_hermitian(vector[i]) for i in range(3)), \
f"pol->vec is non-hermitian for {grid_shape=}, {h=}"
max_err, avg_err = get_divergence_error(vector)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol_to_vec result not transverse for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
vector_h = vector.get()
vector_2 = cla.zeros_like(vector)
project.transversify(queue, vector, vector_2)
vector_2_h = vector_2.get()
max_err, avg_err = get_errs(vector_h, vector_2_h)
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->vector != its own transverse proj. for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
plus1 = cla.zeros_like(plus)
minus1 = cla.zeros_like(minus)
project.vec_to_pol(queue, plus1, minus1, vector)
if isinstance(fft, gDFT):
assert is_hermitian(plus1) and is_hermitian(minus1), \
f"polarizations aren't hermitian for {grid_shape=}, {h=}"
max_err, avg_err = get_errs(plus1.get(), plus.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->vec->pol (plus) is not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(minus1.get(), minus.get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"pol->vec->pol (minus) is not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
project.vec_to_pol(queue, vector[0], vector[1], vector)
max_err, avg_err = get_errs(plus1.get(), vector[0].get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"in-place pol->vec->pol (plus) not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
max_err, avg_err = get_errs(minus1.get(), vector[1].get())
assert max_err < max_rtol and avg_err < avg_rtol, \
f"in-place pol->vec->pol (minus) not identity for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
# reset and test longitudinal component
for mu in range(3):
vector[mu] = make_data(queue, fft).astype(cdtype)
fft.idft(vector[mu], vector_x[mu])
long = cla.zeros_like(minus)
project.decompose_vector(queue, vector, plus1, minus1, long)
long_x = cla.empty(queue, pencil_shape, dtype)
fft.idft(long, long_x)
div_true = cla.empty(queue, rank_shape, dtype)
derivs.divergence(queue, vector_x, div_true)
derivs(queue, fx=long_x, grd=pdx)
div_long = cla.empty(queue, rank_shape, dtype)
if h != 0:
pdx_h = cla.empty(queue, (3, ) + pencil_shape, dtype)
for mu in range(3):
mpi.restore_halos(queue, pdx[mu], pdx_h[mu])
derivs.divergence(queue, pdx_h, div_long)
else:
derivs.divergence(queue, pdx, div_long)
max_err, avg_err = get_errs(div_true.get(), div_long.get())
assert max_err < 1e-6 and avg_err < 1e-11, \
f"lap(longitudinal) != div vector for {grid_shape=}, {h=}" \
f": {max_err=}, {avg_err=}"
if timing:
from common import timer
ntime = 10
t = timer(lambda: project.transversify(queue, vector), ntime=ntime)
print(f"transversify took {t:.3f} ms for {grid_shape=}")
t = timer(lambda: project.pol_to_vec(queue, plus, minus, vector),
ntime=ntime)
print(f"pol_to_vec took {t:.3f} ms for {grid_shape=}")
t = timer(lambda: project.vec_to_pol(queue, plus, minus, vector),
ntime=ntime)
print(f"vec_to_pol took {t:.3f} ms for {grid_shape=}")
t = timer(
lambda: project.decompose_vector(queue, vector, plus, minus, long),
ntime=ntime)
print(f"decompose_vector took {t:.3f} ms for {grid_shape=}")
def main():
## Parameters ##
xSet = "dist_days_time_dayOfWeek"
data = cmn.dataset(xSet = xSet)
data.load(Nparts = 10, N_points = 4000)
futureMask = makeFutureMask(data.tScope, data.tTest, futureTime = -30 * 60)
print "Model\t{}\t{}\t{}".format("Param", "RunTime", "RMSE")
# Predict on time
timer = cmn.timer()
yHat = onTime.regress(data.xScope, data.yScope, data.xTest, 1, futureMask)
time_onTime = timer.dur()
rmse_onTime = cmn.rmse(data.yTest, yHat)
print "onTime\t\t{:.2f}\t{:.2f}".format(time_onTime, rmse_onTime)
data.saveYHat(yHat, model = "onTime")
# Visualize and save the images for the model
data.visualize(yHat, "onTime")
## kNN
# Try many values of k
ks = np.ceil(2 ** (np.arange(15) / 1.5))
rmse_kNN = np.zeros(shape=(len(ks)), dtype=np.float)
time_kNN = np.zeros(shape=(len(ks)), dtype=np.float)
for i in range(len(ks)):
k = ks[i]
timer = cmn.timer()
yHat = kNN.regress(data.xScope, data.yScope, data.xTest, k, futureMask)
time_kNN[i] = timer.dur()
rmse_kNN[i] = cmn.rmse(data.yTest, yHat)
print "kNN\t{: 5.0f}\t{:.2f}\t{:.2f}".format(k, time_kNN[i], rmse_kNN[i])
data.saveYHat(yHat, model = "{}NN".format(k))
# Visualize and save the images for the model
data.visualize(yHat, "{}NN".format(k))
# Plot the historical RMSE
clf()
plot(ks, rmse_kNN)
xlabel("Number of nearest points, k in kNN")
ylabel("Root Mean Squared Error (seconds)")
title("kNN Model, RMSE for different ks")
savefig("{}/{}_{}_k-rmse.png".format(data.figPath, data.serviceName, data.routeName))
## Kernel
# Try many values of k
rhos = np.ceil(10 ** (np.arange(12) / 2.0))
rmse_kernel = np.zeros(shape=(len(rhos)), dtype=np.float)
time_kernel = np.zeros(shape=(len(rhos)), dtype=np.float)
for i in range(len(rhos)):
rho = rhos[i]
timer = cmn.timer()
yHat = kernel.regress(data.xScope, data.yScope, data.xTest, rho, futureMask)
time_kernel[i] = timer.dur()
rmse_kernel[i] = cmn.rmse(data.yTest, yHat)
print "kernel\t{: 5.0f}\t{:.2f}\t{:.2f}".format(rho, time_kernel[i], rmse_kernel[i])
data.saveYHat(yHat, model = "kernel_{}rho".format(rho))
# Visualize and save the images for the model
data.visualize(yHat, "kernel_{}rho".format(rho))
# Plot the historical RMSE
clf()
plot(rhos, rmse_kernel)
xlabel("rho Paramater")
ylabel("Root Mean Squared Error (seconds)")
title("Kernel Regression Model, RMSE for different rhos")
savefig("{}/{}_{}_kernel_rho-rmse.png".format(data.figPath, data.serviceName, data.routeName))
