def classifiers(args): """ Trains a binary classifier using the features generated in the 'features' step. Parameters ---------- args : dict Configuration dict, typically loaded from YAML file """ if 'classifiers' in args.keys(): if 'rfc' in args['classifiers']: with timer("Training Random Forest"): train.randomForestClassifier(args['working'], args['working']) with timer("Testing Random Forest"): clf = os.path.join(args['working'], 'classifiers', 'rfc.joblib') test.test_classifier(args['working'], args['working'], args['test_data'], clf, args['darpa'], args['options']['threads']) if 'gnb' in args['classifiers']: with timer("Training Gaussian Naive Bayes"): train.naiveBayesClassifier(args['working'], args['working']) with timer("Testing Gaussian Naive Bayes"): clf = os.path.join(args['working'], 'classifiers', 'gnb.joblib') test.test_classifier(args['working'], args['working'], args['test_data'], clf, args['darpa'], args['options']['threads'])
def main(): ## Parameters ## data = cmn.dataset(xSet = "dist_days_time_dayOfWeek") data.load()#N_points = 1000) futureMask = makeFutureMask(data.tScope, data.tTest) # Try many values of k vals = np.ceil(2 ** (np.arange(15) / 1.5)) rmse = np.zeros(shape=(len(vals)), dtype=np.float) for i in range(len(vals)): k = vals[i] timer = cmn.timer() yHat = manyNearestNeighborsVector(data.xScope, data.yScope, data.xTest, k, futureMask) print "k = {}\tRuntime = {:.2f}".format(k, timer.dur()) rmse[i] = cmn.rmse(data.yTest, yHat) print "\tRMSE = {:.2f}".format(rmse[i]) data.saveYHat(yHat, model = "{}NN".format(k)) # Visualize and save the images for the model data.visualize(yHat, "{}NN".format(k)) # Plot the historical RMSE clf() plot(vals, rmse) xlabel("Number of nearest points, k in kNN") ylabel("Root Mean Squared Error (seconds)") title("kNN Model, RMSE for different ks") savefig("{}/{}_{}_k-rmse.png".format(data.figPath, data.serviceName, data.routeName))
def test_generate_WKB(ctx_factory, grid_shape, proc_shape, dtype, random, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) L = (10,)*3 volume = np.product(L) dk = tuple(2 * np.pi / Li for Li in L) modes = ps.RayleighGenerator(ctx, fft, dk, volume) # only checking that this call is successful fk, dfk = modes.generate_WKB(queue, random=random) if timing: ntime = 10 from common import timer t = timer(lambda: modes.generate_WKB(queue, random=random), ntime=ntime) print(f"{random=} set_modes took {t:.3f} ms for {grid_shape=}")
def main(): ## Parameters ## data = cmn.dataset(xSet = "dist_days_time_dayOfWeek") data.load(Nparts = 10)#N_points = 1000) futureMask = makeFutureMask(data.tScope, data.tTest) # Try many values of k vals = np.ceil(2 ** np.arange(10)) rmse = np.zeros(shape=(len(vals)), dtype=np.float) for i in range(len(vals)): k = vals[i] timer = cmn.timer() yHat = regress(data.xScope, data.yScope, data.xTest, k, futureMask) print "rho = {}\tRuntime = {:.2f}".format(k, timer.dur()) rmse[i] = cmn.rmse(data.yTest, yHat) print "\tRMSE = {:.2f}".format(rmse[i]) data.saveYHat(yHat, model = "kernel_{}rho".format(k)) # Visualize and save the images for the model data.visualize(yHat, "kernel_{}rho".format(k)) # Plot the historical RMSE clf() plot(vals, rmse) xlabel("rho Paramater") ylabel("Root Mean Squared Error (seconds)") title("Kernel Regression Model, RMSE for different rhos") savefig("{}/{}_{}_kernel_rho-rmse.png".format(data.figPath, data.serviceName, data.routeName))
def test_reduction(ctx_factory, grid_shape, proc_shape, dtype, op, _grid_shape, pass_grid_dims, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) h = 1 grid_shape = _grid_shape or grid_shape mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) from pymbolic import var from pystella import Field tmp_insns = [(var("x"), Field("f") / 2 + .31)] reducers = {} reducers["avg"] = [(var("x"), op)] if pass_grid_dims: reducer = ps.Reduction(mpi, reducers, rank_shape=rank_shape, tmp_instructions=tmp_insns, grid_size=np.product(grid_shape)) else: reducer = ps.Reduction(mpi, reducers, tmp_instructions=tmp_insns) f = clr.rand(queue, rank_shape, dtype=dtype) import pyopencl.tools as clt pool = clt.MemoryPool(clt.ImmediateAllocator(queue)) result = reducer(queue, f=f, allocator=pool) avg = result["avg"] avg_test = reducer.reduce_array(f / 2 + .31, op) if op == "avg": avg_test /= np.product(grid_shape) rtol = 5e-14 if dtype == np.float64 else 1e-5 assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \ f"{op} reduction innaccurate for {grid_shape=}, {proc_shape=}" if timing: from common import timer t = timer(lambda: reducer(queue, f=f, allocator=pool), ntime=1000) if mpi.rank == 0: print( f"reduction took {t:.3f} ms for {grid_shape=}, {proc_shape=}") bandwidth = f.nbytes / 1024**3 / t * 1000 print(f"Bandwidth = {bandwidth:.1f} GB/s")
def test_field_statistics(ctx_factory, grid_shape, proc_shape, dtype, _grid_shape, pass_grid_dims, timing=False): ctx = ctx_factory() queue = cl.CommandQueue(ctx) h = 1 grid_shape = _grid_shape or grid_shape mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) # make select parameters local for convenience h = 2 f = clr.rand(queue, (2, 1) + tuple(ni + 2 * h for ni in rank_shape), dtype=dtype) if pass_grid_dims: statistics = ps.FieldStatistics(mpi, h, rank_shape=rank_shape, grid_size=np.product(grid_shape)) else: statistics = ps.FieldStatistics(mpi, h) import pyopencl.tools as clt pool = clt.MemoryPool(clt.ImmediateAllocator(queue)) stats = statistics(f, allocator=pool) avg = stats["mean"] var = stats["variance"] f_h = f.get() rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h], axis=(-3, -2, -1)) avg_test = mpi.allreduce(rank_sum) / np.product(grid_shape) rank_sum = np.sum(f_h[..., h:-h, h:-h, h:-h]**2, axis=(-3, -2, -1)) var_test = mpi.allreduce(rank_sum) / np.product(grid_shape) - avg_test**2 rtol = 5e-14 if dtype == np.float64 else 1e-5 assert np.allclose(avg, avg_test, rtol=rtol, atol=0), \ f"average innaccurate for {grid_shape=}, {proc_shape=}" assert np.allclose(var, var_test, rtol=rtol, atol=0), \ f"variance innaccurate for {grid_shape=}, {proc_shape=}" if timing: from common import timer t = timer(lambda: statistics(f, allocator=pool)) if mpi.rank == 0: print( f"field stats took {t:.3f} ms " f"for outer shape {f.shape[:-3]}, {grid_shape=}, {proc_shape=}" )
def test_histogram(ctx_factory, grid_shape, proc_shape, dtype, num_bins, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")): max_rtol, avg_rtol = 1e-10, 1e-11 else: max_rtol, avg_rtol = 5e-4, 5e-5 from pymbolic import var _fx = ps.Field("fx") histograms = { "count": (var("abs")(_fx) * num_bins, 1), "squared": (var("abs")(_fx) * num_bins, _fx**2), } hist = ps.Histogrammer(mpi, histograms, num_bins, dtype, rank_shape=rank_shape) rng = clr.ThreefryGenerator(ctx, seed=12321) fx = rng.uniform(queue, rank_shape, dtype) fx_h = fx.get() result = hist(queue, fx=fx) res = result["count"] assert np.sum(res.astype("int64")) == np.product(grid_shape), \ f"Count histogram doesn't sum to grid_size ({np.sum(res)})" bins = np.linspace(0, 1, num_bins+1).astype(dtype) weights = np.ones_like(fx_h) np_res = np.histogram(fx_h, bins=bins, weights=weights)[0] np_res = mpi.allreduce(np_res) max_err, avg_err = get_errs(res, np_res) assert max_err < max_rtol and avg_err < avg_rtol, \ f"Histogrammer inaccurate for grid_shape={grid_shape}" \ f": {max_err=}, {avg_err=}" res = result["squared"] np_res = np.histogram(fx_h, bins=bins, weights=fx_h**2)[0] np_res = mpi.allreduce(np_res) max_err, avg_err = get_errs(res, np_res) assert max_err < max_rtol and avg_err < avg_rtol, \ f"Histogrammer with weights inaccurate for grid_shape={grid_shape}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: hist(queue, fx=fx)) print(f"histogram took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_generate(ctx_factory, grid_shape, proc_shape, dtype, random, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) num_bins = int(sum(Ni**2 for Ni in grid_shape)**.5 / 2 + .5) + 1 L = (10,)*3 volume = np.product(L) dk = tuple(2 * np.pi / Li for Li in L) spectra = ps.PowerSpectra(mpi, fft, dk, volume) modes = ps.RayleighGenerator(ctx, fft, dk, volume, seed=5123) kbins = min(dk) * np.arange(0, num_bins) test_norm = 1 / 2 / np.pi**2 / np.product(grid_shape)**2 for exp in [-1, -2, -3]: def power(k): return k**exp fk = modes.generate(queue, random=random, norm=1, field_ps=power) spectrum = spectra.norm * spectra.bin_power(fk, queue=queue, k_power=3)[1:-1] true_spectrum = test_norm * kbins[1:-1]**3 * power(kbins[1:-1]) err = np.abs(1 - spectrum / true_spectrum) tol = .1 if num_bins < 64 else .3 assert (np.max(err[num_bins//3:-num_bins//3]) < tol and np.average(err[1:]) < tol), \ f"init power spectrum incorrect for {random=}, k**{exp}" if random: fx = fft.idft(cla.to_device(queue, fk)).real if isinstance(fx, cla.Array): fx = fx.get() grid_size = np.product(grid_shape) avg = mpi.allreduce(np.sum(fx)) / grid_size var = mpi.allreduce(np.sum(fx**2)) / grid_size - avg**2 skew = mpi.allreduce(np.sum(fx**3)) / grid_size - 3 * avg * var - avg**3 skew /= var**1.5 assert skew < tol, \ f"init power spectrum has large skewness for k**{exp}" if timing: ntime = 10 from common import timer t = timer(lambda: modes.generate(queue, random=random), ntime=ntime) print(f"{random=} set_modes took {t:.3f} ms for {grid_shape=}")
def features(args): """ Applys the trained Word2Vec model to translate the token vectors into feature vectors. Parameters ---------- args : dict Configuration dict, typically loaded from YAML file """ with timer("Generating Feature Vectors"): pf.finalize_feature_vectors(args['working'], args['working'], args['darpa'])
def tokens(args): """ Generate token vectors from raw pcap files. Parameters ---------- args : dict Configuration dict, typically loaded from YAML file """ with timer("Generating Tokens and Dictionary"): pp.main(args['train_data'], args['working'], num_threads=args['options']['threads'], ngram=[args['hyperparameters']['ngram']], vocab_size=args['hyperparameters']['vocab_size'])
def main(): ## Parameters ## data = cmn.dataset(xSet = "dist_days_time_dayOfWeek") data.load(Nparts = 10)#N_points = 1000) futureMask = makeFutureMask(data.tScope, data.tTest) rmse = np.zeros(shape=(len(vals)), dtype=np.float) timer = cmn.timer() yHat = regress(data.xScope, data.yScope, data.xTest, 1, futureMask) print "onTime\tRuntime = {:.2f}".format(timer.dur()) rmse[i] = cmn.rmse(data.yTest, yHat) print "\tRMSE = {:.2f}".format(rmse[i]) data.saveYHat(yHat, model = "onTime".format(k)) # Visualize and save the images for the model data.visualize(yHat, "onTime".format(k))
def embeddings(args): """ Train a Word2Vec model using Tensorflow with the token vectors generated in the 'tokens' step. Parameters ---------- args : dict Configuration dict, typically loaded from YAML file """ with timer("Training Word2Vec Model"): if 'embeddings' not in args.keys() or args['embeddings'] == None: te.create(args['working'], args['working'], args['hyperparameters']['vocab_size']) else: te.update(args['working'], args['embeddings'], args['working'], args['hyperparameters']['vocab_size'])
def main(): ## Parameters ## data = cmn.dataset(xSet = "dist_days_time_dayOfWeek") data.load()#N_points = 1000) k = 10 futureMask = makeFutureMask(data.tScope, data.tTest) timer = cmn.timer() yHat = manyNearestNeighborsVector(data.xScope, data.yScope, data.xTest, k, futureMask) print "Vectorized Runtime = {:.2f}".format(timer.dur()) print "RMSE = {:.2f}".format(cmn.rmse(data.yTest, yHat)) data.saveYHat(yHat, model = "{}NN".format(k)) #timer.reset() #yHat2 = manyNearestNeighbors(data.xTrain, data.yTrain, data.xTest, k) #print "Iterative Runtime = {:.2f}".format(timer.dur()) #print "RMSE = {}".format(cmn.rmse(data.yTest, yHat)) # Visualize and save the images for the model data.visualize(yHat, "{}NN".format(k))
def main(): ## Parameters ## #data = cmn.dataset(xSet = "traj") #data.load()#N_points = 1000) #futureMask = makeFutureMask(data.tScope, data.tTest) dataPath = "/projects/onebusaway/BakerNiedMLProject/data/routefeatures" resPath = "/projects/onebusaway/BakerNiedMLProject/data/modelPredictions" figPath = "/projects/onebusaway/BakerNiedMLProject/figures/predictions" serviceName = "intercitytransit" routeName = "route13" xSet = "traj" ySet = "dev" x = np.loadtxt("{}/{}_{}_{}.txt".format(dataPath, serviceName, routeName, xSet), dtype=np.float) # Try many values of k vals = np.ceil(2 ** (np.arange(15) / 1.5)) rmse = np.zeros(shape=(len(vals)), dtype=np.float) minK = 0 minRMSE = 0 sel = np.random.permutation(range(len(x))); split = len(x)/4; xTrain = x[sel[:split*2]]; xVal = x[sel[split*2:3*split]]; xTest = x[sel[3*split:]]; yTest = np.zeros(len(xVal)); yHat = np.zeros(len(xVal)); data_norm = np.empty(shape = x.shape) theMean = x[:,:].mean() theStdDev = x[:,:].std() data_norm = (x - theMean)/ theStdDev xTrainNorm = data_norm[sel[:split*2]]; xValNorm = data_norm[sel[split*2:3*split]]; xTestNorm = data_norm[sel[3*split:]]; for i in range(len(vals)): k = vals[i] model = "{}NN".format(k); timer = cmn.timer() print xTrain.shape; print xTest.shape; print xVal.shape; for j in range(len(xVal)): v = len(xVal[0])-15 t = np.random.randint(10,v); yTest[j] = xVal[j][t+10]; #print xTrain[:,:t].shape; #print xTrain[:,t+10].shape; #print xVal[j,:t].shape; #print t; yHat[i] = manyNearestNeighborsVector(xTrainNorm[:,:t], xTrain[:,t+10], xValNorm[j,:t].reshape(1,t), k, weights=np.ones(t)) print "k = {}\tRuntime = {:.2f}".format(k, timer.dur()) rmse[i] = cmn.rmse(yTest, yHat) if i == 0 or rmse[i]<minRMSE: minRMSE = rmse[i] minK = vals[i] print "\tRMSE = {:.2f}".format(rmse[i]) np.savetxt("{}/{}_{}_{}_{}_val.txt".format(resPath, serviceName, routeName, model, xSet), cmn.cmb(xVal, yTest, yHat)) k = minK model = "{}NN".format(k); yTest = np.zeros(len(xTest)); yHat = np.zeros(len(xTest)); for i in range(len(xTest)): v = len(xVal[0])-15 t = np.random.randint(10,v); yTest[i] = xTest[i][t+10]; yHat[i] = manyNearestNeighborsVector(xTrain[:,:t], xTrain[:,t+10], xTest[i,:t].reshape(1,t), k, weights=np.ones(t)) # Visualize and save the images for the model #data.visualize(yHat, "{}NN".format(k)) np.savetxt("{}/{}_{}_{}_{}_test.txt".format(resPath, serviceName, routeName, model, xSet), cmn.cmb(xTest, yTest, yHat)) # Plot the historical RMSE clf() plot(vals, rmse) xlabel("Number of nearest points, k in kNN") ylabel("Root Mean Squared Error (seconds)") title("kNN Model, RMSE for different ks") savefig("{}/{}_{}_k-rmse.png".format(figPath, serviceName, routeName))
def test_elementwise(ctx_factory, grid_shape, proc_shape, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) rank_shape = tuple(Ni // pi for Ni, pi in zip(grid_shape, proc_shape)) from pymbolic import var a = var("a") b = var("b") from pystella.field import Field x = Field("x") y = Field("y") z = Field("z") tmp_dict = {a[0]: x + 2, a[1]: 2 + x * y, b: x + y / 2} map_dict = {x: a[0] * y**2 * x + a[1] * b, z: z + a[1] * b} single_insn = {x: y + z} ew_map = ps.ElementWiseMap(map_dict, tmp_instructions=tmp_dict) x = clr.rand(queue, rank_shape, dtype=dtype) y = clr.rand(queue, rank_shape, dtype=dtype) z = clr.rand(queue, rank_shape, dtype=dtype) a0 = x + 2 a1 = 2 + x * y b = x + y / 2 x_true = a0 * y**2 * x + a1 * b z_true = z + a1 * b ew_map(queue, x=x, y=y, z=z) max_rtol = 5e-14 if dtype == np.float64 else 1e-5 avg_rtol = 5e-14 if dtype == np.float64 else 1e-5 max_err, avg_err = get_errs(x_true.get(), x.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"x innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}" max_err, avg_err = get_errs(z_true.get(), z.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"z innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}" # test success of single instruction ew_map_single = ps.ElementWiseMap(single_insn) ew_map_single(queue, x=x, y=y, z=z) x_true = y + z max_err, avg_err = get_errs(x_true.get(), x.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"x innaccurate for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: ew_map(queue, x=x, y=y, z=z)[0]) print( f"elementwise map took {t:.3f} ms for {grid_shape=}, {proc_shape=}" ) bandwidth = 5 * x.nbytes / 1024**3 / t * 1000 print(f"Bandwidth = {bandwidth:.1f} GB/s")
agent = PongAgent(state_size, n_actions) elif mode == "evo": from evo import PongAgent, training_loop agent = PongAgent(state_size, n_actions) callbacks = [ TrainingLogger(config.training_log), ModelSaver(agent, config.agent), ] agent = training_loop(agent, env, config, callbacks) return agent # ---------------------------------------------------------------------- def main(): config.read_args() initial_seed(config.seed) display_versions() agent = train_agent("dqn") # dqn, evo, pg agent.save(config.agent) print("All OK") # ---------------------------------------------------------------------- if __name__ == "__main__": with timer("Pong RL"): main()
def check_outp(self): log = self.path_for('log') outp = self.path_for('outp') dat = self.path_for('persist') if not log: raise Warning("check_outp: Job %s missing log file" % self.name) if not outp: raise Warning("check_outp: Job %s missing outp file" % self.name) with common.timer('rawfine'): raw_dat = Datum.from_tuples(rawfine.read2csv(log, dat, meta=False)) with common.timer('refine'): csv_dat = [] with open(outp, 'r') as fh: for lineno, line in enumerate(fh): line = line.strip() if not line: continue if line.startswith(COMMENT_LINE): continue mean, err, its = line.split(',') datum = Datum(float(mean), float(err), int(its)) datum._line_no = lineno fin = math.isfinite if not (fin(datum.mean) and fin(datum.stderr)): continue csv_dat.append(datum) # lcs = common.LCS(csv_dat, raw_dat) with common.timer('lcs'): lcs = common.LCSGreedyApprox(raw_dat, csv_dat) lcs.invert() com_dat = lcs.common() diff = lcs.diff()[0] if diff: fix = "#mean,stderr,weight\n" for d, k, x in diff: try: x, _ = x except TypeError: pass fix += str(x) + "\n" warn = "\n" warn += "check_outp: Job %s:\n" % self.name warn += " Inconsistent outp(%d, %s) log(%d, %s) outputs!\n" % ( len(csv_dat), EXTS['outp'], len(raw_dat), EXTS['log']) overs = set() notcsv = 0 warn2 = " in outp but not log (*):\n" warn2 += " #indx(line): mean,stderr,n\n" for d, k, x in diff: if d < 0: if k in self.overrides['outp']: overs.add(k) else: ln = getattr(x, '_line_no', None) ln = '????' if ln is None else '%4d' % ln warn2 += " %4d(%s): %s\n" % (k, ln, x) notcsv += 1 if notcsv: warn2 += " * may indicate incorrect weighting...\n" warn += warn2 excess = self.overrides['outp'].difference(overs) if excess: warn += " overriden unnecessarily: %r\n" % excess notlog = 0 warn2 = " in log but not outp:\n" for d, k, x in diff: if d > 0: warn2 += " %4d: %s (mean~%f)\n" % (k, x, 1 / x.mean) notlog += 1 if notlog: warn += warn2 warn += " A candidate fix has been written to %s" % EXTS[ 'outp_fix'] if notcsv or notlog or excess: with open(self.path_for('outp_fix', True), 'w') as f: f.write(fix) raise Warning(warn)
max_flux = data.groupby(['object_id', 'passband'])['flux'].max().reset_index() max_flux = pd.merge(max_flux, data[['object_id', 'passband', 'flux', 'mjd']], on=['object_id', 'passband', 'flux'], how='left') max_flux.columns = [ 'object_id', 'passband', 'max(flux)', 'time(max(flux))' ] max_flux.drop_duplicates(subset=['object_id', 'passband'], inplace=True) max_flux.reset_index(drop=True).to_feather(config.DATA_DIR + 'passband_meta.f') if __name__ == "__main__": with timer("Make directory"): mkdir(config.DATA_DIR) mkdir(config.DEBUG_CSV_SAVE_DIR) mkdir(config.FEATURE_LOAD_DIR) mkdir(config.FEATURE_SAVE_DIR) mkdir(config.MODEL_DIR) mkdir(config.SHARE_DIR) mkdir(config.SUBMIT_DIR) with timer("Convert metadata"): concat_to_feather(config.DATA_DIR + "training_set_metadata.csv", config.DATA_DIR + "test_set_metadata.csv", config.DATA_DIR + "meta.f", column_order=[ 'object_id', 'ra', 'decl', 'gal_l', 'gal_b', 'ddf', 'hostgal_specz', 'hostgal_photoz',
def test_dft(ctx_factory, grid_shape, proc_shape, dtype, use_fftw, timing=False): if not use_fftw and np.product(proc_shape) > 1: pytest.skip("Must use mpi4py-fft on more than one rank.") ctx = ctx_factory() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) mpi0 = ps.DomainDecomposition(proc_shape, 0, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype, use_fftw=use_fftw) grid_size = np.product(grid_shape) rdtype = fft.rdtype if fft.is_real: np_dft = np.fft.rfftn np_idft = np.fft.irfftn else: np_dft = np.fft.fftn np_idft = np.fft.ifftn rtol = 1e-11 if dtype in ("float64", "complex128") else 2e-3 rng = clr.ThreefryGenerator(ctx, seed=12321 * (mpi.rank + 1)) fx = rng.uniform(queue, rank_shape, rdtype) + 1e-2 if not fft.is_real: fx = fx + 1j * rng.uniform(queue, rank_shape, rdtype) fx = fx.get() fk = fft.dft(fx) if isinstance(fk, cla.Array): fk = fk.get() fk, _fk = fk.copy(), fk # hang on to one that fftw won't overwrite fx2 = fft.idft(_fk) if isinstance(fx2, cla.Array): fx2 = fx2.get() fx_glb = np.empty(shape=grid_shape, dtype=dtype) for root in range(mpi.nranks): mpi0.gather_array(queue, fx, fx_glb, root=root) fk_glb_np = np.ascontiguousarray(np_dft(fx_glb)) fx2_glb_np = np.ascontiguousarray(np_idft(fk_glb_np)) if use_fftw: fk_np = fk_glb_np[fft.fft.local_slice(True)] fx2_np = fx2_glb_np[fft.fft.local_slice(False)] else: fk_np = fk_glb_np fx2_np = fx2_glb_np max_err, avg_err = get_errs(fx, fx2 / grid_size) assert max_err < rtol, \ f"IDFT(DFT(f)) != f for {grid_shape=}, {proc_shape=}: {max_err=}, {avg_err=}" max_err, avg_err = get_errs(fk_np, fk) assert max_err < rtol, \ f"DFT disagrees with numpy for {grid_shape=}, {proc_shape=}:"\ f" {max_err=}, {avg_err=}" max_err, avg_err = get_errs(fx2_np, fx2 / grid_size) assert max_err < rtol, \ f"IDFT disagrees with numpy for {grid_shape=}, {proc_shape=}:"\ f" {max_err=}, {avg_err=}" fx_cl = cla.empty(queue, rank_shape, dtype) pencil_shape = tuple(ni + 2 * h for ni in rank_shape) fx_cl_halo = cla.empty(queue, pencil_shape, dtype) fx_np = np.empty(rank_shape, dtype) fx_np_halo = np.empty(pencil_shape, dtype) fk_cl = cla.empty(queue, fft.shape(True), fft.fk.dtype) fk_np = np.empty(fft.shape(True), fft.fk.dtype) # FIXME: check that these actually produce the correct result fx_types = { "cl": fx_cl, "cl halo": fx_cl_halo, "np": fx_np, "np halo": fx_np_halo, "None": None } fk_types = {"cl": fk_cl, "np": fk_np, "None": None} # run all of these to ensure no runtime errors even if no timing ntime = 20 if timing else 1 from common import timer if mpi.rank == 0: print(f"N = {grid_shape}, ", "complex" if np.dtype(dtype).kind == "c" else "real") from itertools import product for (a, input_), (b, output) in product(fx_types.items(), fk_types.items()): t = timer(lambda: fft.dft(input_, output), ntime=ntime) if mpi.rank == 0: print(f"dft({a}, {b}) took {t:.3f} ms") for (a, input_), (b, output) in product(fk_types.items(), fx_types.items()): t = timer(lambda: fft.idft(input_, output), ntime=ntime) if mpi.rank == 0: print(f"idft({a}, {b}) took {t:.3f} ms")
def extract_features(meta: pd.DataFrame, lc: pd.DataFrame, source: str, normalize: bool = False, snr: int = 3, zbounds: str = 'estimated', skip: int = 0, end: int = -1, clip_bounds: bool = False, t_bounds: bool = False, columns: List[str] = None): if normalize: try: for band in ['g', 'i', 'r', 'u', 'y', 'z']: b = read_bandpass('lsst/total_{}.dat'.format(band), wave_unit=u.nm, trim_level=0.001, name='lsst{}_n'.format(band), normalize=True) sncosmo.register(b, 'lsst{}_n'.format(band)) except: raise pass with timer('dropping meta'): meta = meta[meta.object_id.isin(lc.object_id)].reset_index(drop=True) print('shape(meta): {}'.format(meta.shape)) if 'object_id' in meta: meta.set_index('object_id', inplace=True) if normalize: passbands = [ 'lsstu_n', 'lsstg_n', 'lsstr_n', 'lssti_n', 'lsstz_n', 'lssty_n' ] else: passbands = ['lsstu', 'lsstg', 'lsstr', 'lssti', 'lsstz', 'lssty'] with timer('prep'): lc['band'] = lc['passband'].apply(lambda x: passbands[x]) lc['zpsys'] = 'ab' lc['zp'] = 25.0 # create a model model = Model(source=source) params = model.param_names if columns: ret = pd.DataFrame(columns=columns) else: columns = ['chisq', 'ncall'] + [source + '_' + c for c in params] + [ source + '_' + c + '_err' for c in params ] ret = pd.DataFrame(columns=columns) prefix = source if zbounds == 'fixed': prefix += '_f_' else: prefix += '_p_' prefix += 'sn{}_'.format(snr) if normalize: prefix += 'n_' ret.columns = [prefix + c for c in ret.columns] n_errors = 0 n_loop = len(meta) if end > 0: n_loop = end for i in tqdm(range(skip, n_loop)): object_id = meta.index[i] try: ret.loc[object_id] = fit_lc(model, meta, lc, object_id, zbounds, clip_bounds, t_bounds, snr) except: n_errors += 1 if i == 30 and n_errors == 31: print('All 30 first attempts were failed. stopped') raise print('total {} data processed. {} data was skipped'.format( len(meta), n_errors)) ret.reset_index(inplace=True) ret.rename(columns={'index': 'object_id'}, inplace=True) return ret
def generate_index(args): jobdir = args.jobdir cachef = None jobs = {} cache = {} if args.cache: cachef = os.path.join(jobdir, args.cache) try: if not args.replan: with open(cachef, 'r') as f: cache = json.load(f) except FileNotFoundError: pass with common.timer(1): for root, _, files in os.walk(jobdir): for file in files: path = os.path.join(root, file) rpath = pathlib.PurePath(path).relative_to(jobdir) jobn = str(rpath.with_suffix('')) if rpath.suffix in EXTS_rev: jobs.setdefault(jobn, Job(jobdir, jobn)) jobs[jobn].add_data(rpath.suffix) index = Index(args) with common.timer(2): warnings = False prog_bar = False for jobn, job in jobs.items(): if jobn in cache: params = Parameters.from_dict(cache[jobn]) job.set_parameters(params) else: error('.', end='', flush=True) prog_bar = True with common.timer((3, jobn)): try: with common.timer((3, jobn, 'overrides')): job.get_overrides() with common.timer((3, jobn, 'outp')): job.check_outp() with common.timer((3, jobn, 'params')): params = job.get_parameters() cache[jobn] = params.to_dict() except Warning as w: error(w) warnings = True continue index[params].add(job) if prog_bar: error('') if cachef: with open(cachef, 'w') as f: json.dump(cache, f, sort_keys=True, indent=4) if warnings: error("\n*** Please fix the above warnings before continuing") error( "* For inconsistent outp/log, manually edit relevant files to resolve inconsistencies or use override*" ) error( "* For missing outp files, use rawfine.py cautiously to reconstruct" ) error( "* For missing log files, first touch the log file, then override* the relevant warnings." ) error( "* For uninferred parameters, you may have an empty log file - add params to the log file..." ) error("** .override files: see README.md") exit(1) return False with common.timer(4): warnings = False for p, xs in index.all(): try: xs.resolve() except Warning as w: error(w) warnings = True continue if warnings: error("\n*** Please fix the above warnings before continuing") error("* Staged plans need manual review before use;") error(" if the plan looks right, remove the staged key") exit(1) return False return index
def main(): ## Parameters ## xSet = "dist_days_time_dayOfWeek_normalized" # Load the Data data = cmn.dataset(xSet = xSet) data.load(Nparts = 10)#, N_points = 12000) # Set X, Y, xTest and get sizes Y = data.yScope; (N_train, N_feat) = data.xScope.shape N_test = data.xTest.shape[0] # Precompute the future mask futureMask = makeFutureMask(data.tScope, data.tTest, futureTime = -30 * 60) # Compute the distance matrix (since they all use the same) weights = np.array([3, 0.5, 2, 1]) timer = cmn.timer() traintile = np.tile(np.reshape(data.xScope, (N_train, N_feat, 1)), (1, 1, N_test)); testtile = np.tile(np.transpose(np.reshape(data.xTest, (N_test, N_feat, 1)), (2, 1, 0)), (N_train, 1, 1)); weightstile = np.tile(weights.reshape(1, N_feat, 1), (N_train, 1, N_test)) print "Dist runtime: {:.2f}".format(timer.dur()) dist = np.sum(((traintile - testtile) ** 2) * weightstile, axis=1) ## Try out some models with rhos and ks filename = "{}/{}_{}_tests_Ntrain{}.txt".format(data.resPath, data.serviceName, data.routeName, N_train) f = open(filename, 'w') str = "Model\t{}\t{}\t{}\n".format("Param", "RunTime", "RMSE") print str[:len(str)-1] f.write(str) # Make a list of parameters to test rhos = 2 ** (np.arange(-10, 10) / 2.0) ks = np.ceil( 2 ** (np.arange(18) / 1.5)) # Allocate data containers N_onTime = 1 N_kernel = len(rhos) N_kNN = len(ks) N_attempts = N_onTime + N_kernel + N_kNN attempts = range(0, N_attempts) rmses = np.zeros(shape=(N_attempts), dtype=np.float) times = np.zeros(shape=(N_attempts), dtype=np.float) models = [""] * N_attempts for i in attempts: timer = cmn.timer() if(i < N_onTime): yHat = onTime(dist.shape[1]) models[i] = "onTime\t" elif(i < (N_kernel + N_onTime)): rho = rhos[i - N_onTime] yHat = kernel(dist, futureMask, Y, rho) models[i] = "kernel\t{: 3.2f}".format(rho) elif(i < (N_kNN + N_kernel + N_onTime)): k = ks[i - N_onTime - N_kernel] yHat = kNN(dist, futureMask, Y, k) models[i] = "kNN\t{: 5.0f}".format(k) times[i] = timer.dur() rmses[i] = cmn.rmse(data.yTest, yHat) str = "{}\t{:.2f}\t{:.2f}\n".format(models[i], times[i], rmses[i]) print str[:len(str)-1] f.write(str) f.close()
def test_share_halos(ctx_factory, grid_shape, proc_shape, h, dtype, _grid_shape, pass_grid_shape, timing=False): ctx = ctx_factory() if isinstance(h, int): h = (h, ) * 3 queue = cl.CommandQueue(ctx) grid_shape = _grid_shape or grid_shape mpi = ps.DomainDecomposition( proc_shape, h, grid_shape=(grid_shape if pass_grid_shape else None)) rank_shape, substart = mpi.get_rank_shape_start(grid_shape) # data will be same on each rank rng = clr.ThreefryGenerator(ctx, seed=12321) data = rng.uniform(queue, tuple(Ni + 2 * hi for Ni, hi in zip(grid_shape, h)), dtype).get() if h[0] > 0: data[:h[0], :, :] = data[-2 * h[0]:-h[0], :, :] data[-h[0]:, :, :] = data[h[0]:2 * h[0], :, :] if h[1] > 0: data[:, :h[1], :] = data[:, -2 * h[1]:-h[1], :] data[:, -h[1]:, :] = data[:, h[1]:2 * h[1], :] if h[2] > 0: data[:, :, :h[2]] = data[:, :, -2 * h[2]:-h[2]] data[:, :, -h[2]:] = data[:, :, h[2]:2 * h[2]] subdata = np.empty(tuple(ni + 2 * hi for ni, hi in zip(rank_shape, h)), dtype) rank_slice = tuple( slice(si + hi, si + ni + hi) for ni, si, hi in zip(rank_shape, substart, h)) unpadded_slc = tuple(slice(hi, -hi) if hi > 0 else slice(None) for hi in h) subdata[unpadded_slc] = data[rank_slice] subdata_device = cla.to_device(queue, subdata) mpi.share_halos(queue, subdata_device) subdata2 = subdata_device.get() pencil_slice = tuple( slice(si, si + ri + 2 * hi) for ri, si, hi in zip(rank_shape, substart, h)) assert (subdata2 == data[pencil_slice]).all(), \ f"rank {mpi.rank} {mpi.rank_tuple} has incorrect halo data" # test that can call with different-shaped input if not pass_grid_shape: subdata_device_new = clr.rand( queue, tuple(ni // 2 + 2 * hi for ni, hi in zip(rank_shape, h)), dtype) mpi.share_halos(queue, subdata_device_new) if timing: from common import timer t = timer(lambda: mpi.share_halos(queue, fx=subdata_device)) if mpi.rank == 0: print(f"share_halos took {t:.3f} ms for " f"{grid_shape=}, {h=}, {proc_shape=}")
def test_field_histogram(ctx_factory, grid_shape, proc_shape, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) pencil_shape = tuple(Ni + 2 * h for Ni in rank_shape) num_bins = 432 if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")): max_rtol, avg_rtol = 1e-10, 1e-11 else: max_rtol, avg_rtol = 5e-4, 5e-5 hist = ps.FieldHistogrammer(mpi, num_bins, dtype, rank_shape=rank_shape, halo_shape=h) rng = clr.ThreefryGenerator(ctx, seed=12321) fx = rng.uniform(queue, (2, 2)+pencil_shape, dtype, a=-1.2, b=3.) fx_h = fx.get()[..., h:-h, h:-h, h:-h] result = hist(fx) outer_shape = fx.shape[:-3] from itertools import product slices = list(product(*[range(n) for n in outer_shape])) for slc in slices: res = result["linear"][slc] np_res = np.histogram(fx_h[slc], bins=result["linear_bins"][slc])[0] np_res = mpi.allreduce(np_res) max_err, avg_err = get_errs(res, np_res) assert max_err < max_rtol and avg_err < avg_rtol, \ f"linear Histogrammer inaccurate for grid_shape={grid_shape}" \ f": {max_err=}, {avg_err=}" res = result["log"][slc] bins = result["log_bins"][slc] # avoid FPA comparison issues # numpy sometimes doesn't count the actual maximum/minimum eps = 1e-14 if np.dtype(dtype) == np.dtype("float64") else 1e-4 bins[0] *= (1 - eps) bins[-1] *= (1 + eps) np_res = np.histogram(np.abs(fx_h[slc]), bins=bins)[0] np_res = mpi.allreduce(np_res) norm = np.maximum(np.abs(res), np.abs(np_res)) norm[norm == 0.] = 1. max_err, avg_err = get_errs(res, np_res) assert max_err < max_rtol and avg_err < avg_rtol, \ f"log Histogrammer inaccurate for grid_shape={grid_shape}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: hist(fx[0, 0])) print(f"field histogram took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_spectral_poisson(ctx_factory, grid_shape, proc_shape, h, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) L = (3, 5, 7) dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape)) dk = tuple(2 * np.pi / Li for Li in L) if h == 0: def get_evals_2(k, dx): return - k**2 derivs = ps.SpectralCollocator(fft, dk) else: from pystella.derivs import SecondCenteredDifference get_evals_2 = SecondCenteredDifference(h).get_eigenvalues derivs = ps.FiniteDifferencer(mpi, h, dx, stream=False) solver = ps.SpectralPoissonSolver(fft, dk, dx, get_evals_2) pencil_shape = tuple(ni + 2*h for ni in rank_shape) statistics = ps.FieldStatistics(mpi, 0, rank_shape=rank_shape, grid_size=np.product(grid_shape)) fx = cla.empty(queue, pencil_shape, dtype) rho = clr.rand(queue, rank_shape, dtype) rho -= statistics(rho)["mean"] lap = cla.empty(queue, rank_shape, dtype) rho_h = rho.get() for m_squared in (0, 1.2, 19.2): solver(queue, fx, rho, m_squared=m_squared) fx_h = fx.get() if h > 0: fx_h = fx_h[h:-h, h:-h, h:-h] derivs(queue, fx=fx, lap=lap) diff = np.fabs(lap.get() - rho_h - m_squared * fx_h) max_err = np.max(diff) / cla.max(clm.fabs(rho)) avg_err = np.sum(diff) / cla.sum(clm.fabs(rho)) max_rtol = 1e-12 if dtype == np.float64 else 1e-4 avg_rtol = 1e-13 if dtype == np.float64 else 1e-5 assert max_err < max_rtol and avg_err < avg_rtol, \ f"solution inaccurate for {h=}, {grid_shape=}, {proc_shape=}" if timing: from common import timer time = timer(lambda: solver(queue, fx, rho, m_squared=m_squared), ntime=10) if mpi.rank == 0: print(f"poisson took {time:.3f} ms for {grid_shape=}, {proc_shape=}")
def test_spectra(ctx_factory, grid_shape, proc_shape, dtype, L, timing=False): ctx = ctx_factory() queue = cl.CommandQueue(ctx) h = 1 mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) L = L or (3, 5, 7) dk = tuple(2 * np.pi / Li for Li in L) cdtype = fft.cdtype spec = ps.PowerSpectra(mpi, fft, dk, np.product(L), bin_width=min(dk) + .001) # FIXME: bin_width=min(dk) sometimes disagrees to O(.1%) with numpy... assert int(np.sum(spec.bin_counts)) == np.product(grid_shape), \ "bin counts don't sum to total number of points/modes" k_power = 2. fk = make_data(*fft.shape(True)).astype(cdtype) fk_d = cla.to_device(queue, fk) spectrum = spec.bin_power(fk_d, k_power=k_power) bins = np.arange(-.5, spec.num_bins + .5) * spec.bin_width sub_k = list(x.get() for x in fft.sub_k.values()) kvecs = np.meshgrid(*sub_k, indexing="ij", sparse=False) kmags = np.sqrt(sum((dki * ki)**2 for dki, ki in zip(dk, kvecs))) if fft.is_real: counts = 2. * np.ones_like(kmags) counts[kvecs[2] == 0] = 1 counts[kvecs[2] == grid_shape[-1] // 2] = 1 else: counts = 1. * np.ones_like(kmags) if np.dtype(dtype) in (np.dtype("float64"), np.dtype("complex128")): max_rtol = 1e-8 avg_rtol = 1e-11 else: max_rtol = 2e-2 avg_rtol = 2e-4 bin_counts2 = spec.bin_power(np.ones_like(fk), queue=queue, k_power=0) max_err, avg_err = get_errs(bin_counts2, np.ones_like(bin_counts2)) assert max_err < max_rtol and avg_err < avg_rtol, \ f"bin counting disagrees between PowerSpectra and np.histogram" \ f" for {grid_shape=}: {max_err=}, {avg_err=}" hist = np.histogram(kmags, bins=bins, weights=np.abs(fk)**2 * counts * kmags**k_power)[0] hist = mpi.allreduce(hist) / spec.bin_counts # skip the Nyquist mode and the zero mode max_err, avg_err = get_errs(spectrum[1:-2], hist[1:-2]) assert max_err < max_rtol and avg_err < avg_rtol, \ f"power spectrum inaccurate for {grid_shape=}: {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: spec.bin_power(fk_d, k_power=k_power)) print(f"power spectrum took {t:.3f} ms for {grid_shape=}, {dtype=}")
def test_tensor_projector(ctx_factory, grid_shape, proc_shape, h, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) L = (10, 8, 11.5) dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape)) dk = tuple(2 * np.pi / Li for Li in L) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) cdtype = fft.cdtype if h > 0: stencil = FirstCenteredDifference(h) project = ps.Projector(fft, stencil.get_eigenvalues, dk, dx) derivs = ps.FiniteDifferencer(mpi, h, dx) else: project = ps.Projector(fft, lambda k, dx: k, dk, dx) derivs = ps.SpectralCollocator(fft, dk) vector_x = cla.empty(queue, (3, ) + tuple(ni + 2 * h for ni in rank_shape), dtype) div = cla.empty(queue, rank_shape, dtype) pdx = cla.empty(queue, (3, ) + rank_shape, dtype) def get_divergence_errors(hij): max_errors = [] avg_errors = [] for i in range(1, 4): for mu in range(3): fft.idft(hij[tensor_id(i, mu + 1)], vector_x[mu]) derivs.divergence(queue, vector_x, div) derivs(queue, fx=vector_x[0], pdx=pdx[0]) derivs(queue, fx=vector_x[1], pdy=pdx[1]) derivs(queue, fx=vector_x[2], pdz=pdx[2]) norm = sum([clm.fabs(pdx[mu]) for mu in range(3)]) max_errors.append(cla.max(clm.fabs(div)) / cla.max(norm)) avg_errors.append(cla.sum(clm.fabs(div)) / cla.sum(norm)) return np.array(max_errors), np.array(avg_errors) max_rtol = 1e-11 if dtype == np.float64 else 1e-4 avg_rtol = 1e-13 if dtype == np.float64 else 1e-5 def get_trace_errors(hij_h): trace = sum([hij_h[tensor_id(i, i)] for i in range(1, 4)]) norm = np.sqrt( sum(np.abs(hij_h[tensor_id(i, i)])**2 for i in range(1, 4))) trace = np.abs(trace[norm != 0]) / norm[norm != 0] trace = trace[trace < .9] return np.max(trace), np.sum(trace) / trace.size k_shape = fft.shape(True) hij = cla.empty(queue, shape=(6, ) + k_shape, dtype=cdtype) for mu in range(6): hij[mu] = make_data(queue, fft).astype(cdtype) project.transverse_traceless(queue, hij) hij_h = hij.get() if isinstance(fft, gDFT): assert all(is_hermitian(hij_h[i]) for i in range(6)), \ f"TT projection is non-hermitian for {grid_shape=}, {h=}" max_err, avg_err = get_divergence_errors(hij) assert all(max_err < max_rtol) and all(avg_err < avg_rtol), \ f"TT projection not transverse for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" max_err, avg_err = get_trace_errors(hij_h) assert max_err < max_rtol and avg_err < avg_rtol, \ f"TT projected tensor isn't traceless for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" plus = make_data(queue, fft).astype(cdtype) minus = make_data(queue, fft).astype(cdtype) project.pol_to_tensor(queue, plus, minus, hij) if isinstance(fft, gDFT): assert all(is_hermitian(hij[i]) for i in range(6)), \ f"pol->tensor is non-hermitian for {grid_shape=}, {h=}" max_err, avg_err = get_divergence_errors(hij) assert all(max_err < max_rtol) and all(avg_err < avg_rtol), \ f"pol->tensor not transverse for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" hij_h = hij.get() max_err, avg_err = get_trace_errors(hij_h) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->tensor isn't traceless for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" hij_2 = cla.zeros_like(hij) project.transverse_traceless(queue, hij, hij_2) hij_h_2 = hij_2.get() max_err, avg_err = get_errs(hij_h, hij_h_2) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->tensor != its own TT projection for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" plus1 = cla.zeros_like(plus) minus1 = cla.zeros_like(minus) project.tensor_to_pol(queue, plus1, minus1, hij) if isinstance(fft, gDFT): assert is_hermitian(plus1) and is_hermitian(minus1), \ f"polarizations aren't hermitian for {grid_shape=}, {h=}" max_err, avg_err = get_errs(plus1.get(), plus.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->tensor->pol (plus) is not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" max_err, avg_err = get_errs(minus1.get(), minus.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->tensor->pol (minus) is not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" project.tensor_to_pol(queue, hij[0], hij[1], hij) max_err, avg_err = get_errs(plus1.get(), hij[0].get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"in-place pol->tensor->pol (plus) not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" max_err, avg_err = get_errs(minus1.get(), hij[1].get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"in-place pol->tensor->pol (minus) not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer ntime = 10 t = timer(lambda: project.transverse_traceless(queue, hij), ntime=ntime) print(f"TT projection took {t:.3f} ms for {grid_shape=}") t = timer(lambda: project.pol_to_tensor(queue, plus, minus, hij), ntime=ntime) print(f"pol->tensor took {t:.3f} ms for {grid_shape=}") t = timer(lambda: project.tensor_to_pol(queue, plus, minus, hij), ntime=ntime) print(f"tensor->pol took {t:.3f} ms for {grid_shape=}")
def test_scalar_energy(ctx_factory, grid_shape, proc_shape, h, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) grid_size = np.product(grid_shape) nscalars = 2 def potential(f): phi, chi = f[0], f[1] return 1 / 2 * phi**2 + 1 / 2 * chi**2 + 1 / 2 * phi**2 * chi**2 scalar_sector = ps.ScalarSector(nscalars, potential=potential) scalar_energy = ps.Reduction(mpi, scalar_sector, rank_shape=rank_shape, grid_size=grid_size, halo_shape=h) pencil_shape = tuple(ni + 2 * h for ni in rank_shape) f = clr.rand(queue, (nscalars, ) + pencil_shape, dtype) dfdt = clr.rand(queue, (nscalars, ) + pencil_shape, dtype) lap = clr.rand(queue, (nscalars, ) + rank_shape, dtype) energy = scalar_energy(queue, f=f, dfdt=dfdt, lap_f=lap, a=np.array(1.)) kin_test = [] grad_test = [] for fld in range(nscalars): df_h = dfdt[fld].get() rank_sum = np.sum(df_h[h:-h, h:-h, h:-h]**2) kin_test.append(1 / 2 * mpi.allreduce(rank_sum) / grid_size) f_h = f[fld].get() lap_h = lap[fld].get() rank_sum = np.sum(-f_h[h:-h, h:-h, h:-h] * lap_h) grad_test.append(1 / 2 * mpi.allreduce(rank_sum) / grid_size) energy_test = {} energy_test["kinetic"] = np.array(kin_test) energy_test["gradient"] = np.array(grad_test) phi = f[0].get()[h:-h, h:-h, h:-h] chi = f[1].get()[h:-h, h:-h, h:-h] pot_rank = np.sum(potential([phi, chi])) energy_test["potential"] = np.array(mpi.allreduce(pot_rank) / grid_size) max_rtol = 1e-14 if dtype == np.float64 else 1e-5 avg_rtol = 1e-14 if dtype == np.float64 else 1e-5 for key, value in energy.items(): max_err, avg_err = get_errs(value, energy_test[key]) assert max_err < max_rtol and avg_err < avg_rtol, \ f"{key} inaccurate for {nscalars=}, {grid_shape=}, {proc_shape=}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: scalar_energy( queue, a=np.array(1.), f=f, dfdt=dfdt, lap_f=lap)) if mpi.rank == 0: print(f"scalar energy took {t:.3f} " f"ms for {nscalars=}, {grid_shape=}, {proc_shape=}")
import pandas as pd import numpy as np from sklearn.feature_extraction import DictVectorizer from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer from sklearn.pipeline import FeatureUnion, Pipeline, make_union, make_pipeline from common import timer, read_csv, ItemSelector, TextStats with timer("Load data"): df_protein_train = read_csv("df_protein_train.csv") df_protein_test = read_csv("df_protein_test.csv") df_protein = pd.concat([df_protein_train, df_protein_test]) df_protein.Sequence = df_protein.Sequence.apply(lambda x: x.upper()) feature_union = make_union( make_pipeline(ItemSelector(key="Sequence"), CountVectorizer(analyzer='char', ngram_range=(1, 1))), make_pipeline( ItemSelector(key="Sequence"), TfidfVectorizer(analyzer='char', ngram_range=(1, 1), use_idf=False)), make_pipeline(ItemSelector(key="Sequence"), TextStats(), DictVectorizer())) with timer("Fit feature_union"): feat = feature_union.fit_transform(df_protein) out_col = [f'protein_stat_{i}' for i in range(feat.shape[1])] output_file = "./input/temp/df_protein_stat.csv" with timer(f"Save file to {output_file}"):
def test_gather_scatter(ctx_factory, grid_shape, proc_shape, h, dtype, _grid_shape, pass_grid_shape, timing=False): ctx = ctx_factory() if isinstance(h, int): h = (h, ) * 3 queue = cl.CommandQueue(ctx) grid_shape = _grid_shape or grid_shape mpi = ps.DomainDecomposition(proc_shape, h) rank_shape, substart = mpi.get_rank_shape_start(grid_shape) rank_slice = tuple( slice(si, si + ri) for ri, si, hi in zip(rank_shape, substart, h)) pencil_shape = tuple(ni + 2 * hi for ni, hi in zip(rank_shape, h)) unpadded_slc = tuple(slice(hi, -hi) if hi > 0 else slice(None) for hi in h) # create random data with same seed on all ranks rng = clr.ThreefryGenerator(ctx, seed=12321) data = rng.uniform(queue, grid_shape, dtype) # cl.Array -> cl.Array subdata = cla.zeros(queue, pencil_shape, dtype) mpi.scatter_array(queue, data if mpi.rank == 0 else None, subdata, 0) sub_h = subdata.get() data_h = data.get() assert (sub_h[unpadded_slc] == data_h[rank_slice]).all() data_test = cla.zeros_like(data) mpi.gather_array(queue, subdata, data_test if mpi.rank == 0 else None, 0) data_test_h = data_test.get() if mpi.rank == 0: assert (data_test_h == data_h).all() # np.ndarray -> np.ndarray mpi.scatter_array(queue, data_h if mpi.rank == 0 else None, sub_h, 0) assert (sub_h[unpadded_slc] == data_h[rank_slice]).all() mpi.gather_array(queue, sub_h, data_test_h if mpi.rank == 0 else None, 0) if mpi.rank == 0: assert (data_test_h == data_h).all() # scatter cl.Array -> np.ndarray sub_h[:] = 0 mpi.scatter_array(queue, data if mpi.rank == 0 else None, sub_h, 0) assert (sub_h[unpadded_slc] == data_h[rank_slice]).all() # gather np.ndarray -> cl.Array data_test[:] = 0 mpi.gather_array(queue, sub_h, data_test if mpi.rank == 0 else None, 0) data_test_h = data_test.get() if mpi.rank == 0: assert (data_test_h == data_h).all() # scatter np.ndarray -> cl.Array subdata[:] = 0 mpi.scatter_array(queue, data_h if mpi.rank == 0 else None, subdata, 0) sub_h = subdata.get() assert (sub_h[unpadded_slc] == data_h[rank_slice]).all() # gather cl.Array -> np.ndarray data_test_h[:] = 0 mpi.gather_array(queue, subdata, data_test_h if mpi.rank == 0 else None, 0) if mpi.rank == 0: assert (data_test_h == data_h).all() if timing: from common import timer ntime = 25 times = {} times["scatter cl.Array -> cl.Array"] = \ timer(lambda: mpi.scatter_array(queue, data, subdata, 0), ntime=ntime) times["scatter cl.Array -> np.ndarray"] = \ timer(lambda: mpi.scatter_array(queue, data, sub_h, 0), ntime=ntime) times["scatter np.ndarray -> cl.Array"] = \ timer(lambda: mpi.scatter_array(queue, data_h, subdata, 0), ntime=ntime) times["scatter np.ndarray -> np.ndarray"] = \ timer(lambda: mpi.scatter_array(queue, data_h, sub_h, 0), ntime=ntime) times["gather cl.Array -> cl.Array"] = \ timer(lambda: mpi.gather_array(queue, subdata, data, 0), ntime=ntime) times["gather cl.Array -> np.ndarray"] = \ timer(lambda: mpi.gather_array(queue, subdata, data_h, 0), ntime=ntime) times["gather np.ndarray -> cl.Array"] = \ timer(lambda: mpi.gather_array(queue, sub_h, data, 0), ntime=ntime) times["gather np.ndarray -> np.ndarray"] = \ timer(lambda: mpi.gather_array(queue, sub_h, data_h, 0), ntime=ntime) if mpi.rank == 0: print(f"{grid_shape=}, {h=}, {proc_shape=}") for key, val in times.items(): print(f"{key} took {val:.3f} ms")
import pandas as pd import numpy as np from common import timer, read_csv fp_col = [f'fp_{i}' for i in range(167)] out_col = ["molecule_count"] + fp_col output_file = "./input/temp/df_molecule_stat.csv" with timer("Load data"): df_molecule = read_csv("df_molecule.csv") df_aff_train = read_csv("df_affinity_train.csv") df_aff_test = read_csv("df_affinity_test_toBePredicted.csv") df_aff = pd.concat([df_aff_train, df_aff_test]) with timer("Make molecule count feature"): df_molecule_count = df_aff.groupby("Molecule_ID", as_index=False).Ki.agg( {"molecule_count": "count"}) df_molecule = df_molecule.merge(df_molecule_count, on=["Molecule_ID"]) with timer("Parse fingerprint"): fingerprint = df_molecule.Fingerprint.apply( lambda x: np.array(x.split(', '))).values fingerprint = np.vstack(fingerprint).astype(np.uint8) df_fingerprint = pd.DataFrame(fingerprint, columns=fp_col, dtype=np.uint8) df_molecule = pd.concat([df_molecule, df_fingerprint], axis=1) del df_fingerprint, fingerprint with timer(f"Save file to {output_file}"): df_molecule[['Molecule_ID'] + out_col].to_csv(output_file, index=False)
warnings.filterwarnings('ignore') import pandas as pd import numpy as np import gc from sklearn.linear_model import Ridge from sklearn.model_selection import KFold from common import timer, read_csv, ItemSelector fp_col = [f'fp_{i}' for i in range(167)] w2v_col = [f'w2v_{i}' for i in range(128)] with timer("Load data"): df_affinity_train = read_csv("df_affinity_train.csv") df_molecule = read_csv("df_molecule_stat.csv", "./input/temp/") df_protein_w2v = read_csv("df_w2v_ws3.csv", "./input/temp/") df_affinity_test = read_csv("df_affinity_test_toBePredicted.csv") df_train = df_affinity_train.merge(df_molecule) df_train = df_train.merge(df_protein_w2v) df_test = df_affinity_test.merge(df_molecule) df_test = df_test.merge(df_protein_w2v) test = df_test all_protein_id = df_train.Protein_ID.unique() kfold = KFold(n_splits=5, shuffle=True, random_state=2018)
def test_gradient_laplacian(ctx_factory, grid_shape, proc_shape, h, dtype, stream, timing=False): if h == 0 and stream is True: pytest.skip("no streaming spectral") ctx = ctx_factory() queue = cl.CommandQueue(ctx) mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, start = mpi.get_rank_shape_start(grid_shape) L = (3, 5, 7) dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape)) dk = tuple(2 * np.pi / Li for Li in L) if h == 0: def get_evals_1(k, dx): return k def get_evals_2(k, dx): return -k**2 fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) derivs = ps.SpectralCollocator(fft, dk) else: from pystella.derivs import FirstCenteredDifference, SecondCenteredDifference get_evals_1 = FirstCenteredDifference(h).get_eigenvalues get_evals_2 = SecondCenteredDifference(h).get_eigenvalues if stream: try: derivs = ps.FiniteDifferencer(mpi, h, dx, rank_shape=rank_shape, stream=stream) except: # noqa pytest.skip("StreamingStencil unavailable") else: derivs = ps.FiniteDifferencer(mpi, h, dx, rank_shape=rank_shape) pencil_shape = tuple(ni + 2 * h for ni in rank_shape) # set up test data fx_h = np.empty(pencil_shape, dtype) kvec = np.array(dk) * np.array([-5, 4, -3]).astype(dtype) xvec = np.meshgrid(*[ dxi * np.arange(si, si + ni) for dxi, si, ni in zip(dx, start, rank_shape) ], indexing="ij") phases = sum(ki * xi for ki, xi in zip(kvec, xvec)) if h > 0: fx_h[h:-h, h:-h, h:-h] = np.sin(phases) else: fx_h[:] = np.sin(phases) fx_cos = np.cos(phases) fx = cla.to_device(queue, fx_h) lap = cla.empty(queue, rank_shape, dtype) grd = cla.empty(queue, (3, ) + rank_shape, dtype) derivs(queue, fx=fx, lap=lap, grd=grd) eff_kmag_sq = sum( get_evals_2(kvec_i, dxi) for dxi, kvec_i in zip(dx, kvec)) lap_true = eff_kmag_sq * np.sin(phases) max_rtol = 1e-9 if dtype == np.float64 else 3e-4 avg_rtol = 1e-11 if dtype == np.float64 else 5e-5 # filter small values dominated by round-off error mask = np.abs(lap_true) > 1e-11 max_err, avg_err = get_errs(lap_true[mask], lap.get()[mask]) assert max_err < max_rtol and avg_err < avg_rtol, \ f"lap inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \ f" {max_err=}, {avg_err=}" for i in range(3): eff_k = get_evals_1(kvec[i], dx[i]) pdi_true = eff_k * fx_cos # filter small values dominated by round-off error mask = np.abs(pdi_true) > 1e-11 max_err, avg_err = get_errs(pdi_true[mask], grd[i].get()[mask]) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pd{i} inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \ f" {max_err=}, {avg_err=}" vec = cla.empty(queue, (3, ) + pencil_shape, dtype) for mu in range(3): vec[mu] = fx div = cla.empty(queue, rank_shape, dtype) derivs.divergence(queue, vec, div) div_true = sum(grd[i] for i in range(3)).get() # filter small values dominated by round-off error mask = np.abs(div_true) > 1e-11 max_err, avg_err = get_errs(div_true[mask], div.get()[mask]) assert max_err < max_rtol and avg_err < avg_rtol, \ f"div inaccurate for {h=}, {grid_shape=}, {proc_shape=}:" \ f" {max_err=}, {avg_err=}" if timing: from common import timer base_args = dict(queue=queue, fx=fx) div_args = dict(queue=queue, vec=vec, div=div) if h == 0: import pyopencl.tools as clt pool = clt.MemoryPool(clt.ImmediateAllocator(queue)) base_args["allocator"] = pool div_args["allocator"] = pool times = {} times["gradient and laplacian"] = timer( lambda: derivs(lap=lap, grd=grd, **base_args)) times["gradient"] = timer(lambda: derivs(grd=grd, **base_args)) times["laplacian"] = timer(lambda: derivs(lap=lap, **base_args)) times["pdx"] = timer(lambda: derivs(pdx=grd[0], **base_args)) times["pdy"] = timer(lambda: derivs(pdy=grd[1], **base_args)) times["pdz"] = timer(lambda: derivs(pdz=grd[2], **base_args)) times["divergence"] = timer(lambda: derivs.divergence(**div_args)) if mpi.rank == 0: print(f"{grid_shape=}, {h=}, {proc_shape=}") for key, val in times.items(): print(f"{key} took {val:.3f} ms")
def test_stencil(ctx_factory, grid_shape, proc_shape, dtype, stream, h=1, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) rank_shape = tuple(Ni // pi for Ni, pi in zip(grid_shape, proc_shape)) from pymbolic import var x = var("x") y = var("y") i, j, k = var("i"), var("j"), var("k") map_dict = {} map_dict[y[i, j, k]] = (x[i + h + h, j + h, k + h] + x[i + h, j + h + h, k + h] + x[i + h, j + h, k + h + h] + x[i - h + h, j + h, k + h] + x[i + h, j - h + h, k + h] + x[i + h, j + h, k - h + h]) if stream: try: stencil_map = ps.StreamingStencil(map_dict, prefetch_args=["x"], halo_shape=h) except: # noqa pytest.skip("StreamingStencil unavailable") else: stencil_map = ps.Stencil(map_dict, h, prefetch_args=["x"]) x = clr.rand(queue, tuple(ni + 2 * h for ni in rank_shape), dtype) y = clr.rand(queue, rank_shape, dtype) x_h = x.get() y_true = (x_h[2 * h:, h:-h, h:-h] + x_h[h:-h, 2 * h:, h:-h] + x_h[h:-h, h:-h, 2 * h:] + x_h[:-2 * h, h:-h, h:-h] + x_h[h:-h, :-2 * h, h:-h] + x_h[h:-h, h:-h, :-2 * h]) stencil_map(queue, x=x, y=y) max_rtol = 5e-14 if dtype == np.float64 else 1e-5 avg_rtol = 5e-14 if dtype == np.float64 else 1e-5 max_err, avg_err = get_errs(y_true, y.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"y innaccurate for {grid_shape=}, {h=}, {proc_shape=}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer t = timer(lambda: stencil_map(queue, x=x, y=y)[0]) print( f"stencil took {t:.3f} ms for {grid_shape=}, {h=}, {proc_shape=}") bandwidth = (x.nbytes + y.nbytes) / 1024**3 / t * 1000 print(f"Bandwidth = {bandwidth} GB/s")
---------- args : dict Configuration dict, typically loaded from YAML file """ tokens(args) embeddings(args) features(args) classifiers(args) if __name__ == '__main__': # Parse arguments and call appropriate method docargs = docopt(__doc__) # Program modes modes = { 'run': run, 'tokens': tokens, 'embeddings': embeddings, 'features': features, 'classifiers': classifiers } f = modes[[mode for mode in modes if docargs[mode]][0]] c = docargs['--config'] with open(c, 'r') as yml: args = yaml.safe_load(yml) with timer("Packet2Vec"): f(args)
def test_vector_projector(ctx_factory, grid_shape, proc_shape, h, dtype, timing=False): if ctx_factory: ctx = ctx_factory() else: ctx = ps.choose_device_and_make_context() queue = cl.CommandQueue(ctx) mpi = ps.DomainDecomposition(proc_shape, h, grid_shape=grid_shape) rank_shape, _ = mpi.get_rank_shape_start(grid_shape) pencil_shape = tuple(ni + 2 * h for ni in rank_shape) L = (10, 8, 11.5) dx = tuple(Li / Ni for Li, Ni in zip(L, grid_shape)) dk = tuple(2 * np.pi / Li for Li in L) fft = ps.DFT(mpi, ctx, queue, grid_shape, dtype) cdtype = fft.cdtype if h > 0: stencil = FirstCenteredDifference(h) project = ps.Projector(fft, stencil.get_eigenvalues, dk, dx) derivs = ps.FiniteDifferencer(mpi, h, dx) else: project = ps.Projector(fft, lambda k, dx: k, dk, dx) derivs = ps.SpectralCollocator(fft, dk) vector_x = cla.empty(queue, (3, ) + pencil_shape, dtype) div = cla.empty(queue, rank_shape, dtype) pdx = cla.empty(queue, (3, ) + rank_shape, dtype) def get_divergence_error(vector): for mu in range(3): fft.idft(vector[mu], vector_x[mu]) derivs.divergence(queue, vector_x, div) derivs(queue, fx=vector_x[0], pdx=pdx[0]) derivs(queue, fx=vector_x[1], pdy=pdx[1]) derivs(queue, fx=vector_x[2], pdz=pdx[2]) norm = sum([clm.fabs(pdx[mu]) for mu in range(3)]) max_err = cla.max(clm.fabs(div)) / cla.max(norm) avg_err = cla.sum(clm.fabs(div)) / cla.sum(norm) return max_err, avg_err max_rtol = 1e-11 if dtype == np.float64 else 1e-4 avg_rtol = 1e-13 if dtype == np.float64 else 1e-5 k_shape = fft.shape(True) vector = cla.empty(queue, (3, ) + k_shape, cdtype) for mu in range(3): vector[mu] = make_data(queue, fft).astype(cdtype) project.transversify(queue, vector) max_err, avg_err = get_divergence_error(vector) assert max_err < max_rtol and avg_err < avg_rtol, \ f"transversify failed for {grid_shape=}, {h=}: {max_err=}, {avg_err=}" plus = make_data(queue, fft).astype(cdtype) minus = make_data(queue, fft).astype(cdtype) project.pol_to_vec(queue, plus, minus, vector) if isinstance(fft, gDFT): assert all(is_hermitian(vector[i]) for i in range(3)), \ f"pol->vec is non-hermitian for {grid_shape=}, {h=}" max_err, avg_err = get_divergence_error(vector) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol_to_vec result not transverse for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" vector_h = vector.get() vector_2 = cla.zeros_like(vector) project.transversify(queue, vector, vector_2) vector_2_h = vector_2.get() max_err, avg_err = get_errs(vector_h, vector_2_h) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->vector != its own transverse proj. for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" plus1 = cla.zeros_like(plus) minus1 = cla.zeros_like(minus) project.vec_to_pol(queue, plus1, minus1, vector) if isinstance(fft, gDFT): assert is_hermitian(plus1) and is_hermitian(minus1), \ f"polarizations aren't hermitian for {grid_shape=}, {h=}" max_err, avg_err = get_errs(plus1.get(), plus.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->vec->pol (plus) is not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" max_err, avg_err = get_errs(minus1.get(), minus.get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"pol->vec->pol (minus) is not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" project.vec_to_pol(queue, vector[0], vector[1], vector) max_err, avg_err = get_errs(plus1.get(), vector[0].get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"in-place pol->vec->pol (plus) not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" max_err, avg_err = get_errs(minus1.get(), vector[1].get()) assert max_err < max_rtol and avg_err < avg_rtol, \ f"in-place pol->vec->pol (minus) not identity for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" # reset and test longitudinal component for mu in range(3): vector[mu] = make_data(queue, fft).astype(cdtype) fft.idft(vector[mu], vector_x[mu]) long = cla.zeros_like(minus) project.decompose_vector(queue, vector, plus1, minus1, long) long_x = cla.empty(queue, pencil_shape, dtype) fft.idft(long, long_x) div_true = cla.empty(queue, rank_shape, dtype) derivs.divergence(queue, vector_x, div_true) derivs(queue, fx=long_x, grd=pdx) div_long = cla.empty(queue, rank_shape, dtype) if h != 0: pdx_h = cla.empty(queue, (3, ) + pencil_shape, dtype) for mu in range(3): mpi.restore_halos(queue, pdx[mu], pdx_h[mu]) derivs.divergence(queue, pdx_h, div_long) else: derivs.divergence(queue, pdx, div_long) max_err, avg_err = get_errs(div_true.get(), div_long.get()) assert max_err < 1e-6 and avg_err < 1e-11, \ f"lap(longitudinal) != div vector for {grid_shape=}, {h=}" \ f": {max_err=}, {avg_err=}" if timing: from common import timer ntime = 10 t = timer(lambda: project.transversify(queue, vector), ntime=ntime) print(f"transversify took {t:.3f} ms for {grid_shape=}") t = timer(lambda: project.pol_to_vec(queue, plus, minus, vector), ntime=ntime) print(f"pol_to_vec took {t:.3f} ms for {grid_shape=}") t = timer(lambda: project.vec_to_pol(queue, plus, minus, vector), ntime=ntime) print(f"vec_to_pol took {t:.3f} ms for {grid_shape=}") t = timer( lambda: project.decompose_vector(queue, vector, plus, minus, long), ntime=ntime) print(f"decompose_vector took {t:.3f} ms for {grid_shape=}")
def main(): ## Parameters ## xSet = "dist_days_time_dayOfWeek" data = cmn.dataset(xSet = xSet) data.load(Nparts = 10, N_points = 4000) futureMask = makeFutureMask(data.tScope, data.tTest, futureTime = -30 * 60) print "Model\t{}\t{}\t{}".format("Param", "RunTime", "RMSE") # Predict on time timer = cmn.timer() yHat = onTime.regress(data.xScope, data.yScope, data.xTest, 1, futureMask) time_onTime = timer.dur() rmse_onTime = cmn.rmse(data.yTest, yHat) print "onTime\t\t{:.2f}\t{:.2f}".format(time_onTime, rmse_onTime) data.saveYHat(yHat, model = "onTime") # Visualize and save the images for the model data.visualize(yHat, "onTime") ## kNN # Try many values of k ks = np.ceil(2 ** (np.arange(15) / 1.5)) rmse_kNN = np.zeros(shape=(len(ks)), dtype=np.float) time_kNN = np.zeros(shape=(len(ks)), dtype=np.float) for i in range(len(ks)): k = ks[i] timer = cmn.timer() yHat = kNN.regress(data.xScope, data.yScope, data.xTest, k, futureMask) time_kNN[i] = timer.dur() rmse_kNN[i] = cmn.rmse(data.yTest, yHat) print "kNN\t{: 5.0f}\t{:.2f}\t{:.2f}".format(k, time_kNN[i], rmse_kNN[i]) data.saveYHat(yHat, model = "{}NN".format(k)) # Visualize and save the images for the model data.visualize(yHat, "{}NN".format(k)) # Plot the historical RMSE clf() plot(ks, rmse_kNN) xlabel("Number of nearest points, k in kNN") ylabel("Root Mean Squared Error (seconds)") title("kNN Model, RMSE for different ks") savefig("{}/{}_{}_k-rmse.png".format(data.figPath, data.serviceName, data.routeName)) ## Kernel # Try many values of k rhos = np.ceil(10 ** (np.arange(12) / 2.0)) rmse_kernel = np.zeros(shape=(len(rhos)), dtype=np.float) time_kernel = np.zeros(shape=(len(rhos)), dtype=np.float) for i in range(len(rhos)): rho = rhos[i] timer = cmn.timer() yHat = kernel.regress(data.xScope, data.yScope, data.xTest, rho, futureMask) time_kernel[i] = timer.dur() rmse_kernel[i] = cmn.rmse(data.yTest, yHat) print "kernel\t{: 5.0f}\t{:.2f}\t{:.2f}".format(rho, time_kernel[i], rmse_kernel[i]) data.saveYHat(yHat, model = "kernel_{}rho".format(rho)) # Visualize and save the images for the model data.visualize(yHat, "kernel_{}rho".format(rho)) # Plot the historical RMSE clf() plot(rhos, rmse_kernel) xlabel("rho Paramater") ylabel("Root Mean Squared Error (seconds)") title("Kernel Regression Model, RMSE for different rhos") savefig("{}/{}_{}_kernel_rho-rmse.png".format(data.figPath, data.serviceName, data.routeName))