def _test_Distribution(display=False): # Verify that the distribution works under a weighted sum. import numpy as np d = [] count = 3 scale = 2**(-20) scale = 2**(30) for i in range(count): pts = np.random.random(20) * np.random.random() * scale d.append(cdf_fit(pts, fit="cubic")) wts = np.random.random((count, )) wts /= sum(wts) min_max = (-scale / 3, scale + scale / 3) if display: print(sum(dist * w for (dist, w) in zip(d, wts))) from util.plot import Plot p = Plot("Weighted cubic fit") out = sum(dist * w for (dist, w) in zip(d, wts)) p.add_func("Weighted sum", out, min_max) for i, (dist, w) in enumerate(zip(d, wts)): p.add_func(f"Dist {i+1} -- {round(w,3)}", dist, min_max, opacity=.3) p.show()
def _test_cdf_fit(): import numpy as np from util.math import SMALL from util.random import cdf from util.plot import Plot n = 10000 f = cdf(nodes=5, fit="linear") sample = f.inverse(np.random.random((n, ))) g = cdf_fit(sample, fit=None) print("Fit first CDF point: ", g.nodes[0]) print("Fit second CDF point:", g.nodes[1]) print("Fit last CDF point: ", g.nodes[-1]) print("Expected max error:", samples(n, confidence=.99)) print("Actual max error: ", f - g) min_max = (f.min - SMALL, f.max + SMALL) p = Plot() p.add_func("Truth", f, min_max) p.add_func("EDF", g, min_max) p.show(show=False, height=700, width=800) p = Plot() p.add_func("Truth", f.inverse, min_max) p.add_func("EDF", g.inverse, min_max) p.show(append=True, height=700, width=800)
def test_random_cdf(display=True): if not display: return from util.plot import Plot p = Plot("") for nodes in range(1, 100): f = cdf() p.add_func(f"Random PDF {nodes}", f.derivative, f(), color=p.color(nodes - 1), group=nodes) # p.add(f"Points {nodes}", *list(zip(*f.nodes)), # color=p.color(nodes-1,alpha=.3), group=nodes) p.show(show=False) print() p = Plot("") for nodes in range(1, 30): f = cdf(nodes=nodes) p.add_func(f"Random {nodes} node CDF", f, f(), color=p.color(nodes - 1), group=nodes) p.add(f"Points {nodes}", *list(zip(*f.nodes)), color=p.color(nodes - 1, alpha=.3), group=nodes) p.show(append=True)
def _test_mesh(show=True): from weakly_admissable_meshes import polar_wam, box_wam from util.plot import Plot p = Plot("Weakly admissable mesh") p.add("Polar", *(polar_wam(8).T), color=p.color(0)) p.add("Box", *(box_wam(8).T), color=p.color(1)) if show: p.show(width=1000*.7, height=900*.7, append=True)
def rank_by_slope(components, points, values, metric, max_pairs=10000, display=True): # Compute the magnitudes using average metric slope. if display: print(" computing average metric slope per component.. ", end="", flush=True) avg_slope = np.zeros(len(components)) update = end = "" for i in range(len(components)): update = "\b" * len(end) end = f"{i+1} of {len(components)}" if display: print(update, end=end, flush=True) x = np.matmul(points, components[i]) for (p1, p2) in gen_random_pairs(len(x), count=max_pairs): avg_slope[i] += metric(values[p1], values[p2]) / abs(x[p1] - x[p2]) print("Results so far:") from util.plot import Plot p = Plot() for (comp, slope) in zip(components, avg_slope): print(comp, slope) x, y = np.matmul(points, comp), values p.add(f"Points: {slope:.2f}", x, y) print() p.show() exit() # Invert the total average metric slope. if (min(avg_slope) <= 0.0): avg_slope = np.where(avg_slope == 0, 1., 0.) else: avg_slope = 1 / avg_slope avg_slope /= np.sum(avg_slope) # Re-order the components according to inverse average metric slope. order = np.argsort(avg_slope)[::-1] # If they are not already ordered correctly, re-order the returns. if not all(order[i] < order[i + 1] for i in range(len(order) - 1)): if display: print(" reordering components by average metric slope..", end="\r", flush=True) components, avg_slope = components[order], avg_slope[order] if display: print(" ", end="\r", flush=True) return components, avg_slope
def test_latin(display=False): from util.random import latin print("Testing latin..", end=" ") if display: print() if display: from util.plot import Plot D = 2 N = 400 p = Plot("Latin hyper cube design") # Add grid lines for the cube. for i in range(N + 1): p.add(f"gly {i}", [0, 1], [i / N] * 2, mode="lines", group="grid", color="rgba(.6,.1,.1,.2)", show_in_legend=(i == 0)) p.add(f"glx {i}", [i / N] * 2, [0, 1], mode="lines", group="grid", color="rgba(.6,.1,.1,.2)", show_in_legend=False) # Add the random points. p.add("Points", *(latin(N, D).T)) p.show(file_name="/Users/thomaslux/Desktop/lhc.html") from numpy import sort N = 10000 D = 1000 pts = latin(N, D) for i in range(D): values = sort(pts[:, i]) max_adj_diff = max(abs(values[:-1] - values[1:])) if max_adj_diff >= (2 / N): print("BAD!", " ", i, max_adj_diff, 1 / N, 2 / N) assert (max_adj_diff < (2 / N)) print("passed.")
def _test_fekete(max_n=2000, max_d=1000, start_n=1000, start_d=10, steps=2, show=True): if show: # Generate some points in 3 dimensions for showing. pts = fekete_points(47, 3) from util.plot import Plot p = Plot("Fekete") p.add("fekete points", *(pts.T)) p.show(append=True) from util.system import Timer t = Timer() steps -= 1 for n in range(start_n,max_n+1, (max_n-start_n-1) // steps): for d in range(start_d, max_d+1, (max_d-start_d-1) // steps): t.start() print() print('-'*70) print(f" {n}, {d}", flush=True) pts = fekete_points(n, d) print(f" {pts.shape}") print(f" {t()} seconds", flush=True) t.stop() print() print()
steps = 50 update_p("Begin") for step in range(steps): update_vecs() update_p(str(step) + " updated") reorder_vecs() update_p(str(step) + " reordered") orthogonalize_vecs() update_p(str(step) + " orthogonalized") if np.linalg.norm(previous_vecs - vecs) <= 0: break else: previous_vecs = vecs.copy() update_p("End") p.show(x_range=(-4.5, 4.5), y_range=(-2, 2), width=1000, height=600, show_legend=False, file_name="test.html") print() print(vecs) print(value_vecs()) print() vecs[:, :] = np.diag(np.ones(2))[:, :] print(vecs) print(value_vecs()) print() from util.stats import ca a, b = ca(points) vecs[:, :] = a[:, :]
def _test_samples(display=True, test_correctness=False): from util.math import Fraction from util.plot import Plot if display: for size in tuple(range(2, 42)) + (128, 129, 256, 257): p = Plot(f"Error at x with {size} samples") for confidence in (Fraction(9, 10), Fraction(185, 200), Fraction(95, 100), Fraction(97, 100), Fraction(99, 100)): f = lambda x: samples( size=size, confidence=confidence, at=x[0]) p.add_func(f"{confidence} confidence", f, [0, 1]) p.show(append=True, show=(size == 2)) exit() if display: print() print("-" * 70) print("Begin tests for 'samples'") print() key_values = [ (11, Fraction(3, 10), Fraction(95, 100)), (25, Fraction(2, 10), Fraction(95, 100)), (97, Fraction(1, 10), Fraction(95, 100)), (385, Fraction(5, 100), Fraction(95, 100)), (9604, Fraction(1, 100), Fraction(95, 100)), (19, Fraction(3, 10), Fraction(99, 100)), (42, Fraction(2, 10), Fraction(99, 100)), (166, Fraction(1, 10), Fraction(99, 100)), (664, Fraction(5, 100), Fraction(99, 100)), (16588, Fraction(1, 100), Fraction(99, 100)), (33733, Fraction(1, 10), Fraction(999, 1000)), (134930, Fraction(5, 100), Fraction(999, 1000)), (3373242, Fraction(1, 100), Fraction(999, 1000)), ] if display: print("samples (max error, confidence)") for (s, e, c) in key_values[:-3]: needed = samples(error=e, confidence=c) if display: print("needed: ", needed) print("%6d (%2.0f%%, %2.0f%%)" % (needed, 100 * e, 100 * c)) # if (s != None): assert(needed == s) if display: print() for n in (10, 25, 90, 350, 9000): print( f"With {n:4d} samples we are 95% confident in max CDF error <=", round(samples(n, confidence=.95), 1 if n < 350 else 2)) print() for n in (20, 40, 160, 660, 16000): print( f"With {n:5d} samples we are 99% confident in max CDF error <=", round(samples(n, confidence=.99), 1 if n < 600 else 2)) print("-" * 70) if test_correctness: # Generate a random CDF from util import random from util.plot import Plot TESTS = 10000 DIFFS = 100 N = 100 fit = "linear" truth = random.cdf(nodes=5, fit=fit) max_error = samples(N, confidence=.99) if display: print("Largest expected error:", max_error) mid_error = [] errors = {(1 / 100): [], (1 / 4): [], (1 / 3): [], (1 / 2): []} max_errors = [] mean_failed = [] for i in range(TESTS): sample = truth.inverse(np.random.random((N, ))) guess = cdf_fit(sample, fit=fit) diff = truth - guess diff_func = lambda x: abs(truth(x) - guess(x)) diffs = diff_func(np.linspace(0, 1, DIFFS)) mean_failed += [sum(diffs > max_error)] if display: print( f"Failed: {mean_failed[-1]:4d} {sum(mean_failed)/len(mean_failed):.0f}" ) max_errors.append(diff) for v in errors: errors[v].append(truth(v) - guess(v)) # if (diff > max_error): # print(i, N, diff, max_error) # p = Plot() # p.add_func("Truth", truth, truth()) # p.add_func("Guess", guess, guess()) # p.add_func("Error", diff_func, truth()) # p.show(show=False) # p = Plot() # p.add_func("Truth Inverse", truth.inverse, (0.,1.)) # p.add_func("Guess Inverse", guess.inverse, (0.,1.)) # p.show(show=False, append=True) # break total_failed = sum(e > max_error for e in max_errors) if display or (total_failed > 0): print( f"Failed {total_failed} out of {TESTS}, or {100*total_failed/TESTS:.1f}%." ) p = Plot() # Add the distribution of maximum errors. f = cdf_fit(max_errors) p.add_func(f"{len(max_errors)} max errors", f, f()) # Add the distribution of errors at different values. for v in sorted(errors)[::-1]: mean = np.mean(errors[v]) std = np.std(errors[v]) f = cdf_fit(errors[v]) p.add_func( f"{v:.1f} errors ({mean:.1e}, {std:.1e}) {samples(N,confidence=.99):.2f}", f, f()) p.show(append=True) p = Plot() p.add_func("Truth", truth, truth()) p.show(append=True)
def _test_fit_funcs(display=False): if display: print() print("-" * 70) print("Begin tests for 'fit_funcs'") from util.plot import Plot # ============================================== # Test the fit functions and smoothing # ============================================== # Make data smooth = .1 data = np.random.normal(size=(1000, )) # data[:len(data)//2] += 2 min_max = (min(data) - .1, max(data) + .1) if display: print() print("(min, max) : (%.2f, %.2f)" % (min_max)) print("Normal confidence: %.2f%%" % (100 * normal_confidence(data))) print() # Make PDF fits pfit = pdf_fit(data) smooth_pfit = pdf_fit(data, smooth=smooth) # Make CDF fits cfit = cdf_fit(data) stdev = .05 * (cfit.max - cfit.min) smooth_cfit = gauss_smooth(cfit, stdev) stdev = smooth * (cfit.max - cfit.min) smoother_cfit = gauss_smooth(cfit, stdev) # Make PDF plots p = Plot() p.add_func("PDF", pfit, min_max) # Make smooth PDF p.add_func("Smooth PDF", smooth_pfit, min_max) if display: p.show(show=False) # Make CDF plots p = Plot() p.add_func("CDF", cfit, min_max) # Make CDF whose derivative is the default PDF. p.add_func("CDF for default PDF", smooth_cfit, min_max) # Make smoother cdf. p.add_func("Smooth CDF", smoother_cfit, min_max) if display: p.show(append=True) # Make an animation transitioning between two normal distributions. np.random.seed(0) d1 = np.random.normal(0, .5, size=(500, )) d2 = np.random.normal(3, 1, size=(500, )) f1 = cdf_fit(d1, smooth=.1) f2 = cdf_fit(d2, smooth=.1) p = Plot() for w in np.linspace(0, 1, 21): w = round(w, 2) f3 = w * f1 + (1 - w) * f2 p.add_func("0, 1/2", f1, f1(), frame=w) p.add_func("3, 1", f2, f2(), frame=w) p.add_func("weighted sum", f3, f3(), frame=w) if display: p.show(bounce=True, append=True) if display: print() if display: print("-" * 70)
def _test_epdf_diff(display=False): if display: print() print("-" * 70) print("Begin tests for 'epdf_diff'") # ---------------------------------------------------------------- def demo(seq): if display: print('~' * 70) print(len(seq), seq) print() total = 0 for vals in edf_pair_gen(seq): total += vals[-1] if display: print("[% 4s, % 3s] (%.2f) --" % vals, round(total, 3)) if display: print('~' * 70) print() demo([0] + list(range(9))) demo(sorted(np.random.random(size=(10, )))) demo(list(range(9)) + [8]) # ---------------------------------------------------------------- # a = [1, 1, 3, 3, 5, 6] # b = [0, 1, 2, 3, 4] # n = 100 if display: print(f"a = [0,100] ({n} samples)") print(f"b = [v + d for v in a]") print() for d in (.0, .01, .1, .5, .55, .9, 1., 1.5): a = [v / n for v in list(range(n + 1))] b = [v + d for v in a] if display: print( f"d = {d:.2f} a~b = {epdf_diff(a,b):.2f} b~a = {epdf_diff(b,a):.2f} a~a = {epdf_diff(a,a):.2f} b~b = {epdf_diff(b,b):.2f}" ) if display: print() for d in (.0, .01, .1, .5, .55, .9, 1., 1.5): # Generate a random sequence. a = sorted((np.random.random(size=(10, )))) b = sorted((np.random.random(size=(1000, )) + d)) diff = epdf_diff(a, b) if display: print(f"d = {d:.2f}", "", "[%.2f, %.2f]" % (min(a), max(a)), "[%.2f, %.2f]" % (min(b), max(b)), "", diff) if display: print() # ---------------------------------------------------------------- from util.plot import Plot # Generate a random sequence. a = sorted((np.random.random(size=(2000, )))) b = sorted((np.random.random(size=(2000, )))) p = Plot("Empirical PDF Diff Test") p1 = pdf_fit(a, smooth=0.00001) p2 = pdf_fit(b, smooth=0.00001) p.add_func("a", p1, p1()) #(-.5,2)) p.add_func("b", p2, p2()) #(-.5,2)) if display: p.show(show=False, y_range=[-.5, 1.5]) p = Plot("Empirical CDF Diff Test") p1 = cdf_fit(a) p2 = cdf_fit(b) p.add_func("a", p1, p1()) #(-.5,2)) p.add_func("b", p2, p2()) #(-.5,2)) if display: p.show(append=True) # ---------------------------------------------------------------- if display: print("-" * 70)
# ============================================== # Display the boxes that were computed # ============================================== p = Plot() # Get the extreme points. min_x = np.min(x[:,0]) - .1 max_x = np.max(x[:,0]) + .1 min_y = np.min(x[:,1]) - .1 max_y = np.max(x[:,1]) + .1 # Get the box edges (about the centers). boxes = model.box_sizes.T boxes[:,0] = np.where(boxes[:,0] != -1, boxes[:,0], min_x) boxes[:,1] = np.where(boxes[:,1] != -1, boxes[:,1], min_y) boxes[:,2] = np.where(boxes[:,2] != -1, boxes[:,2], max_x) boxes[:,3] = np.where(boxes[:,3] != -1, boxes[:,3], max_y) # Add boxes to plot. for i,(pt,bounds) in enumerate(zip(x, boxes)): shifts = np.array([[- boxes[i,0], - boxes[i, 1]], [ boxes[i,2], - boxes[i, 1]], [ boxes[i,2], boxes[i, 3]], [- boxes[i,0], boxes[i, 3]], [- boxes[i,0], - boxes[i, 1]] ]) box = (pt + shifts) p.add(f"Box {i+1}", *(box.T), color=p.color(i), mode="lines") for i,(pt,bounds) in enumerate(zip(x, boxes)): p.add("", [pt[0]], [pt[1]], color=p.color(i), show_in_legend=False) p.show(append=True) exit()
print(d) config_cols = ["Function", "SNR", "Train"] configs = d[:, config_cols].unique() configs.sort() configs.max_display = float('inf') print(configs) for conf in configs: interest = d[d[:, config_cols] == conf] interest.sort() print(interest) # exit() F, S, N = conf p = Plot(f"{N} train with SNR {S} on '{F}' function") seen = {} for row in interest: name = row["Algorithm"] # Skip algorithms that have already been plotted. if name in seen: print(f" skipping extra '{name}'") continue else: seen.add(name) # Plot the performance of this algorithm. fit_time = row["Fit Time"] pred_time = row["Predict Time"] errors = row["Abs Errors"] # f = cdf_fit(errors) # p.add_func(name, f, f()) p.add_box(name, errors) p.show(append=True, show=all(conf == configs[-1]))
def modes(data, confidence=.99, tol=1/1000): from util.optimize import zero num_samples = len(data) error = 2*samples(num_samples, confidence=confidence) cdf = cdf_fit(data, fit="cubic") print("error: ",error) # Find all of the zeros of the derivative (mode centers / dividers) checks = np.linspace(cdf.min, cdf.max, np.ceil(1/tol)) second_deriv = cdf.derivative.derivative deriv_evals = second_deriv(checks) modes = [i for i in range(1, len(deriv_evals)) if (deriv_evals[i-1] * deriv_evals[i] <= 0) and (deriv_evals[i-1] >= deriv_evals[i])] antimodes = [i for i in range(1, len(deriv_evals)) if (deriv_evals[i-1] * deriv_evals[i] <= 0) and (deriv_evals[i-1] < deriv_evals[i])] # Compute exact modes and antimodes using a zero-finding function. modes = [zero(second_deriv, checks[i-1], checks[i]) for i in modes] antimodes = [zero(second_deriv, checks[i-1], checks[i]) for i in antimodes] original_antimodes = antimodes[:] # Fix the bounds of the antimodes to match the distribution. if modes[0] < antimodes[0]: antimodes = [cdf.min] + antimodes else: antimodes[0] = cdf.min if modes[-1] > antimodes[-1]: antimodes += [cdf.max] else: antimodes[-1] = cdf.max # Make sure that there is an antimode between each mode. for i in range(len(modes)): if antimodes[i] > modes[i]: # Update the next antimode with this one (as long as it's not the max). if (i < len(modes)-1): antimodes[i+1] = (antimodes[i] + antimodes[i+1]) / 2 # Always update this antimode to properly be LESS than the mode. antimodes[i] = (modes[i] + modes[i-1]) / 2 print("len(modes): ",len(modes)) print("len(antimodes): ",len(antimodes)) # Define a function that counts the number of modes thta are too small. def count_too_small(): return sum( (cdf(upp) - cdf(low)) < error for (low,upp) in zip(antimodes[:-1],antimodes[1:]) ) # Show PDF from util.plot import Plot p = Plot() pdf = pdf_fit(cdf.inverse(np.random.random((1000,)))) # Loop until all modes are big enough to be accepted given error tolerance. step = 1 while count_too_small() > 0: print() print("step: ",step, (len(modes), len(antimodes))) f = len(modes) p.add_func("PDF", pdf, cdf(), color=p.color(1), frame=f, show_in_legend=(step==1)) # Generate the mode lines. mode_lines = [[],[]] for z in modes: mode_lines[0] += [z,z,None] mode_lines[1] += [0,.2,None] p.add("modes", *mode_lines, color=p.color(0), mode="lines", group="modes", show_in_legend=(z==modes[0] and step==1), frame=f) # Generate the antimode lines. anti_lines = [[],[]] for z in antimodes: anti_lines[0] += [z,z,None] anti_lines[1] += [0,.2,None] p.add("seperator", *anti_lines, color=p.color(3,alpha=.3), mode="lines", group="seperator", show_in_legend=(z==antimodes[0] and (step==1)), frame=f) step += 1 # Compute the densities and the sizes of each mode. sizes = [cdf(antimodes[i+1]) - cdf(antimodes[i]) for i in range(len(modes))] densities = [(cdf(antimodes[i+1]) - cdf(antimodes[i])) / (antimodes[i+1] - antimodes[i]) for i in range(len(modes))] # Compute those modes that have neighbors that are too small. to_grow = [i for i in range(len(modes)) if (i > 0 and sizes[i-1] < error) or (i < len(sizes)-1 and sizes[i+1] < error)] if len(to_grow) == 0: break print("modes: ",modes) print("antimodes: ",antimodes) print("sizes: ",sizes) print("densities: ",densities) print("to_grow: ",to_grow) # Sort the modes to be grown by their size, largest first. to_grow = sorted(to_grow, key=lambda i: -densities[i]) # Keep track of the modes that have already been absorbed. preference = {} taken = set() conflicts = set() modes_to_remove = [] anti_to_remove = [] while len(to_grow) > 0: i = to_grow.pop(0) # Pick which of the adjacent nodes to absorb. to_absorb = None if (i < len(modes)-1) and (sizes[i+1] < error): direction = 1 to_absorb = i + 1 if (i > 0) and (sizes[i-1] < error): # If there wasn't a right mode, take the left by default. if (to_absorb == None): direction = -1 to_absorb = i - 1 # Otherwise we have to pick based on the density similarity. elif (abs(modes[i-1]-modes[i]) < abs(modes[i+1]-modes[i])): # Take the other one if its density is more similar. direction = -1 to_absorb = i - 1 # If there is no good option to absorb, the skip. if (to_absorb in preference): continue # Record the preferred pick of this mode. preference[i] = (direction, to_absorb) # If this mode is already absorbed, then add it to conflict list. if to_absorb in taken: conflicts.add( to_absorb ) # Remove the ability to 'absorb' from modes getting absorbed. if to_absorb in to_grow: to_grow.remove(to_absorb) # Add the absorbed value to the set of "taken" modes. taken.add(to_absorb) # Resolve conflicts by giving absorbed modes to closer modes. for i in sorted(conflicts, key=lambda i: -densities[i]): if (abs(modes[i-1] - modes[i]) < abs(modes[i+1] - modes[i])): preference.pop(i+1) else: preference.pop(i-1) # Update the boundaries for i in sorted(preference, key=lambda i: -densities[i]): direction, to_absorb = preference[i] # Update the boundary of this mode. antimodes[i+(direction>0)] = antimodes[to_absorb + (direction>0)] # Update the "to_remove" lists. anti_to_remove.append( antimodes[to_absorb + (direction>0)] ) modes_to_remove.append( modes[to_absorb] ) # Remove the modes and antimodes that were merged. for m in modes_to_remove: modes.remove(m) for a in anti_to_remove: antimodes.remove(a) # Update the remaining antimodes to be nearest to the middle # of the remaining modes (making them representative dividers). for i in range(len(modes)-1): middle = (modes[i] + modes[i+1]) / 2 closest = np.argmin([abs(oam - middle) for oam in original_antimodes]) antimodes[i+1] = original_antimodes[closest] f = len(modes) p.add_func("PDF", pdf, cdf(), color=p.color(1), frame=f, show_in_legend=(step==1)) # Generate the mode lines. mode_lines = [[],[]] for z in modes: mode_lines[0] += [z,z,None] mode_lines[1] += [0,.2,None] p.add("modes", *mode_lines, color=p.color(0), mode="lines", group="modes", show_in_legend=(z==modes[0] and step==1), frame=f) # Generate the antimode lines. anti_lines = [[],[]] for z in antimodes: anti_lines[0] += [z,z,None] anti_lines[1] += [0,.2,None] p.add("seperator", *anti_lines, color=p.color(3,alpha=.3), mode="lines", group="seperator", show_in_legend=(z==antimodes[0] and (step==1)), frame=f) p.show(append=True, y_range=[0,.15]) p = Plot() p.add_func("CDF", cdf, cdf(), color=p.color(1)) for z in modes: p.add("modes", [z,z], [0,1], color=p.color(0), mode="lines", group="modes", show_in_legend=(z==modes[0])) for z in antimodes: p.add("seperator", [z,z], [0,1], color=p.color(3), mode="lines", group="sep", show_in_legend=(z==antimodes[0])) p.show(append=True)
test_x = test_mat[:, out_idxs] m = Voronoi() m.fit(train_x) lengths += [len(ids) for ids, wts in m(test_x)] save(lengths) print("len(lengths): ", len(lengths)) from util.plot import Plot from util.stats import cdf_fit p = Plot("Distribution of Number of Influencers", "Number of records used to make prediction", "CDF value") cdf = cdf_fit(lengths) p.add_func("lengths", cdf, cdf(), show_in_legend=False, color=p.color(1)) p.show() exit() from util.approximate import Voronoi, NearestNeighbor, NeuralNetwork, DecisionTree from util.approximate import ShepMod, BoxMesh, LSHEP, Delaunay from util.approximate.testing import test_plot # model = NearestNeighbor() # model = Voronoi() # model = DecisionTree() # model = NeuralNetwork() # model = BoxMesh() # model = LSHEP() # model = ShepMod()
def modes(data, confidence=.99, tol=1/1000): from util.optimize import zero num_samples = len(data) error = 2*samples(num_samples, confidence=confidence) print() print("Smallest allowed mode: ",error) print() # Get the CDF points (known to be true based on data). x, y = cdf_points(data) cdf = cdf_fit((x,y), fit="linear") x, y = x[1:], y[1:] # Generate the candidate break points based on linear interpolation. slopes = [(y[i+1] - y[i]) / (x[i+1] - x[i]) for i in range(len(x)-1)] candidates = [i for i in range(1,len(slopes)-1) if (slopes[i] < slopes[i-1]) and (slopes[i] < slopes[i+1])] # Sort candidates by their 'width', the widest being the obvious divisors. candidates = sorted(candidates, key=lambda i: -(x[i+1] - x[i])) print("candidates: ",candidates) print("slopes: ",[slopes[c] for c in candidates]) # Break the data at candidates as much as can be done with confidence. breaks = [min(x), max(x)] sizes = [1.] chosen = [] print() print("breaks: ",breaks) print("sizes: ",sizes) print("chosen: ",chosen) print() # Loop until there are no candidate break points left. while len(candidates) > 0: new_break_idx = candidates.pop(0) new_break = (x[new_break_idx + 1] + x[new_break_idx]) / 2 b_idx = np.searchsorted(breaks, new_break, side="right") # Compute the CDF values at the upper, break, and lower positions. upp = cdf(breaks[b_idx+1]) if (b_idx < len(breaks)-1) else cdf.max mid = cdf(new_break) low = cdf(breaks[b_idx-1]) print() print("new_break: ", new_break) print("b_idx: ",b_idx) print(" upp: ",upp, upp - mid) print(" mid: ",mid) print(" low: ",low, mid - low) # Compute the size of the smallest mode resulting from the break. smallest_result = min(upp - mid, mid - low) # Skip the break if it makes a mode smaller than error. if smallest_result < error: continue print() print("Num_modes: ", len(sizes)) print("breaks: ", breaks) print("sizes: ", sizes) print() # Update the "breaks" and "sizes" lists. breaks.insert(b_idx, new_break) sizes.insert(b_idx, upp - mid) sizes[b_idx-1] = mid - low chosen.append(new_break_idx) # From the "breaks" and "sizes", construct a list of "modes". # Consider the most dense point between breaks the "mode". modes = [] for i in range(len(chosen)): low = 0 if (i == 0) else chosen[i-1] upp = len(slopes)-1 if (i == len(chosen)-1) else chosen[i+1] mode_idx = low + np.argmax(slopes[low:upp]) modes.append( (x[mode_idx+1] + x[mode_idx]) / 2 ) from util.plot import Plot p = Plot() pdf = pdf_fit(data) p.add_func("PDF", pdf, pdf(), color=p.color(1)) # Generate the mode lines. mode_lines = [[],[]] for z in modes: mode_lines[0] += [z,z,None] mode_lines[1] += [0,.2,None] p.add("modes", *mode_lines, color=p.color(0), mode="lines", group="modes") # Generate the antimode lines. break_lines = [[],[]] for z in breaks: break_lines[0] += [z,z,None] break_lines[1] += [0,.2,None] p.add("seperator", *break_lines, color=p.color(3,alpha=.3), mode="lines", group="seperator") p.show() # Show CDF p = Plot() pdf = pdf_fit(data) p.add_func("CDF", cdf, cdf(), color=p.color(1)) # Generate the mode lines. mode_lines = [[],[]] for z in modes: mode_lines[0] += [z,z,None] mode_lines[1] += [0,1,None] p.add("modes", *mode_lines, color=p.color(0), mode="lines", group="modes") # Generate the antimode lines. break_lines = [[],[]] for z in breaks: break_lines[0] += [z,z,None] break_lines[1] += [0,1,None] p.add("seperator", *break_lines, color=p.color(3,alpha=.3), mode="lines", group="seperator") p.show(append=True)