def get_valley(target, sector_count, sector_angle, threshold, vision_angle, data): hist = histogram.Histogram(sector_count, threshold, vision_angle, data) # TODO histogram and parameters hist.plot_histogram() valley_start = None best_valley = [None, None] # the furthest a valley could every be from is 360 degrees since this is a circle best_distance = 361 for i in range(len(hist.sectors)): if valley_start is None: if hist.sectors[i] < threshold: valley_start = i elif hist.sectors[i] > threshold: dist = get_target_distance(valley_start, i, target, sector_angle) if dist < best_distance: best_distance = dist best_valley = [valley_start, i] valley_start = None if valley_start is not None: dist = get_target_distance(valley_start, sector_count, target, sector_angle) if dist < best_distance: best_valley = [valley_start, i] return best_valley
def testCustomFormatter(self): expected_str = ('Yes |################ | 5000 (4.8 KiB) (83.3%)\n' 'No |### | 1000 (16.6%)') actual_str = str( histogram.Histogram([('Yes', 5000), ('No', 1000)], formatter=self.AddHumanReadableSize)) self.assertEqual(actual_str, expected_str)
def __init__(self, segment_width, segment_height, bin_threshold=225, histogram_threshold=0): sg.Segmentation.__init__(self, segment_width, segment_height, bin_threshold) self.histogram = histogram.Histogram(histogram_threshold)
def transitionProbability(self): for file in self.dataFiles: temp_hist = hst.Histogram(self.timerange, self.situationList) temp_hist.callingSupport([file]) aggregatedHist = temp_hist.aggregate(self.mapping_macros_situation, self.macros_number) situation = [0] + hst.resolve(aggregatedHist) for timestamp in range(1, self.timerange): sList = situation[timestamp - 1:timestamp + 1] self.transitionTable[timestamp][sList[0]][sList[1]] += 1 self.updateTransitionProbability()
def histo(self, trial, n, gated=True, zoom_peak=False): def peak_test(b): if b < 3000: return True return False def always_false(b): return False dt = self.data.intervals[trial] dtp = dt[np.where(dt < 4000)] return hist.Histogram(dt if not zoom_peak else dtp, n, peak_test if gated else always_false)
def __init__(self, identifier, grid_array, run_detect=True, threshold=None, blob_detect=BlobDetectionTypes.DEFAULT, image_color_logic="norm", center=None, radius=None): CellItem.__init__(self, identifier, grid_array) self.threshold = threshold if not isinstance(blob_detect, BlobDetectionTypes): try: blob_detect = BlobDetectionTypes[blob_detect.upper()] except KeyError: blob_detect = BlobDetectionTypes.DEFAULT if blob_detect is BlobDetectionTypes.THRESHOLD: self.detect_function = self.threshold elif blob_detect is BlobDetectionTypes.ITERATIVE: self.detect_function = self.iterative_threshold_detect else: self.detect_function = self.default_detect self.old_trash = None self.trash_array = None self.image_color_logic = image_color_logic self._features_key_list += [MEASURES.Centroid, MEASURES.Perimeter] self.features.shape = (len(self._features_key_list), ) self.histogram = histogram.Histogram(self.grid_array, run_at_init=False) if run_detect: if center is not None and radius is not None: self.manual_detect(center, radius) else: self.detect_function() self._debug_ticker = 0
def plot_bin_of_bins(self, trial): b = bn.Binned(self, trial) h = hist.Histogram(b.bin_count, b.bin_count.max(), lambda x: False) plt.plot(h.bin_centers, h.bin_counts, drawstyle='steps-mid', color='k') mean = np.mean(h.dt) x_range = np.arange(h.bin_centers.min(), h.bin_centers.max(), 0.1) def poisson(xx): return np.exp(-1. * mean) * np.power(mean, xx) / scifn.gamma(xx + 1) plt.plot(x_range, (h.bw * h.dt.size * np.array([poisson(x) for x in x_range])), '--', color='k') plt.xlabel("Counts per Bin") plt.ylabel("Frequency") plt.legend(['Data', 'Poisson Model']) plt.title("Histogram of events per time bin") plt.show()
def __init__(self, width, modelFile, dataFiles): self.width = width self.timerange = int(1440 / width) self.transitionTable = [] self.persistentTable = [] self.macros_number = [] # macro self.mapping_macros_situation = {} # macro : list of situations self.situationList = [] self.read(modelFile) self.hist = hst.Histogram(self.timerange, self.situationList) self.number_situations = len(self.mapping_macros_situation) self.transitionTable = (np.zeros(shape=(self.timerange, self.number_situations, self.number_situations))) self.persistentTable = np.zeros(shape=(self.timerange, self.number_situations)) self.dataFiles = dataFiles self.hist.callingSupport(self.dataFiles) self.clustering_angle = {} self.cluster_outward_angle() self.persistentProbability()
def main(): #Command line options (-c,-d,-i,-p,-o,-s,-g) with argparse module: parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-c", "--conceal", action="store_true", help="conceal ELF file in image; use with -i, -p, -o") group.add_argument("-d", "--deploy", action="store_true", help="extract ELF file from image and execute in memory; use with -i") parser.add_argument("-i", "--image", type=str, help="image input required as argument", metavar='<image file>') parser.add_argument("-p", "--payload", type=str, help="elf executable file required as argument", metavar='<elf file>') parser.add_argument("-o", "--output", type=str, help="save your output file in PNG format", metavar='<output file name.png>') parser.add_argument("-s", "--seed", type=str, help="seed for reproducible initiation of the PRNG that determines the pixel sequence.") parser.add_argument("-g", "--histogram", action="store_true", help="display histogram of superimposed input and output PNGs") args = parser.parse_args() #Set variables with command-line inputs conceal = args.conceal deploy = args.deploy in_image = args.image in_payload = args.payload outfile = args.output rand_seed = args.seed histogram = args.histogram #Runtime if conceal: concealer(in_image, in_payload, rand_seed, outfile) #if -g, display a histogram of input and output images superimposed if histogram: import histogram h = histogram.Histogram() h.images = [in_image, outfile] h.plotter() elif deploy: result = deployer(in_image, rand_seed) hex_chars = hex(int(result, 2)) file_bytes = binascii.a2b_hex(hex_chars[2:]) execute(file_bytes)
def main(argv): lamb = float(argv[1]) # Arrival rate mu = float(argv[2]) # Service rate hist = histogram.Histogram(60 + 1) q = linkedlistqueue.Queue() stddraw.createWindow(700, 500) nextArrival = stdrandom.exp(lamb) # Time of next arrival nextService = nextArrival + 1.0 / mu # Time of next completed service # Simulate the M/D/1 queue while True: # Next event is an arrival. while nextArrival < nextService: # Simulate an arrival q.enqueue(nextArrival) nextArrival += stdrandom.exp(lamb) # Next event is a service completion. arrival = q.dequeue() wait = nextService - arrival # Update the histogram. stddraw.clear() hist.addDataPoint(min([60, int(wait + 0.5)])) hist.draw() #stddraw.sleep(20) stddraw.sleep(20) stddraw.show() # Update the queue. if q.isEmpty(): nextService = nextArrival + 1.0 / mu else: nextService = nextService + 1.0 / mu
def indicator(b): if b: return 1.0 return 0.0 # income distribution bin_boundaries_HI = [ 0.0, 10000.0, 15000.0, 25000.0, 35000.0, 50000.0, 75000.0, 100000.0, 150000.0, 200000.0, 2000000.0 ] bin_weights_HI = [ 0.014, 0.011, 0.043, 0.041, 0.055, 0.171, 0.162, 0.306, 0.108, 0.0910 ] h_income = h.Histogram(bin_boundaries_HI, bin_weights_HI) household_income = bayes.Variable( "household_income", [], lambda par, prev: h_income.gen_random_walk_sample(prev), lambda par, gen: h_income.get_interpolated_unnormalized_prob(gen)) # BN = bayes.BayesianNetwork([household_income]) # S = sampling.MetropolisSampler(BN) # population, acceptance_rate = S.generate_population(1000) # print "acceptance rate:", acceptance_rate # from matplotlib import pyplot as plt # plt.hist([x["household_income"] for x in population], bins=50) # plt.show() # exit() # household type distribution
print "rho: ", rho # Fokker Planck for SDE a = 1 D = 0.1 lambd = scipy.array([a,D]) Dt = 0.01 # Fokker Planck for SDE sampler_param = inv_transform.Sampler.getDefaultParameters() sampler_param['seed']=0 lifting = inv_transform.Sampler(sampler_param) param_histogram = histogram.Histogram.getDefaultParameters() restriction = histogram.Histogram(param_histogram) param=particles.Particles.getDefaultParameters() param['Dt'] = Dt param['N']=100000 precond_param=precond.Precond.getDefaultParameters() precond_param['Dstar']=0. precond_param['sigma']=scipy.zeros_like(grid) precond_param['kappa']=particles.doublewell param['precond']=precond.Precond(precond_param) param_histogram = histogram.Histogram.getDefaultParameters() param_histogram['h']=2e-2 restriction = histogram.Histogram(param_kde) fp_sde = particles.Particles(lifting,restriction,rho,grid,lambd,param)
from flask import Flask, render_template, request from Nth_markov import Nth_Order_Markov from markov import First_Order_Markov from utils import get_clean_words import histogram import os app = Flask(__name__) SOURCE = "text_files/markov.txt" word_list = get_clean_words(SOURCE) num_words = request.args.get("num_words", default=10, type=int) nth_order = request.args.get("nth_order", default=2, type=int) # INITIALIZERS histogram = histogram.Histogram() markov_1 = First_Order_Markov() nth_markov = Nth_Order_Markov(word_list, 2) # CREATE LISTS histogram_list = histogram.dictionary_histogram(SOURCE) markov_dict = markov_1.chain(word_list) @app.route('/') def index(): return render_template('index.html') @app.route('Histogram/') def Histogram():
def testCustomScale(self): expected_str = ('Yes |#### | 5 (83.3%)\n' 'No | | 1 (16.6%)') actual_str = str(histogram.Histogram([('Yes', 5), ('No', 1)], scale=5)) self.assertEqual(actual_str, expected_str)
def testExampleHistogram(self): self.CompareToExpectedDefault( str(histogram.Histogram([('Yes', 5), ('No', 1)])))
xL = -2. xR = 2. grid = scipy.arange(xL + dx / 2., xR, dx) rho = scipy.ones_like(grid) / (xR - xL) print("rho: ", rho) # Fokker Planck for SDE sampler_param = inv_transform.Sampler.getDefaultParameters() sampler_param['seed'] = 0 lifting = inv_transform.Sampler(sampler_param) # param_kde = kde.KDE.getDefaultParameters() # param_kde['h']=h # restriction = kde.KDE(param_kde) restriction = histogram.Histogram() param = particles.Particles.getDefaultParameters() param['eps'] = eps param['Dt'] = Dt param['N'] = 100000 precond_param = precond.Precond.getDefaultParameters() precond_param['Dstar'] = 0. precond_param['sigma'] = scipy.zeros_like(grid) precond_param['kappa'] = particles.doublewell param['precond'] = precond.Precond(precond_param) fp_sde = particles.Particles(lifting, restriction, rho, grid, lambd, param) v = scipy.zeros_like(grid) v[10] = 1.
m = args.m X = [None] * n random_funcs = { 'Unimodal gaussian': lambda: rng.gauss(5, 3), 'Bimodal gaussian': lambda: rng.gauss(5, 1) if rng.random() < .5 else rng.gauss(10, 2), 'Exponential': lambda: rng.expovariate(5) } for func_name, random_func in random_funcs.items(): methods = { 'Histogram': histogram.Histogram(max_bins=256), 'Gaussian': gaussian.Gaussian(), 'KLL': kll.KLL(k=256, seed=42), 'StreamHist': streamhist.StreamHist(maxbins=256), 't-digest': tdigest.TDigest(K=256), } errors = collections.defaultdict(list) update_durations = collections.defaultdict(float) query_durations = collections.defaultdict(float) print('Updating each method...') for i in tqdm.tqdm(range(n)): X[i] = random_func() for name, method in methods.items(): tic = time.perf_counter_ns() method.update(X[i])
def test_histogram(input_df, Delta_0, n, distribution, algorithm, num_iter, delta=np.e**(-10), eps=3, passes=1, alpha=2, save_hist=False, ngram_union=False): ''' method for testing various histogram generation algorithms :param input_df: input dataframe of clean, tokenized data :param Delta_0: budget parameter :param n: n gram n :param distribution: noise distribution :param algorithm: set union / histogram algorithm :param num_iter: number of random shuffles to build set union over :param delta: dp delta parameter :param eps: dp epsilon parameter :param save_hist: whether histogram should be saved :return: list of release counts ''' output_arr = [] for i in range(num_iter): print("generating {}-gram histogram, iteration {}".format(n, i)) #new_hist = hgram.Histogram(n, input_df.sample(frac=1), 'askreddit') new_hist = hgram.Histogram(n, input_df, 'askreddit', ngram_union) if distribution == hgram.Noise.LAPLACE: # calculating LaPlace parameter: using Delta_0/eps if Delta not provided (same in hist gen methods if algorithm == hgram.Algorithm.COUNT: l_param, l_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_delta_hist(delta_0=Delta_0) if save_hist: new_hist.save_hist('count_laplace', str(Delta_0)) elif algorithm == hgram.Algorithm.WEIGHTED: l_param, l_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_weighted_hist(delta_0=Delta_0, weighted_dist=distribution) if save_hist: save_str = str(Delta_0) new_hist.save_hist("weighted_laplace", save_str) elif algorithm == hgram.Algorithm.POLICY: l_param, l_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_policy_laplace_hist(delta_0=Delta_0, Gamma=l_rho + alpha * l_param, passes=passes) if save_hist: save_str = str(Delta_0) new_hist.save_hist("policy_laplace", save_str) elif algorithm == hgram.Algorithm.GREEDY: l_param, l_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_policy_greedy_hist(delta_0=Delta_0, Gamma=l_rho + alpha * l_param) if save_hist: save_str = str(Delta_0) new_hist.save_hist("greedy_laplace", save_str) else: print('Error check input algorithm input') output_vocab = {} for key, val in new_hist.ngram_hist.items(): nval = val + np.random.laplace(0, l_param) if nval > l_rho: output_vocab[key] = val output_arr.append(len(output_vocab)) elif distribution == hgram.Noise.GAUSSIAN: if algorithm == hgram.Algorithm.COUNT: g_param, g_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_delta_hist(delta_0=Delta_0) if save_hist: new_hist.save_hist('count_gaussian', str(Delta_0)) elif algorithm == hgram.Algorithm.WEIGHTED: g_param, g_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_weighted_hist(delta_0=Delta_0, weighted_dist=distribution) if save_hist: save_str = str(Delta_0) new_hist.save_hist("weighted_gaussian", save_str) elif algorithm == hgram.Algorithm.POLICY: g_param, g_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_policy_gaussian_hist(delta_0=Delta_0, Gamma=g_rho + alpha * g_param, passes=passes) if save_hist: save_str = str(Delta_0) new_hist.save_hist("policy_gaussian", save_str) elif algorithm == hgram.Algorithm.MAXSUM: g_param, g_rho = calculate_threshold(algorithm, distribution, eps, delta, Delta_0) new_hist.generate_maxsum_gaussian_hist(delta_0=Delta_0, Gamma=g_rho + alpha * g_param, passes=passes) if save_hist: save_str = str(Delta_0) new_hist.save_hist("maxsum_gaussian", save_str) else: print('Error check input algorithm string') return output_vocab = {} for key, val in new_hist.ngram_hist.items(): nval = val + np.random.normal(0, g_param) if nval > g_rho: output_vocab[key] = val output_arr.append(len(output_vocab)) else: print("Error check input distribution string") return output_arr
fileName = "config-tmp-" + s["type"] with open(fileName, "wt") as f: f.write(data) f.close() # Run the executable with this config file outName = "out-tmp-" + s["type"] with open(outName, "wt") as f: subprocess.call( ["../../../build_ninja2/testconfig-opt-debug", fileName], stdout=f) f.close() # Process generated value in histogram h = histogram.Histogram(s["plotrange"][0], s["plotrange"][1], 100) with open(outName, "rt") as f: l = f.readline() while l: val = float(l) h.process(val) l = f.readline() f.close() histName = "hist-tmp-" + s["type"] with open(histName, "wt") as f: h.printProb(f) f.close()