def get_valley(target, sector_count, sector_angle, threshold, vision_angle, data): hist = histogram.Histogram(sector_count, threshold, vision_angle, data) # TODO histogram and parameters hist.plot_histogram() valley_start = None best_valley = [None, None] # the furthest a valley could every be from is 360 degrees since this is a circle best_distance = 361 for i in range(len(hist.sectors)): if valley_start is None: if hist.sectors[i] < threshold: valley_start = i elif hist.sectors[i] > threshold: dist = get_target_distance(valley_start, i, target, sector_angle) if dist < best_distance: best_distance = dist best_valley = [valley_start, i] valley_start = None if valley_start is not None: dist = get_target_distance(valley_start, sector_count, target, sector_angle) if dist < best_distance: best_valley = [valley_start, i] return best_valley
def testCustomFormatter(self):
expected_str = ('Yes |################ | 5000 (4.8 KiB) (83.3%)\n'
'No |### | 1000 (16.6%)')
actual_str = str(
histogram.Histogram([('Yes', 5000), ('No', 1000)],
formatter=self.AddHumanReadableSize))
self.assertEqual(actual_str, expected_str)
def __init__(self,
segment_width,
segment_height,
bin_threshold=225,
histogram_threshold=0):
sg.Segmentation.__init__(self, segment_width, segment_height,
bin_threshold)
self.histogram = histogram.Histogram(histogram_threshold)
def transitionProbability(self):
for file in self.dataFiles:
temp_hist = hst.Histogram(self.timerange, self.situationList)
temp_hist.callingSupport([file])
aggregatedHist = temp_hist.aggregate(self.mapping_macros_situation,
self.macros_number)
situation = [0] + hst.resolve(aggregatedHist)
for timestamp in range(1, self.timerange):
sList = situation[timestamp - 1:timestamp + 1]
self.transitionTable[timestamp][sList[0]][sList[1]] += 1
self.updateTransitionProbability()
def histo(self, trial, n, gated=True, zoom_peak=False):
def peak_test(b):
if b < 3000:
return True
return False
def always_false(b):
return False
dt = self.data.intervals[trial]
dtp = dt[np.where(dt < 4000)]
return hist.Histogram(dt if not zoom_peak else dtp, n,
peak_test if gated else always_false)
def __init__(self,
identifier,
grid_array,
run_detect=True,
threshold=None,
blob_detect=BlobDetectionTypes.DEFAULT,
image_color_logic="norm",
center=None,
radius=None):
CellItem.__init__(self, identifier, grid_array)
self.threshold = threshold
if not isinstance(blob_detect, BlobDetectionTypes):
try:
blob_detect = BlobDetectionTypes[blob_detect.upper()]
except KeyError:
blob_detect = BlobDetectionTypes.DEFAULT
if blob_detect is BlobDetectionTypes.THRESHOLD:
self.detect_function = self.threshold
elif blob_detect is BlobDetectionTypes.ITERATIVE:
self.detect_function = self.iterative_threshold_detect
else:
self.detect_function = self.default_detect
self.old_trash = None
self.trash_array = None
self.image_color_logic = image_color_logic
self._features_key_list += [MEASURES.Centroid, MEASURES.Perimeter]
self.features.shape = (len(self._features_key_list), )
self.histogram = histogram.Histogram(self.grid_array,
run_at_init=False)
if run_detect:
if center is not None and radius is not None:
self.manual_detect(center, radius)
else:
self.detect_function()
self._debug_ticker = 0
def plot_bin_of_bins(self, trial):
b = bn.Binned(self, trial)
h = hist.Histogram(b.bin_count, b.bin_count.max(), lambda x: False)
plt.plot(h.bin_centers, h.bin_counts, drawstyle='steps-mid', color='k')
mean = np.mean(h.dt)
x_range = np.arange(h.bin_centers.min(), h.bin_centers.max(), 0.1)
def poisson(xx):
return np.exp(-1. * mean) * np.power(mean,
xx) / scifn.gamma(xx + 1)
plt.plot(x_range,
(h.bw * h.dt.size * np.array([poisson(x) for x in x_range])),
'--',
color='k')
plt.xlabel("Counts per Bin")
plt.ylabel("Frequency")
plt.legend(['Data', 'Poisson Model'])
plt.title("Histogram of events per time bin")
plt.show()
def __init__(self, width, modelFile, dataFiles):
self.width = width
self.timerange = int(1440 / width)
self.transitionTable = []
self.persistentTable = []
self.macros_number = [] # macro
self.mapping_macros_situation = {} # macro : list of situations
self.situationList = []
self.read(modelFile)
self.hist = hst.Histogram(self.timerange, self.situationList)
self.number_situations = len(self.mapping_macros_situation)
self.transitionTable = (np.zeros(shape=(self.timerange,
self.number_situations,
self.number_situations)))
self.persistentTable = np.zeros(shape=(self.timerange,
self.number_situations))
self.dataFiles = dataFiles
self.hist.callingSupport(self.dataFiles)
self.clustering_angle = {}
self.cluster_outward_angle()
self.persistentProbability()
def main():
#Command line options (-c,-d,-i,-p,-o,-s,-g) with argparse module:
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
group.add_argument("-c", "--conceal", action="store_true", help="conceal ELF file in image; use with -i, -p, -o")
group.add_argument("-d", "--deploy", action="store_true", help="extract ELF file from image and execute in memory; use with -i")
parser.add_argument("-i", "--image", type=str, help="image input required as argument", metavar='<image file>')
parser.add_argument("-p", "--payload", type=str, help="elf executable file required as argument", metavar='<elf file>')
parser.add_argument("-o", "--output", type=str, help="save your output file in PNG format", metavar='<output file name.png>')
parser.add_argument("-s", "--seed", type=str, help="seed for reproducible initiation of the PRNG that determines the pixel sequence.")
parser.add_argument("-g", "--histogram", action="store_true", help="display histogram of superimposed input and output PNGs")
args = parser.parse_args()
#Set variables with command-line inputs
conceal = args.conceal
deploy = args.deploy
in_image = args.image
in_payload = args.payload
outfile = args.output
rand_seed = args.seed
histogram = args.histogram
#Runtime
if conceal:
concealer(in_image, in_payload, rand_seed, outfile)
#if -g, display a histogram of input and output images superimposed
if histogram:
import histogram
h = histogram.Histogram()
h.images = [in_image, outfile]
h.plotter()
elif deploy:
result = deployer(in_image, rand_seed)
hex_chars = hex(int(result, 2))
file_bytes = binascii.a2b_hex(hex_chars[2:])
execute(file_bytes)
def main(argv):
lamb = float(argv[1]) # Arrival rate
mu = float(argv[2]) # Service rate
hist = histogram.Histogram(60 + 1)
q = linkedlistqueue.Queue()
stddraw.createWindow(700, 500)
nextArrival = stdrandom.exp(lamb) # Time of next arrival
nextService = nextArrival + 1.0 / mu # Time of next completed service
# Simulate the M/D/1 queue
while True:
# Next event is an arrival.
while nextArrival < nextService:
# Simulate an arrival
q.enqueue(nextArrival)
nextArrival += stdrandom.exp(lamb)
# Next event is a service completion.
arrival = q.dequeue()
wait = nextService - arrival
# Update the histogram.
stddraw.clear()
hist.addDataPoint(min([60, int(wait + 0.5)]))
hist.draw()
#stddraw.sleep(20)
stddraw.sleep(20)
stddraw.show()
# Update the queue.
if q.isEmpty():
nextService = nextArrival + 1.0 / mu
else:
nextService = nextService + 1.0 / mu
def indicator(b):
if b:
return 1.0
return 0.0
# income distribution
bin_boundaries_HI = [
0.0, 10000.0, 15000.0, 25000.0, 35000.0, 50000.0, 75000.0, 100000.0,
150000.0, 200000.0, 2000000.0
]
bin_weights_HI = [
0.014, 0.011, 0.043, 0.041, 0.055, 0.171, 0.162, 0.306, 0.108, 0.0910
]
h_income = h.Histogram(bin_boundaries_HI, bin_weights_HI)
household_income = bayes.Variable(
"household_income", [],
lambda par, prev: h_income.gen_random_walk_sample(prev),
lambda par, gen: h_income.get_interpolated_unnormalized_prob(gen))
# BN = bayes.BayesianNetwork([household_income])
# S = sampling.MetropolisSampler(BN)
# population, acceptance_rate = S.generate_population(1000)
# print "acceptance rate:", acceptance_rate
# from matplotlib import pyplot as plt
# plt.hist([x["household_income"] for x in population], bins=50)
# plt.show()
# exit()
# household type distribution
print "rho: ", rho
# Fokker Planck for SDE
a = 1
D = 0.1
lambd = scipy.array([a,D])
Dt = 0.01
# Fokker Planck for SDE
sampler_param = inv_transform.Sampler.getDefaultParameters()
sampler_param['seed']=0
lifting = inv_transform.Sampler(sampler_param)
param_histogram = histogram.Histogram.getDefaultParameters()
restriction = histogram.Histogram(param_histogram)
param=particles.Particles.getDefaultParameters()
param['Dt'] = Dt
param['N']=100000
precond_param=precond.Precond.getDefaultParameters()
precond_param['Dstar']=0.
precond_param['sigma']=scipy.zeros_like(grid)
precond_param['kappa']=particles.doublewell
param['precond']=precond.Precond(precond_param)
param_histogram = histogram.Histogram.getDefaultParameters()
param_histogram['h']=2e-2
restriction = histogram.Histogram(param_kde)
fp_sde = particles.Particles(lifting,restriction,rho,grid,lambd,param)
from flask import Flask, render_template, request
from Nth_markov import Nth_Order_Markov
from markov import First_Order_Markov
from utils import get_clean_words
import histogram
import os
app = Flask(__name__)
SOURCE = "text_files/markov.txt"
word_list = get_clean_words(SOURCE)
num_words = request.args.get("num_words", default=10, type=int)
nth_order = request.args.get("nth_order", default=2, type=int)
# INITIALIZERS
histogram = histogram.Histogram()
markov_1 = First_Order_Markov()
nth_markov = Nth_Order_Markov(word_list, 2)
# CREATE LISTS
histogram_list = histogram.dictionary_histogram(SOURCE)
markov_dict = markov_1.chain(word_list)
@app.route('/')
def index():
return render_template('index.html')
@app.route('Histogram/')
def Histogram():
def testCustomScale(self):
expected_str = ('Yes |#### | 5 (83.3%)\n' 'No | | 1 (16.6%)')
actual_str = str(histogram.Histogram([('Yes', 5), ('No', 1)], scale=5))
self.assertEqual(actual_str, expected_str)
def testExampleHistogram(self):
self.CompareToExpectedDefault(
str(histogram.Histogram([('Yes', 5), ('No', 1)])))
xL = -2.
xR = 2.
grid = scipy.arange(xL + dx / 2., xR, dx)
rho = scipy.ones_like(grid) / (xR - xL)
print("rho: ", rho)
# Fokker Planck for SDE
sampler_param = inv_transform.Sampler.getDefaultParameters()
sampler_param['seed'] = 0
lifting = inv_transform.Sampler(sampler_param)
# param_kde = kde.KDE.getDefaultParameters()
# param_kde['h']=h
# restriction = kde.KDE(param_kde)
restriction = histogram.Histogram()
param = particles.Particles.getDefaultParameters()
param['eps'] = eps
param['Dt'] = Dt
param['N'] = 100000
precond_param = precond.Precond.getDefaultParameters()
precond_param['Dstar'] = 0.
precond_param['sigma'] = scipy.zeros_like(grid)
precond_param['kappa'] = particles.doublewell
param['precond'] = precond.Precond(precond_param)
fp_sde = particles.Particles(lifting, restriction, rho, grid, lambd, param)
v = scipy.zeros_like(grid)
v[10] = 1.
m = args.m
X = [None] * n
random_funcs = {
'Unimodal gaussian':
lambda: rng.gauss(5, 3),
'Bimodal gaussian':
lambda: rng.gauss(5, 1) if rng.random() < .5 else rng.gauss(10, 2),
'Exponential':
lambda: rng.expovariate(5)
}
for func_name, random_func in random_funcs.items():
methods = {
'Histogram': histogram.Histogram(max_bins=256),
'Gaussian': gaussian.Gaussian(),
'KLL': kll.KLL(k=256, seed=42),
'StreamHist': streamhist.StreamHist(maxbins=256),
't-digest': tdigest.TDigest(K=256),
}
errors = collections.defaultdict(list)
update_durations = collections.defaultdict(float)
query_durations = collections.defaultdict(float)
print('Updating each method...')
for i in tqdm.tqdm(range(n)):
X[i] = random_func()
for name, method in methods.items():
tic = time.perf_counter_ns()
method.update(X[i])
def test_histogram(input_df,
Delta_0,
n,
distribution,
algorithm,
num_iter,
delta=np.e**(-10),
eps=3,
passes=1,
alpha=2,
save_hist=False,
ngram_union=False):
'''
method for testing various histogram generation algorithms
:param input_df: input dataframe of clean, tokenized data
:param Delta_0: budget parameter
:param n: n gram n
:param distribution: noise distribution
:param algorithm: set union / histogram algorithm
:param num_iter: number of random shuffles to build set union over
:param delta: dp delta parameter
:param eps: dp epsilon parameter
:param save_hist: whether histogram should be saved
:return: list of release counts
'''
output_arr = []
for i in range(num_iter):
print("generating {}-gram histogram, iteration {}".format(n, i))
#new_hist = hgram.Histogram(n, input_df.sample(frac=1), 'askreddit')
new_hist = hgram.Histogram(n, input_df, 'askreddit', ngram_union)
if distribution == hgram.Noise.LAPLACE:
# calculating LaPlace parameter: using Delta_0/eps if Delta not provided (same in hist gen methods
if algorithm == hgram.Algorithm.COUNT:
l_param, l_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_delta_hist(delta_0=Delta_0)
if save_hist:
new_hist.save_hist('count_laplace', str(Delta_0))
elif algorithm == hgram.Algorithm.WEIGHTED:
l_param, l_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_weighted_hist(delta_0=Delta_0,
weighted_dist=distribution)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("weighted_laplace", save_str)
elif algorithm == hgram.Algorithm.POLICY:
l_param, l_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_policy_laplace_hist(delta_0=Delta_0,
Gamma=l_rho +
alpha * l_param,
passes=passes)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("policy_laplace", save_str)
elif algorithm == hgram.Algorithm.GREEDY:
l_param, l_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_policy_greedy_hist(delta_0=Delta_0,
Gamma=l_rho +
alpha * l_param)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("greedy_laplace", save_str)
else:
print('Error check input algorithm input')
output_vocab = {}
for key, val in new_hist.ngram_hist.items():
nval = val + np.random.laplace(0, l_param)
if nval > l_rho:
output_vocab[key] = val
output_arr.append(len(output_vocab))
elif distribution == hgram.Noise.GAUSSIAN:
if algorithm == hgram.Algorithm.COUNT:
g_param, g_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_delta_hist(delta_0=Delta_0)
if save_hist:
new_hist.save_hist('count_gaussian', str(Delta_0))
elif algorithm == hgram.Algorithm.WEIGHTED:
g_param, g_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_weighted_hist(delta_0=Delta_0,
weighted_dist=distribution)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("weighted_gaussian", save_str)
elif algorithm == hgram.Algorithm.POLICY:
g_param, g_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_policy_gaussian_hist(delta_0=Delta_0,
Gamma=g_rho +
alpha * g_param,
passes=passes)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("policy_gaussian", save_str)
elif algorithm == hgram.Algorithm.MAXSUM:
g_param, g_rho = calculate_threshold(algorithm, distribution,
eps, delta, Delta_0)
new_hist.generate_maxsum_gaussian_hist(delta_0=Delta_0,
Gamma=g_rho +
alpha * g_param,
passes=passes)
if save_hist:
save_str = str(Delta_0)
new_hist.save_hist("maxsum_gaussian", save_str)
else:
print('Error check input algorithm string')
return
output_vocab = {}
for key, val in new_hist.ngram_hist.items():
nval = val + np.random.normal(0, g_param)
if nval > g_rho:
output_vocab[key] = val
output_arr.append(len(output_vocab))
else:
print("Error check input distribution string")
return output_arr
fileName = "config-tmp-" + s["type"]
with open(fileName, "wt") as f:
f.write(data)
f.close()
# Run the executable with this config file
outName = "out-tmp-" + s["type"]
with open(outName, "wt") as f:
subprocess.call(
["../../../build_ninja2/testconfig-opt-debug", fileName], stdout=f)
f.close()
# Process generated value in histogram
h = histogram.Histogram(s["plotrange"][0], s["plotrange"][1], 100)
with open(outName, "rt") as f:
l = f.readline()
while l:
val = float(l)
h.process(val)
l = f.readline()
f.close()
histName = "hist-tmp-" + s["type"]
with open(histName, "wt") as f:
h.printProb(f)
f.close()
