def interval_creator(arg):
""" """
if type(arg) is int:
return Interval(arg)
elif type(arg) is str:
return Interval(default_name_to_id[default_name(arg)])
elif isinstance(arg, Container) and len(arg) == 2:
# arg es una tupla/lista de 2 notas
for elem in arg:
if not isinstance(elem, n.Note):
raise NotImplementedError('Wrong arg/s type')
return Interval(arg[0].id - arg[1].id)
else:
raise NotImplementedError('Wrong arg/s type')
def get_intervals(self):
intervals = []
for info in self.ceres_node.slice_info:
(start, end, step) = info
intervals.append(Interval(start, end))
return IntervalSet(intervals)
def select_event(algorithm,
args,
kwargs,
D1,
D2,
epsilon,
iterations=100000,
search_space=None,
cores=0):
assert isfunction(algorithm)
from .core import test_statistics
result_d1 = [algorithm(D1, *args, **kwargs) for _ in range(iterations)]
result_d2 = [algorithm(D2, *args, **kwargs) for _ in range(iterations)]
if search_space is None:
# determine the search space based on the return type
# a subset of results to determine return type
sub_result = result_d1 + result_d2
counter = Counter(sub_result)
# categorical output
if len(counter) < iterations * 0.02 * 0.1:
search_space = tuple((key, ) for key in counter.keys())
else:
sub_result_sorted = np.sort(sub_result)
average = np.average(sub_result_sorted)
idx = np.searchsorted(sub_result_sorted, average, side='left')
# find the densest 70% range
search_min = int(idx - 0.35 * len(sub_result_sorted)) if int(
idx - 0.4 * len(sub_result_sorted)) > 0 else 0
search_max = int(0.7 * len(sub_result_sorted) - (idx - search_min))
search_space = tuple(
Interval((-float('inf'), alpha))
for alpha in np.linspace(sub_result_sorted[search_min],
sub_result_sorted[search_max],
num=25))
logger.info('search space is set to {0}'.format(search_space))
global _process_pool
# find an event which has minimum p value from search space
threshold = 0.001 * iterations * np.exp(epsilon)
results = list(map(__EvaluateEvent(result_d1, result_d2, epsilon, iterations), search_space)) if cores == 1 \
else _process_pool.map(__EvaluateEvent(result_d1, result_d2, epsilon, iterations), search_space)
p_values = [
test_statistics(x[0], x[1], epsilon, iterations)
if x[0] + x[1] > threshold else float('inf') for x in results
]
for i, (s, (cx, cy), p) in enumerate(zip(search_space, results, p_values)):
logger.debug(
'event: %s | p: %f | cx: %d | cy: %d | ratio: %f' %
(s, p, cx, cy, float(cy) / cx if cx != 0 else float('inf')))
return search_space[np.argmin(p_values)]
def _gff_interval_iterator(f):
"""
Shallow parser that returns information in Interval records.
Line is stripped of whitespace and (start, end) is zero-based, half-open for more
standardized processing in Python. Ignores empty lines and presumes strand is + unless
explicitly set to -.
"""
# columns for each of the fields we're interested in (0-based)
chrom_col, start_col, end_col, strand_col = 0, 3, 4, 6
for line in f:
if line.startswith("#") or line.isspace():
continue
fields = line.split()
chrom = fields[chrom_col]
# in GFF, numbering is 1-based, but in Python, we use 0-based, half-open,
# so subtract 1 from the value given in the start column
start, end = int(fields[start_col]) - 1, int(fields[end_col])
if start > end: warn("interval start after end")
strand = "+"
if len(fields) > strand_col:
if fields[strand_col] == "-": strand = "-"
yield Interval(chrom, start, end, strand, line.strip())
def _bed_interval_iterator(f):
"""
Shallow parser that returns information in Interval records.
Line is stripped of whitespace and (start, end) is zero-based, half-open. Ignores empty
lines and presumes strand is + unless explicitly set to -.
"""
# columns for each of the fields we're interested in (0-based)
chrom_col, start_col, end_col, strand_col = 0, 1, 2, 5
# in BED format, the initial lines may be a header, but without comment hashes (annoying!)
header_allowed = True
for line in f:
if line.startswith("#") or line.isspace():
continue
fields = line.split()
if header_allowed and len(fields) == 1:
continue # the iterator does not return header contents or do anything with them
elif header_allowed:
header_allowed = False
chrom = fields[chrom_col]
start, end = int(fields[start_col]), int(fields[end_col])
if start > end: warn("interval start after end")
strand = "+"
if len(fields) > strand_col:
if fields[strand_col] == "-": strand = "-"
yield Interval(chrom, start, end, strand, line.strip())
def __init__(self, pattern, startTime, endTime):
self.pattern = pattern
self.startTime = startTime
self.endTime = endTime
self.isExact = is_pattern(pattern)
self.interval = Interval(float('-inf') if startTime is None else startTime,
float('inf') if endTime is None else endTime)
def select_event(algorithm, input_list, epsilon, iterations=100000, search_space=None, process_pool=None):
"""
:param algorithm: The algorithm to run on
:param input_list: list of (d1, d2, kwargs) input pair for the algorithm to run
:param epsilon: Test epsilon value
:param iterations: The iterations to run algorithms
:param search_space: The result search space to run on, auto-determine based on return type if None
:param process_pool: The process pool to use, run with single process if None
:return: (d1, d2, kwargs, event) pair which has minimum p value from search space
"""
assert isfunction(algorithm)
from .hypotest import test_statistics
input_event_pairs = []
p_values = []
for (d1, d2, kwargs) in input_list:
result_d1 = [algorithm(d1, **kwargs) for _ in range(iterations)]
result_d2 = [algorithm(d2, **kwargs) for _ in range(iterations)]
if search_space is None:
# determine the search space based on the return type
# a subset of results to determine return type
sub_result = result_d1 + result_d2
counter = Counter(sub_result)
# categorical output
if len(counter) < iterations * 0.02 * 0.1:
search_space = tuple((key,) for key in counter.keys())
else:
sub_result_sorted = np.sort(sub_result)
average = np.average(sub_result_sorted)
idx = np.searchsorted(sub_result_sorted, average, side='left')
# find the densest 70% range
search_min = int(idx - 0.35 * len(sub_result_sorted)) if int(idx - 0.4 * len(sub_result_sorted)) > 0 else 0
search_max = int(0.7 * len(sub_result_sorted) - (idx - search_min))
search_space = tuple(Interval((-float('inf'), alpha)) for alpha in
np.linspace(sub_result_sorted[search_min], sub_result_sorted[search_max], num=25))
logger.info('search space is set to {0}'.format(search_space))
threshold = 0.001 * iterations * np.exp(epsilon)
results = list(map(__EvaluateEvent(result_d1, result_d2, iterations), search_space)) if process_pool is None \
else process_pool.map(__EvaluateEvent(result_d1, result_d2, iterations), search_space)
input_p_values = [test_statistics(cx, cy, epsilon, iterations)
if cx + cy > threshold else float('inf') for (cx, cy) in results]
for (s, (cx, cy), p) in zip(search_space, results, input_p_values):
logger.debug('d1: %s | d2: %s | event: %s | p: %f | cx: %d | cy: %d | ratio: %f' %
(d1, d2, s, p, cx, cy, float(cy) / cx if cx != 0 else float('inf')))
input_event_pairs.extend(list((d1, d2, kwargs, event) for event in search_space))
p_values.extend(input_p_values)
# find an (d1, d2, kwargs, event) pair which has minimum p value from search space
return input_event_pairs[np.argmin(p_values)]
def get_intervals(self):
fh = gzip.GzipFile(self.fs_path, 'rb')
try:
info = whisper.__readHeader(fh) # evil, but necessary.
finally:
fh.close()
start = time.time() - info['maxRetention']
end = max(os.stat(self.fs_path).st_mtime, start)
return IntervalSet([Interval(start, end)])
def __init__(self):
"""line_map = dict()
line_map['left'] = (-1.8, -5.4)
line_map['mid'] = (-1.8, 1.8)
line_map['right'] = (1.8, 5.4)"""
self.lane_map = {
'pre_left': Interval((-9.0, -5.4)),
'left': Interval((-5.4, -1.8)),
'mid': Interval((-1.8, 1.8)),
'right': Interval((1.8, 5.4)),
'after_right': Interval((5.4, 9.0))
}
self.test = Interval((-1, 1))
self.ego_pos_init = 'mid'
self.left_lane = self.lane_map['left']
self.mid_lane = self.lane_map['mid']
self.right_lane = self.lane_map['right']
self.obj_pos = dict()
def test_guesses_types(self, number_range, type):
assert Interval(number_range).type == type
def main():
jobs = [
{
"algorithm": noisy_max_v1a,
"kwargs": {},
"databases": ([0] + [2 for _ in range(4)], [1 for _ in range(5)]),
"search_space": tuple([i] for i in range(5))
},
{
"algorithm":
noisy_max_v1b,
"kwargs": {},
"databases": ([2 for _ in range(5)], [1 for _ in range(5)]),
"search_space":
tuple(Interval([-inf, alpha]) for alpha in range(-5, 6))
},
{
"algorithm": noisy_max_v2a,
"kwargs": {},
"databases": ([0] + [2 for _ in range(4)], [1 for _ in range(5)]),
"search_space": tuple([i] for i in range(5))
},
{
"algorithm":
noisy_max_v2b,
"kwargs": {},
"databases": ([2] + [0 for _ in range(4)], [1 for _ in range(5)]),
"search_space":
tuple(
Interval([-inf, 1 + alpha / 10.0])
for alpha in range(0, 80, 2))
},
{
"algorithm": histogram,
"kwargs": {},
"databases": ([2 for _ in range(5)], [1] + [2 for _ in range(4)]),
"search_space":
[Interval([-inf, alpha]) for alpha in range(-17, 17)]
},
{
"algorithm":
histogram_eps,
"kwargs": {},
"databases": ([0] + [1 for _ in range(4)], [1 for _ in range(5)]),
"search_space":
[Interval([-inf, alpha / 10]) for alpha in range(-30, 30, 2)]
},
{
"algorithm":
iSVT1,
"kwargs": {
'T': 1,
'N': 1
},
"databases": ([1 for _ in range(10)],
[0 for _ in range(5)] + [2 for _ in range(5)]),
"search_space": [[i] for i in range(10)]
},
{
"algorithm":
iSVT2,
"kwargs": {
'T': 1,
'N': 1
},
"databases": ([1 for _ in range(5)] + [0 for _ in range(5)],
[0 for _ in range(5)] + [1 for _ in range(5)]),
"search_space": [[i] for i in range(10)]
},
{
"algorithm":
iSVT3,
"kwargs": {
'T': 1,
'N': 1
},
"databases": ([1 for _ in range(5)] + [0 for _ in range(5)],
[0 for _ in range(5)] + [1 for _ in range(5)]),
"search_space": [[i] for i in range(10)]
},
]
results = {}
start_time = time.time()
for job in jobs:
results.clear()
algorithm, search_space, = job['algorithm'], job['search_space']
databases, kwargs = job['databases'], job['kwargs']
for algorithm_epsilon in [0.2, 0.5, 0.7] + list(range(1, 4)):
kwargs['epsilon'] = algorithm_epsilon
results[algorithm_epsilon] = detect_counterexample(
algorithm, [x / 10.0 for x in range(1, 34, 1)], kwargs,
search_space, databases)
draw_graph('Test $\epsilon$', 'P Value', results,
algorithm.__name__.replace('_', ' ').title(),
algorithm.__name__ + '.pdf')
# dump the results to file
with open('./{}.json'.format(algorithm.__name__), 'w') as f:
# json.dump(results, f)
# cannot use json.dump since Interval class is not JSON-serializable
f.write(jsonpickle.encode(results))
print('{} | D1: {} | D2: {} | Time: {}'.format(
algorithm.__name__, databases[0], databases[1],
time.time() - start_time))
start_time = time.time()
def __init__(self, database, data=None, _id=None):
# Create a model controller object.
ModelController.__init__(self, 'target', database, data=data, _id=_id)
self.queue = Queue(maxsize=0)
self.camera = self.db.camera.find_one(qwrap(self.model['owner_id']))
self.interval = Interval(self)
def get_intervals(self):
start = time.time() - self.get_retention(self.fs_path)
end = max(os.stat(self.fs_path).st_mtime, start)
return IntervalSet([Interval(start, end)])
def main():
# auto-generated data for sparsevector
# D1 = [10.747997673023637, 9.176688735248929, 9.685779806342222, 9.276506422051003, 10.14197198493314, 10.777285751217708, 10.033518297158158, 10.739107887579431, 9.466024642462145, 10.046506781898248]
# D2 = [9.74799, 9.999989999999999, 10.19486, 9.99999, 9.57099, 9.888639999999999, 9.08919, 9.76694, 9.99999, 9.73776]
# auto-generated data for noisymax
# D1 = [10.047838788362302, 9.156301318712105, 9.934576874381165, 10.818923545533439, 10.964414221707846, 9.814549818258723, 10.343685529387757, 10.270895438249106, 10.422325599687007, 9.498322242077997]
# D2 = [11.04783, 10.10206, 10.85322, 10.04784, 10.04784, 10.40406, 10.04784, 9.664679999999999, 9.660929999999999, 10.06784]
# manual data for niosymax version 1a and 2a
#D1 = [0] + [2 for _ in range(4)]
#D2 = [1 for _ in range(5)]
# manual data for noisymax version 1b and 2b
D1 = [2 for _ in range(5)]
D2 = [1 for _ in range(5)]
jobs = [{
"algorithm":
noisy_max_v1a,
"D1":
lambda eps: [0] + [2 for _ in range(4)],
"D2":
lambda eps: [1 for _ in range(5)],
"S":
lambda: select_event(algorithm, (),
kwargs,
D1,
D2,
epsilon,
search_space=[[i] for i in range(5)])
}, {
"algorithm":
noisy_max_v1b,
"D1":
lambda eps: [2 for _ in range(5)],
"D2":
lambda eps: [1 for _ in range(5)],
"S":
lambda: select_event(
algorithm, (),
kwargs,
D1,
D2,
epsilon,
search_space=[Interval([-inf, alpha]) for alpha in range(-5, 6)])
}, {
"algorithm":
noisy_max_v2a,
"D1":
lambda eps: [0] + [2 for _ in range(4)],
"D2":
lambda eps: [1 for _ in range(5)],
"S":
lambda: select_event(algorithm, (),
kwargs,
D1,
D2,
epsilon,
search_space=[[i] for i in range(5)])
}, {
"algorithm":
noisy_max_v2b,
"D1":
lambda eps: [2] + [0 for _ in range(4)],
"D2":
lambda eps: [1 for _ in range(5)],
"S":
lambda: select_event(algorithm, (),
kwargs,
D1,
D2,
epsilon,
search_space=[
Interval([-inf, 1 + alpha / 10.0])
for alpha in range(0, 80, 2)
])
}, {
"algorithm":
histogram,
"D1":
lambda eps: [2 for _ in range(5)],
"D2":
lambda eps: [1] + [2 for _ in range(4)],
"S":
lambda: select_event(algorithm, (),
kwargs,
D1,
D2,
epsilon,
search_space=[
Interval([2 + alpha / 10.0, inf])
for alpha in range(0, 20, 2)
])
}]
# manual data for sparsevector
results = {}
start_time = time.time()
for job in jobs:
results.clear()
algorithm = job['algorithm']
for algorithm_epsilon in [0.2, 0.5, 0.7] + list(range(1, 4)):
results[algorithm_epsilon] = []
for epsilon in [x / 10.0 for x in range(1, 34, 1)]:
# algorithm's fixed privacy budget
kwargs = {'eps': algorithm_epsilon}
kwargs.update(job.get('kwargs', {}))
D1 = job['D1'](epsilon)
D2 = job['D2'](epsilon)
# call s selector to find s
S = job['S']()
p1, p2 = hypothesis_test(algorithm, (),
kwargs,
D1,
D2,
S,
epsilon,
100000,
cores=0)
results[algorithm_epsilon].append((epsilon, p1, p2))
print("epsilon: %f, p1 = %f, p2 = %f | S = %s" %
(epsilon, p1, p2, S))
draw_graph('Test $\epsilon$', 'P Value', results,
algorithm.__name__.replace('_', ' ').title(),
algorithm.__name__ + '.pdf')
# print output
print("%s" % algorithm.__name__)
print("D1 : %s" % D1)
print("D2 : %s" % D2)
print("Time elapsed: %f seconds" % (time.time() - start_time))
def test_Interval_Instantiation(self):
with self.assertRaises(TypeError):
Interval(1, 3)
def get_intervals(self):
start = 0
end = int(time.time())
return IntervalSet([Interval(start, end)])
def find(self, pattern, startTime=None, endTime=None, local=False):
query = FindQuery(pattern, startTime, endTime)
# Start remote searches
if not local:
remote_requests = [r.find(query) for r in self.remote_stores if r.available]
matching_nodes = set()
# Search locally
for finder in self.finders:
for node in finder.find_nodes(query):
#log.info("find() :: local :: %s" % node)
matching_nodes.add(node)
# Gather remote search results
if not local:
for request in remote_requests:
for node in request.get_results():
#log.info("find() :: remote :: %s from %s" % (node,request.store.host))
matching_nodes.add(node)
# Group matching nodes by their path
nodes_by_path = {}
for node in matching_nodes:
if node.path not in nodes_by_path:
nodes_by_path[node.path] = []
nodes_by_path[node.path].append(node)
# Reduce matching nodes for each path to a minimal set
found_branch_nodes = set()
for path, nodes in nodes_by_path.iteritems():
leaf_nodes = []
# First we dispense with the BranchNodes
for node in nodes:
if node.is_leaf:
leaf_nodes.append(node)
elif node.path not in found_branch_nodes: #TODO need to filter branch nodes based on requested interval... how?!?!?
yield node
found_branch_nodes.add(node.path)
if not leaf_nodes:
continue
# Calculate best minimal node set
minimal_node_set = set()
covered_intervals = IntervalSet([])
# If the query doesn't fall entirely within the FIND_TOLERANCE window
# we disregard the window. This prevents unnecessary remote fetches
# caused when carbon's cache skews node.intervals, giving the appearance
# remote systems have data we don't have locally, which we probably do.
now = int(time.time())
tolerance_window = now - settings.FIND_TOLERANCE
disregard_tolerance_window = query.interval.start < tolerance_window
prior_to_window = Interval(float('-inf'), tolerance_window)
def measure_of_added_coverage(node, drop_window=disregard_tolerance_window):
relevant_intervals = node.intervals.intersect_interval(query.interval)
if drop_window:
relevant_intervals = relevant_intervals.intersect_interval(prior_to_window)
return covered_intervals.union(relevant_intervals).size - covered_intervals.size
nodes_remaining = list(leaf_nodes)
# Prefer local nodes first (and do *not* drop the tolerance window)
for node in leaf_nodes:
if node.local and measure_of_added_coverage(node, False) > 0:
nodes_remaining.remove(node)
minimal_node_set.add(node)
covered_intervals = covered_intervals.union(node.intervals)
while nodes_remaining:
node_coverages = [(measure_of_added_coverage(n), n) for n in nodes_remaining]
best_coverage, best_node = max(node_coverages)
if best_coverage == 0:
break
nodes_remaining.remove(best_node)
minimal_node_set.add(best_node)
covered_intervals = covered_intervals.union(best_node.intervals)
# Sometimes the requested interval falls within the caching window.
# We include the most likely node if the gap is within tolerance.
if not minimal_node_set:
def distance_to_requested_interval(node):
latest = sorted(node.intervals, key=lambda i: i.end)[-1]
distance = query.interval.start - latest.end
return distance if distance >= 0 else float('inf')
best_candidate = min(leaf_nodes, key=distance_to_requested_interval)
if distance_to_requested_interval(best_candidate) <= settings.FIND_TOLERANCE:
minimal_node_set.add(best_candidate)
if len(minimal_node_set) == 1:
yield minimal_node_set.pop()
elif len(minimal_node_set) > 1:
reader = MultiReader(minimal_node_set)
yield LeafNode(path, reader)
from intervals import Interval
import threading
# 5 seconds
stop_time = 10
# 1 image per second
interval = 2
# From the webcam
source = 0
Interval(stop_time, interval, source)
Interval(5, 1, 0)
print(threading.active_count())
print("done!")
def if_unsafe(self):
if self.state[0].cpu().data in Interval((self.x0_low, self.x0_high)) and \
self.state[1].cpu().data in Interval((self.x1_low, self.x1_high)):
return 0
else:
return 1
def get_intervals(self):
start = time.time() - whisper.info(self.fs_path)['maxRetention']
end = max(os.stat(self.fs_path).st_mtime, start)
return IntervalSet([Interval(start, end)])