def interval_creator(arg): """ """ if type(arg) is int: return Interval(arg) elif type(arg) is str: return Interval(default_name_to_id[default_name(arg)]) elif isinstance(arg, Container) and len(arg) == 2: # arg es una tupla/lista de 2 notas for elem in arg: if not isinstance(elem, n.Note): raise NotImplementedError('Wrong arg/s type') return Interval(arg[0].id - arg[1].id) else: raise NotImplementedError('Wrong arg/s type')
def get_intervals(self): intervals = [] for info in self.ceres_node.slice_info: (start, end, step) = info intervals.append(Interval(start, end)) return IntervalSet(intervals)
def select_event(algorithm, args, kwargs, D1, D2, epsilon, iterations=100000, search_space=None, cores=0): assert isfunction(algorithm) from .core import test_statistics result_d1 = [algorithm(D1, *args, **kwargs) for _ in range(iterations)] result_d2 = [algorithm(D2, *args, **kwargs) for _ in range(iterations)] if search_space is None: # determine the search space based on the return type # a subset of results to determine return type sub_result = result_d1 + result_d2 counter = Counter(sub_result) # categorical output if len(counter) < iterations * 0.02 * 0.1: search_space = tuple((key, ) for key in counter.keys()) else: sub_result_sorted = np.sort(sub_result) average = np.average(sub_result_sorted) idx = np.searchsorted(sub_result_sorted, average, side='left') # find the densest 70% range search_min = int(idx - 0.35 * len(sub_result_sorted)) if int( idx - 0.4 * len(sub_result_sorted)) > 0 else 0 search_max = int(0.7 * len(sub_result_sorted) - (idx - search_min)) search_space = tuple( Interval((-float('inf'), alpha)) for alpha in np.linspace(sub_result_sorted[search_min], sub_result_sorted[search_max], num=25)) logger.info('search space is set to {0}'.format(search_space)) global _process_pool # find an event which has minimum p value from search space threshold = 0.001 * iterations * np.exp(epsilon) results = list(map(__EvaluateEvent(result_d1, result_d2, epsilon, iterations), search_space)) if cores == 1 \ else _process_pool.map(__EvaluateEvent(result_d1, result_d2, epsilon, iterations), search_space) p_values = [ test_statistics(x[0], x[1], epsilon, iterations) if x[0] + x[1] > threshold else float('inf') for x in results ] for i, (s, (cx, cy), p) in enumerate(zip(search_space, results, p_values)): logger.debug( 'event: %s | p: %f | cx: %d | cy: %d | ratio: %f' % (s, p, cx, cy, float(cy) / cx if cx != 0 else float('inf'))) return search_space[np.argmin(p_values)]
def _gff_interval_iterator(f): """ Shallow parser that returns information in Interval records. Line is stripped of whitespace and (start, end) is zero-based, half-open for more standardized processing in Python. Ignores empty lines and presumes strand is + unless explicitly set to -. """ # columns for each of the fields we're interested in (0-based) chrom_col, start_col, end_col, strand_col = 0, 3, 4, 6 for line in f: if line.startswith("#") or line.isspace(): continue fields = line.split() chrom = fields[chrom_col] # in GFF, numbering is 1-based, but in Python, we use 0-based, half-open, # so subtract 1 from the value given in the start column start, end = int(fields[start_col]) - 1, int(fields[end_col]) if start > end: warn("interval start after end") strand = "+" if len(fields) > strand_col: if fields[strand_col] == "-": strand = "-" yield Interval(chrom, start, end, strand, line.strip())
def __init__(self, pattern, startTime, endTime): self.pattern = pattern self.startTime = startTime self.endTime = endTime self.isExact = is_pattern(pattern) self.interval = Interval(float('-inf') if startTime is None else startTime, float('inf') if endTime is None else endTime)
def _bed_interval_iterator(f): """ Shallow parser that returns information in Interval records. Line is stripped of whitespace and (start, end) is zero-based, half-open. Ignores empty lines and presumes strand is + unless explicitly set to -. """ # columns for each of the fields we're interested in (0-based) chrom_col, start_col, end_col, strand_col = 0, 1, 2, 5 # in BED format, the initial lines may be a header, but without comment hashes (annoying!) header_allowed = True for line in f: if line.startswith("#") or line.isspace(): continue fields = line.split() if header_allowed and len(fields) == 1: continue # the iterator does not return header contents or do anything with them elif header_allowed: header_allowed = False chrom = fields[chrom_col] start, end = int(fields[start_col]), int(fields[end_col]) if start > end: warn("interval start after end") strand = "+" if len(fields) > strand_col: if fields[strand_col] == "-": strand = "-" yield Interval(chrom, start, end, strand, line.strip())
def select_event(algorithm, input_list, epsilon, iterations=100000, search_space=None, process_pool=None): """ :param algorithm: The algorithm to run on :param input_list: list of (d1, d2, kwargs) input pair for the algorithm to run :param epsilon: Test epsilon value :param iterations: The iterations to run algorithms :param search_space: The result search space to run on, auto-determine based on return type if None :param process_pool: The process pool to use, run with single process if None :return: (d1, d2, kwargs, event) pair which has minimum p value from search space """ assert isfunction(algorithm) from .hypotest import test_statistics input_event_pairs = [] p_values = [] for (d1, d2, kwargs) in input_list: result_d1 = [algorithm(d1, **kwargs) for _ in range(iterations)] result_d2 = [algorithm(d2, **kwargs) for _ in range(iterations)] if search_space is None: # determine the search space based on the return type # a subset of results to determine return type sub_result = result_d1 + result_d2 counter = Counter(sub_result) # categorical output if len(counter) < iterations * 0.02 * 0.1: search_space = tuple((key,) for key in counter.keys()) else: sub_result_sorted = np.sort(sub_result) average = np.average(sub_result_sorted) idx = np.searchsorted(sub_result_sorted, average, side='left') # find the densest 70% range search_min = int(idx - 0.35 * len(sub_result_sorted)) if int(idx - 0.4 * len(sub_result_sorted)) > 0 else 0 search_max = int(0.7 * len(sub_result_sorted) - (idx - search_min)) search_space = tuple(Interval((-float('inf'), alpha)) for alpha in np.linspace(sub_result_sorted[search_min], sub_result_sorted[search_max], num=25)) logger.info('search space is set to {0}'.format(search_space)) threshold = 0.001 * iterations * np.exp(epsilon) results = list(map(__EvaluateEvent(result_d1, result_d2, iterations), search_space)) if process_pool is None \ else process_pool.map(__EvaluateEvent(result_d1, result_d2, iterations), search_space) input_p_values = [test_statistics(cx, cy, epsilon, iterations) if cx + cy > threshold else float('inf') for (cx, cy) in results] for (s, (cx, cy), p) in zip(search_space, results, input_p_values): logger.debug('d1: %s | d2: %s | event: %s | p: %f | cx: %d | cy: %d | ratio: %f' % (d1, d2, s, p, cx, cy, float(cy) / cx if cx != 0 else float('inf'))) input_event_pairs.extend(list((d1, d2, kwargs, event) for event in search_space)) p_values.extend(input_p_values) # find an (d1, d2, kwargs, event) pair which has minimum p value from search space return input_event_pairs[np.argmin(p_values)]
def get_intervals(self): fh = gzip.GzipFile(self.fs_path, 'rb') try: info = whisper.__readHeader(fh) # evil, but necessary. finally: fh.close() start = time.time() - info['maxRetention'] end = max(os.stat(self.fs_path).st_mtime, start) return IntervalSet([Interval(start, end)])
def __init__(self): """line_map = dict() line_map['left'] = (-1.8, -5.4) line_map['mid'] = (-1.8, 1.8) line_map['right'] = (1.8, 5.4)""" self.lane_map = { 'pre_left': Interval((-9.0, -5.4)), 'left': Interval((-5.4, -1.8)), 'mid': Interval((-1.8, 1.8)), 'right': Interval((1.8, 5.4)), 'after_right': Interval((5.4, 9.0)) } self.test = Interval((-1, 1)) self.ego_pos_init = 'mid' self.left_lane = self.lane_map['left'] self.mid_lane = self.lane_map['mid'] self.right_lane = self.lane_map['right'] self.obj_pos = dict()
def main(): jobs = [ { "algorithm": noisy_max_v1a, "kwargs": {}, "databases": ([0] + [2 for _ in range(4)], [1 for _ in range(5)]), "search_space": tuple([i] for i in range(5)) }, { "algorithm": noisy_max_v1b, "kwargs": {}, "databases": ([2 for _ in range(5)], [1 for _ in range(5)]), "search_space": tuple(Interval([-inf, alpha]) for alpha in range(-5, 6)) }, { "algorithm": noisy_max_v2a, "kwargs": {}, "databases": ([0] + [2 for _ in range(4)], [1 for _ in range(5)]), "search_space": tuple([i] for i in range(5)) }, { "algorithm": noisy_max_v2b, "kwargs": {}, "databases": ([2] + [0 for _ in range(4)], [1 for _ in range(5)]), "search_space": tuple( Interval([-inf, 1 + alpha / 10.0]) for alpha in range(0, 80, 2)) }, { "algorithm": histogram, "kwargs": {}, "databases": ([2 for _ in range(5)], [1] + [2 for _ in range(4)]), "search_space": [Interval([-inf, alpha]) for alpha in range(-17, 17)] }, { "algorithm": histogram_eps, "kwargs": {}, "databases": ([0] + [1 for _ in range(4)], [1 for _ in range(5)]), "search_space": [Interval([-inf, alpha / 10]) for alpha in range(-30, 30, 2)] }, { "algorithm": iSVT1, "kwargs": { 'T': 1, 'N': 1 }, "databases": ([1 for _ in range(10)], [0 for _ in range(5)] + [2 for _ in range(5)]), "search_space": [[i] for i in range(10)] }, { "algorithm": iSVT2, "kwargs": { 'T': 1, 'N': 1 }, "databases": ([1 for _ in range(5)] + [0 for _ in range(5)], [0 for _ in range(5)] + [1 for _ in range(5)]), "search_space": [[i] for i in range(10)] }, { "algorithm": iSVT3, "kwargs": { 'T': 1, 'N': 1 }, "databases": ([1 for _ in range(5)] + [0 for _ in range(5)], [0 for _ in range(5)] + [1 for _ in range(5)]), "search_space": [[i] for i in range(10)] }, ] results = {} start_time = time.time() for job in jobs: results.clear() algorithm, search_space, = job['algorithm'], job['search_space'] databases, kwargs = job['databases'], job['kwargs'] for algorithm_epsilon in [0.2, 0.5, 0.7] + list(range(1, 4)): kwargs['epsilon'] = algorithm_epsilon results[algorithm_epsilon] = detect_counterexample( algorithm, [x / 10.0 for x in range(1, 34, 1)], kwargs, search_space, databases) draw_graph('Test $\epsilon$', 'P Value', results, algorithm.__name__.replace('_', ' ').title(), algorithm.__name__ + '.pdf') # dump the results to file with open('./{}.json'.format(algorithm.__name__), 'w') as f: # json.dump(results, f) # cannot use json.dump since Interval class is not JSON-serializable f.write(jsonpickle.encode(results)) print('{} | D1: {} | D2: {} | Time: {}'.format( algorithm.__name__, databases[0], databases[1], time.time() - start_time)) start_time = time.time()
def __init__(self, database, data=None, _id=None): # Create a model controller object. ModelController.__init__(self, 'target', database, data=data, _id=_id) self.queue = Queue(maxsize=0) self.camera = self.db.camera.find_one(qwrap(self.model['owner_id'])) self.interval = Interval(self)
class TargetController(ModelController): # Handles the individual timelapses, including creation and deletion. # A timelapse is a single collection of images from one location. def __init__(self, database, data=None, _id=None): # Create a model controller object. ModelController.__init__(self, 'target', database, data=data, _id=_id) self.queue = Queue(maxsize=0) self.camera = self.db.camera.find_one(qwrap(self.model['owner_id'])) self.interval = Interval(self) def start(self): # Begin capturing data from the microscope. int_data = {} int_data['time'] = self.model['time'] int_data['interval'] = self.model['interval'] int_data['source'] = self.model['source'] int_data['motor'] = self.model['motor'] self.interval.begin(int_data) self.model['start'] = time.time() self.model['end'] = time.time() + self.model['time'] self._update() self.camera['schedule'].put(self.model['cords']) self.db['camera'].update_one(qry(self.camera),\ { '$set': self.model['owner_id'] }) def add_image(self, image): # Add the image data to this object, and update the database. image_data = {} image_data['owner_id'] = self._id image_data['order'] = self.model['num_images'] self.model['num_images'] += 1 new_image = ImageController(self.db, data=image_data) new_image.write_image(image) self._update() def read(self): # Read the current timelapse from the database. return self._read() def get_lapse(self): # First, get and save all of the images stored in the queue. while not self.queue.empty(): self.add_image(self.queue.get()) # Return a list of all of the encoded images. cursor = self.db.image.find({'owner_id': self._id}) images = [] for img in cursor: images.append(pickle.loads(img['image'])) return images def get_latest(self): # First, get and save all of the images stored in the queue. while not self.queue.empty(): self.add_image(self.queue.get()) # Find the latest recorded image. # Current bug: when there are no images, sort throws an error # about NoneType objects (there are no images associated with this # target), so the server ends up serving nothing. latest = {} images = list(self.db.image.find({'owner_id': self._id})) if len(images) > 0: latest = self.db.image.find({ 'owner_id': self._id }).\ sort([( 'order', -1 )]).limit(1) return pickle.loads(latest['img']) else: return latest def delete(self): # First, delete all the images. self.db.images.delete_many({'owner_id': self._id}) # Finally, delete myself! self._delete() def get_status(self): # Return the status of the interval return [self.interval.status_message, self.interval.is_connected] def get_id(self): return self._id
def get_intervals(self): start = time.time() - self.get_retention(self.fs_path) end = max(os.stat(self.fs_path).st_mtime, start) return IntervalSet([Interval(start, end)])
def main(): # auto-generated data for sparsevector # D1 = [10.747997673023637, 9.176688735248929, 9.685779806342222, 9.276506422051003, 10.14197198493314, 10.777285751217708, 10.033518297158158, 10.739107887579431, 9.466024642462145, 10.046506781898248] # D2 = [9.74799, 9.999989999999999, 10.19486, 9.99999, 9.57099, 9.888639999999999, 9.08919, 9.76694, 9.99999, 9.73776] # auto-generated data for noisymax # D1 = [10.047838788362302, 9.156301318712105, 9.934576874381165, 10.818923545533439, 10.964414221707846, 9.814549818258723, 10.343685529387757, 10.270895438249106, 10.422325599687007, 9.498322242077997] # D2 = [11.04783, 10.10206, 10.85322, 10.04784, 10.04784, 10.40406, 10.04784, 9.664679999999999, 9.660929999999999, 10.06784] # manual data for niosymax version 1a and 2a #D1 = [0] + [2 for _ in range(4)] #D2 = [1 for _ in range(5)] # manual data for noisymax version 1b and 2b D1 = [2 for _ in range(5)] D2 = [1 for _ in range(5)] jobs = [{ "algorithm": noisy_max_v1a, "D1": lambda eps: [0] + [2 for _ in range(4)], "D2": lambda eps: [1 for _ in range(5)], "S": lambda: select_event(algorithm, (), kwargs, D1, D2, epsilon, search_space=[[i] for i in range(5)]) }, { "algorithm": noisy_max_v1b, "D1": lambda eps: [2 for _ in range(5)], "D2": lambda eps: [1 for _ in range(5)], "S": lambda: select_event( algorithm, (), kwargs, D1, D2, epsilon, search_space=[Interval([-inf, alpha]) for alpha in range(-5, 6)]) }, { "algorithm": noisy_max_v2a, "D1": lambda eps: [0] + [2 for _ in range(4)], "D2": lambda eps: [1 for _ in range(5)], "S": lambda: select_event(algorithm, (), kwargs, D1, D2, epsilon, search_space=[[i] for i in range(5)]) }, { "algorithm": noisy_max_v2b, "D1": lambda eps: [2] + [0 for _ in range(4)], "D2": lambda eps: [1 for _ in range(5)], "S": lambda: select_event(algorithm, (), kwargs, D1, D2, epsilon, search_space=[ Interval([-inf, 1 + alpha / 10.0]) for alpha in range(0, 80, 2) ]) }, { "algorithm": histogram, "D1": lambda eps: [2 for _ in range(5)], "D2": lambda eps: [1] + [2 for _ in range(4)], "S": lambda: select_event(algorithm, (), kwargs, D1, D2, epsilon, search_space=[ Interval([2 + alpha / 10.0, inf]) for alpha in range(0, 20, 2) ]) }] # manual data for sparsevector results = {} start_time = time.time() for job in jobs: results.clear() algorithm = job['algorithm'] for algorithm_epsilon in [0.2, 0.5, 0.7] + list(range(1, 4)): results[algorithm_epsilon] = [] for epsilon in [x / 10.0 for x in range(1, 34, 1)]: # algorithm's fixed privacy budget kwargs = {'eps': algorithm_epsilon} kwargs.update(job.get('kwargs', {})) D1 = job['D1'](epsilon) D2 = job['D2'](epsilon) # call s selector to find s S = job['S']() p1, p2 = hypothesis_test(algorithm, (), kwargs, D1, D2, S, epsilon, 100000, cores=0) results[algorithm_epsilon].append((epsilon, p1, p2)) print("epsilon: %f, p1 = %f, p2 = %f | S = %s" % (epsilon, p1, p2, S)) draw_graph('Test $\epsilon$', 'P Value', results, algorithm.__name__.replace('_', ' ').title(), algorithm.__name__ + '.pdf') # print output print("%s" % algorithm.__name__) print("D1 : %s" % D1) print("D2 : %s" % D2) print("Time elapsed: %f seconds" % (time.time() - start_time))
def test_Interval_Instantiation(self): with self.assertRaises(TypeError): Interval(1, 3)
def get_intervals(self): start = 0 end = int(time.time()) return IntervalSet([Interval(start, end)])
def convert_integer_open_to_closed(possibly_open: i.Interval): if possibly_open.left == i.OPEN: possibly_open = possibly_open.replace(left=i.CLOSED, lower=lambda x: x + 1) if possibly_open.right == i.OPEN and possibly_open.upper != i.inf: possibly_open = possibly_open.replace(right=i.CLOSED, upper=lambda x: x - 1) return possibly_open
def find(self, pattern, startTime=None, endTime=None, local=False): query = FindQuery(pattern, startTime, endTime) # Start remote searches if not local: remote_requests = [r.find(query) for r in self.remote_stores if r.available] matching_nodes = set() # Search locally for finder in self.finders: for node in finder.find_nodes(query): #log.info("find() :: local :: %s" % node) matching_nodes.add(node) # Gather remote search results if not local: for request in remote_requests: for node in request.get_results(): #log.info("find() :: remote :: %s from %s" % (node,request.store.host)) matching_nodes.add(node) # Group matching nodes by their path nodes_by_path = {} for node in matching_nodes: if node.path not in nodes_by_path: nodes_by_path[node.path] = [] nodes_by_path[node.path].append(node) # Reduce matching nodes for each path to a minimal set found_branch_nodes = set() for path, nodes in nodes_by_path.iteritems(): leaf_nodes = [] # First we dispense with the BranchNodes for node in nodes: if node.is_leaf: leaf_nodes.append(node) elif node.path not in found_branch_nodes: #TODO need to filter branch nodes based on requested interval... how?!?!? yield node found_branch_nodes.add(node.path) if not leaf_nodes: continue # Calculate best minimal node set minimal_node_set = set() covered_intervals = IntervalSet([]) # If the query doesn't fall entirely within the FIND_TOLERANCE window # we disregard the window. This prevents unnecessary remote fetches # caused when carbon's cache skews node.intervals, giving the appearance # remote systems have data we don't have locally, which we probably do. now = int(time.time()) tolerance_window = now - settings.FIND_TOLERANCE disregard_tolerance_window = query.interval.start < tolerance_window prior_to_window = Interval(float('-inf'), tolerance_window) def measure_of_added_coverage(node, drop_window=disregard_tolerance_window): relevant_intervals = node.intervals.intersect_interval(query.interval) if drop_window: relevant_intervals = relevant_intervals.intersect_interval(prior_to_window) return covered_intervals.union(relevant_intervals).size - covered_intervals.size nodes_remaining = list(leaf_nodes) # Prefer local nodes first (and do *not* drop the tolerance window) for node in leaf_nodes: if node.local and measure_of_added_coverage(node, False) > 0: nodes_remaining.remove(node) minimal_node_set.add(node) covered_intervals = covered_intervals.union(node.intervals) while nodes_remaining: node_coverages = [(measure_of_added_coverage(n), n) for n in nodes_remaining] best_coverage, best_node = max(node_coverages) if best_coverage == 0: break nodes_remaining.remove(best_node) minimal_node_set.add(best_node) covered_intervals = covered_intervals.union(best_node.intervals) # Sometimes the requested interval falls within the caching window. # We include the most likely node if the gap is within tolerance. if not minimal_node_set: def distance_to_requested_interval(node): latest = sorted(node.intervals, key=lambda i: i.end)[-1] distance = query.interval.start - latest.end return distance if distance >= 0 else float('inf') best_candidate = min(leaf_nodes, key=distance_to_requested_interval) if distance_to_requested_interval(best_candidate) <= settings.FIND_TOLERANCE: minimal_node_set.add(best_candidate) if len(minimal_node_set) == 1: yield minimal_node_set.pop() elif len(minimal_node_set) > 1: reader = MultiReader(minimal_node_set) yield LeafNode(path, reader)
def test_guesses_types(self, number_range, type): assert Interval(number_range).type == type
from intervals import Interval import threading # 5 seconds stop_time = 10 # 1 image per second interval = 2 # From the webcam source = 0 Interval(stop_time, interval, source) Interval(5, 1, 0) print(threading.active_count()) print("done!")
def if_unsafe(self): if self.state[0].cpu().data in Interval((self.x0_low, self.x0_high)) and \ self.state[1].cpu().data in Interval((self.x1_low, self.x1_high)): return 0 else: return 1
def get_intervals(self): start = time.time() - whisper.info(self.fs_path)['maxRetention'] end = max(os.stat(self.fs_path).st_mtime, start) return IntervalSet([Interval(start, end)])