def test_copy(): alpha = SortedList(range(100)) alpha._reset(7) beta = alpha.copy() alpha.add(100) assert len(alpha) == 101 assert len(beta) == 100
class Episode_scores(object): def __init__(self, options): self.maxlen = options.score_averaging_length self.threshold = options.score_highest_ratio self.episode_scores = deque() self.episode_scores.append(0) # to avoid 0-div in first averaging self.episode_scores_sum = 0 self.sorted_scores = SortedList() self.sorted_scores.add(0) # align to episode_scores self.num_episode = 0 self.options = options def add(self, n, global_t, thread_index): self.episode_scores_sum += n self.episode_scores.append(n) self.sorted_scores.add(-n) # trick to use SortedList in reverse order if len(self.episode_scores) > self.maxlen: oldest = self.episode_scores.popleft() self.sorted_scores.remove(-oldest) self.episode_scores_sum -= oldest self.num_episode += 1 if self.num_episode % self.options.average_score_log_interval == 0: print("@@@ Average Episode score = {:.6f}, s={:9d},th={}".format(self.average(), global_t, thread_index)) def average(self): return self.episode_scores_sum / len(self.episode_scores) def is_highscore(self, n): sorted_scores = self.sorted_scores num_scores = len(sorted_scores) sorted_scores.add(-n) index = sorted_scores.index(-n) highest_ratio = (index + 1) / num_scores sorted_scores.remove(-n) return highest_ratio <= self.threshold
def extract_collocations(self, metric_class): assert issubclass(metric_class, Metric) metric = metric_class() collocations = SortedList(key=lambda x: -x[0]) unigram_counts = self.language_model.get_unigrams() bigram_counts = self.language_model.get_bigrams() for (first, last), freq_bigram in bigram_counts.items(): if self.exclude_punctuation: if first in self.PUNCT or last in self.PUNCT: continue if self.exclude_conj: if first in self.CONJ_RU or last in self.CONJ_RU: continue if self.exclude_props: if first in self.PROPOSITIONS_RU or last in self.PROPOSITIONS_RU: continue freq_first, freq_last = unigram_counts[first], unigram_counts[last] metric_val = metric.evaluate(freq_first, freq_last, freq_bigram, self.language_model.get_vocab_size()) collocations.add((metric_val, freq_first, freq_last, freq_bigram, first, last)) return collocations
def predict(self, X): y = np.zeros(len(X)) for i,x in enumerate(X): # test points sl = SortedList(load=self.k) # stores (distance, class) tuples for j,xt in enumerate(self.X): # training points diff = x - xt d = diff.dot(diff) if len(sl) < self.k: # don't need to check, just add sl.add( (d, self.y[j]) ) else: if d < sl[-1][0]: del sl[-1] sl.add( (d, self.y[j]) ) # print "input:", x # print "sl:", sl # vote votes = {} for _, v in sl: # print "v:", v votes[v] = votes.get(v,0) + 1 # print "votes:", votes, "true:", Ytest[i] max_votes = 0 max_votes_class = -1 for v,count in votes.iteritems(): if count > max_votes: max_votes = count max_votes_class = v y[i] = max_votes_class return y
def iana_rir_gen_ip_list(user_rir_list): # generates a list of networks that can be blocked by RIR # we use a SortedList so that elements are inserted in order. This allows cidr_merge to work rir_slash_eight_list = SortedList() with open('iana') as iana_file: iana_csv = csv.reader(iana_file) for line in iana_csv: for rir in user_rir_list: # case in which the whois line from our csv contains the RIR if rir in line[3]: network = line[0].lstrip('0') rir_slash_eight_list.add(netaddr.IPNetwork(network)) # if we find a match, there is no reason to see if the other RIRs are on the same line break # run cidr_merge to summarize rir_slash_eight_list = netaddr.cidr_merge(rir_slash_eight_list) return rir_slash_eight_list
def test_copy_copy(): import copy alpha = SortedList(range(100), load=7) beta = copy.copy(alpha) alpha.add(100) assert len(alpha) == 101 assert len(beta) == 100
def score_of_a_vacated_people(self, universo, work='translations'): factor = math.sqrt(len(universo)) scores = SortedList(load=round(factor)) for (people, score) in self.__scores__().items(): if people in universo: scores.add(TranslatorScore(people, score[work])) return scores.pop(0)
def find_latest(self): sorted = SortedList() for i in self.bucket.list(prefix=self.db_name): parts = i.name.split('/') if len(parts) == 3: d = datetime.datetime.strptime(parts[1], "%m%d%Y").date() sorted.add(d) return sorted[len(sorted)-1].strftime('%m%d%Y')
class InMemoryBackend(object): """ The backend that keeps the results in the memory. """ def __init__(self, *args, **kwargs): def get_timestamp(result): return timestamp_parser.parse(result['timestamp']) self._results = dict() self._sorted = SortedList(key=get_timestamp) def disconnect(self): return succeed(None) def store(self, result): """ Store a single benchmarking result and return its identifier. :param dict result: The result in the JSON compatible format. :return: A Deferred that produces an identifier for the stored result. """ id = uuid4().hex self._results[id] = result self._sorted.add(result) return succeed(id) def retrieve(self, id): """ Retrive a result by the given identifier. """ try: return succeed(self._results[id]) except KeyError: return fail(ResultNotFound(id)) def query(self, filter, limit=None): """ Return matching results. """ matching = [] for result in reversed(self._sorted): if len(matching) == limit: break if filter.viewitems() <= result.viewitems(): matching.append(result) return succeed(matching) def delete(self, id): """ Delete a result by the given identifier. """ try: result = self._results.pop(id) self._sorted.remove(result) return succeed(None) except KeyError: return fail(ResultNotFound(id))
def read_rirs(country_list, permit, rir_list=RIR_NAMES): # list containing our file objects file_list = [] # we use a SortedList so that elements are inserted in order. This allows cidr_merge to work rir_ips = SortedList() # Open the files we downloaded earlier and store the file object for rir in rir_list: file_list.append(open(rir)) for f in file_list: for line in f: curr_line = line.split('|') try: # we want only the ipv4 lines that are for a specific country # also only want countries that we are going to block if (curr_line[2] == "ipv4" and curr_line[1] != "*") and \ ((permit and curr_line[1] not in country_list) or (not permit and curr_line[1] in country_list)): country_code = curr_line[1] network_id = curr_line[3] wildcard = int(curr_line[4])-1 try: # Add network to list, if the number of IPs was not a # power of 2 (wildcard is not valid). # AddrFormatError is thrown rir_ips.add(netaddr.IPNetwork(network_id + "/" + str(netaddr.IPAddress(wildcard)))) # Handle case in where our mask is invalid by rounding DOWN except netaddr.AddrFormatError: print "rounded network " + network_id + " with " + str(wildcard) + \ " hosts up to nearest power of 2" wildcard = next_power_of_2(wildcard) - 1 print wildcard + 1 rir_ips.add(netaddr.IPNetwork(network_id + "/" + str(netaddr.IPAddress(wildcard)))) # IndexErrors only occur when parsing columns we don't need except IndexError: pass f.close() # cidr_merge takes our list of IPs and summarizes subnets where possible # this greatly decreases the number of ACL entries rir_ips = netaddr.cidr_merge(rir_ips) return rir_ips
def dir(self, file_pattern): attrs = self.sftp.listdir_attr(self.remote_dir) filtered = SortedList() for attr in attrs: if hasattr(attr, "filename"): filename = attr.filename if re.match(file_pattern, filename): remote_file = RemoteFile(filename, attr.st_mtime) filtered.add(remote_file) return filtered
def test_count(): slt = SortedList(load=7) assert slt.count(0) == 0 for iii in range(100): for jjj in range(iii): slt.add(iii) slt._check() for iii in range(100): assert slt.count(iii) == iii
def arrayRDP(arr, epsilon=0.0, n=None): """ This is a slightly modified version of the _aRDP function, that accepts as arguments the tolerance in the distance and the maximum number of points the algorithm can select. **Note:** The results of this algoritm should be identical to the arrayRDP function if the *n* parameter is not specified. In that case, the performance is slightly worse, although the asymptotic complexity is the same. For this reason, this function internally delegates the solution in that function if the *n* parameter is missing. Parameters ---------- arr: Array of values of consecutive points. epsilon: Maximum difference allowed in the simplification process. n: Maximum number of points of the resulted simplificated array. Returns ------- out: Array of indices of the selected points. """ if n is None: return _aRDP(arr, epsilon) if epsilon <= 0.0: raise ValueError('Epsilon must be > 0.0') n = n or len(arr) if n < 3: return arr fragments = SortedDict() #We store the distances as negative values due to the default order of #sorteddict dist, idx = max_vdist(arr, 0, len(arr) - 1) fragments[(-dist, idx)] = (0, len(arr) - 1) while len(fragments) < n-1: (dist, idx), (first, last) = fragments.popitem(last=False) if -dist <= epsilon: #We have to put again the last item to prevent loss fragments[(dist, idx)] = (first, last) break else: #We have to break the fragment in the selected index dist, newidx = max_vdist(arr, first, idx) fragments[(-dist, newidx)] = (first, idx) dist, newidx = max_vdist(arr, idx, last) fragments[(-dist, newidx)] = (idx, last) #Now we have to get all the indices in the keys of the fragments in order. result = SortedList(i[0] for i in fragments.itervalues()) result.add(len(arr) - 1) return np.array(result)
def test_getitem(): random.seed(0) slt = SortedList(load=17) lst = list() for rpt in range(100): val = random.random() slt.add(val) lst.append(val) lst.sort() assert all(slt[idx] == lst[idx] for idx in range(100)) assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def get_3_most_ambiguous(self, X, Y): P = self.predict_proba(X) N = len(X) sl = SortedList(load=3) # stores (distance, sample index) tuples for n in xrange(N): p = P[n] dist = np.abs(p - 0.5) if len(sl) < 3: sl.add( (dist, n) ) else: if dist < sl[-1][0]: del sl[-1] sl.add( (dist, n) ) indexes = [v for k, v in sl] return X[indexes], Y[indexes]
class _WaitingTasksQueue(object): """A FIFO queue of tasks that keeps track of RAM limits. This class serves two purposes: 1) It is a FIFO queue for tasks that also supports fast deletion. 2) It keeps RAM requirements of all tasks in the queue in sorted order. This is necessary for the implementation of worker blocking, see ``PrioritingScheduler._getNumberOfBlockedAnyCpuWorkers()`` for more details. """ def __init__(self): self._dict = OrderedDict() self._tasks_required_ram = SortedList() def __contains__(self, task): return task in self._dict def __len__(self): return len(self._dict) def __nonzero__(self): return bool(self._dict) __bool__ = __nonzero__ # for Python 2/3 compatibility def add(self, task): self._dict[task] = True self._tasks_required_ram.add(task.required_ram_mb) def remove(self, task): del self._dict[task] self._tasks_required_ram.discard(task.required_ram_mb) def left(self): if self._dict: return six.next(six.iteritems(self._dict))[0] else: return None def popleft(self): task = self._dict.popitem(False)[0] self._tasks_required_ram.discard(task.required_ram_mb) return task def getTasksRequiredRam(self): return self._tasks_required_ram
def load_waveforms(annfile): """Obtains a sorted list of waveform objects from an annotations file""" waveforms = SortedList() anns = mit.read_annotations(annfile) for i in xrange(len(anns)): a = anns[i] if mit.is_qrs_annotation(a): wf = WaveForm() wf.type = mit.ECGCodes.NORMAL wf.peak = a.time start = next(anns[j].time for j in xrange(i, -1, -1) if anns[j].code is mit.ECGCodes.WFON) end = next(anns[j].time for j in xrange(i, len(anns)) if anns[j].code is mit.ECGCodes.WFOFF) wf.interval = Iv(start, end) waveforms.add(wf) return waveforms
def test_stress(repeat=1000): slt = SortedList((random.random() for rpt in range(1000))) slt._reset(23) for rpt in range(repeat): action = random.choice(actions) action(slt) slt._check() fourth = int(len(slt) / 4) count = 0 if fourth == 0 else random.randrange(-fourth, fourth) while count > 0: slt.add(random.random()) count -= 1 while count < 0: pos = random.randrange(len(slt)) del slt[pos] count += 1 while len(slt) > 2000: # Shorten the sortedlist. This maintains the "jaggedness" # of the sublists which helps coverage. pos = random.randrange(len(slt._maxes)) del slt._maxes[pos] del slt._lists[pos] slt._len = sum(len(sublist) for sublist in slt._lists) slt._index = [] slt._check() slt._check() slt._check() stress_update(slt) while len(slt) > 0: pos = random.randrange(len(slt)) del slt[pos] slt._check()
class Store: def __init__(self,N=10): self.store = SortedList() self.N = N def add(self,item): self.store.add(item) if len(self.store) > self.N: self.store.pop(0) def pop(self,i): self.store.pop(i) def __len__(self): return len(self.store) def __getitem__(self,i): return self.store[i] def __str__(self): return str(self.store)
def _getOpcodeAddrs(self): addrs = SortedList() funcs = self.getFunctions() for f in funcs: print('addrs for', f['name']) ofs = f['offset'] sz = f['size'] end = ofs + sz self.seek(ofs) cur = ofs while cur < end: addrs.add(cur) self.cmd('so') cur = self.tell() return addrs
def ToEdgeClusters(edgeListPath, clustersPath): ''' Convert a node clusters file to an edge clusters path. ''' # Load the edge list and clusters as a *sorted* list of lists of tuples edgeList = LoadNumberFile(edgeListPath) edgeClusters = SortedList() with open(clustersPath, 'r') as f: for l in f: nodes = [int(x) for x in l.strip().split()] cluster = SortedList() for e in combinations(sorted(nodes), 2): if edgeList.count([e]) is not 0: cluster.add(e) edgeClusters.add(cluster) with open(clustersPath, 'w') as f: for c in edgeClusters: for t in c: f.write(str(t[0]) + " " + str(t[1]) + " ") f.write("\n")
def collect_matches(): initial_summoner_name = "GustavEnk" region = "EUW" summoner = Summoner(name=initial_summoner_name, region=region) patch = Patch.from_str("8.9", region=region) unpulled_summoner_ids = SortedList([summoner.id]) pulled_summoner_ids = SortedList() unpulled_match_ids = SortedList() pulled_match_ids = SortedList() while unpulled_summoner_ids: # Get a random summoner from our list of unpulled summoners and pull their match history new_summoner_id = random.choice(unpulled_summoner_ids) new_summoner = Summoner(id=new_summoner_id, region=region) matches = filter_match_history(new_summoner, patch) unpulled_match_ids.update([match.id for match in matches]) unpulled_summoner_ids.remove(new_summoner_id) pulled_summoner_ids.add(new_summoner_id) while unpulled_match_ids: # Get a random match from our list of matches new_match_id = random.choice(unpulled_match_ids) new_match = Match(id=new_match_id, region=region) for participant in new_match.participants: if participant.summoner.id not in pulled_summoner_ids and participant.summoner.id not in unpulled_summoner_ids: unpulled_summoner_ids.add(participant.summoner.id) # The above lines will trigger the match to load its data by iterating over all the participants. # If you have a database in your datapipeline, the match will automatically be stored in it. unpulled_match_ids.remove(new_match_id) pulled_match_ids.add(new_match_id)
def test_getitem_slice(): random.seed(0) slt = SortedList() slt._reset(17) lst = list() for rpt in range(100): val = random.random() slt.add(val) lst.append(val) lst.sort() assert all(slt[start:] == lst[start:] for start in [-75, -25, 0, 25, 75]) assert all(slt[:stop] == lst[:stop] for stop in [-75, -25, 0, 25, 75]) assert all(slt[::step] == lst[::step] for step in [-5, -1, 1, 5]) assert all(slt[start:stop] == lst[start:stop] for start in [-75, -25, 0, 25, 75] for stop in [-75, -25, 0, 25, 75]) assert all(slt[:stop:step] == lst[:stop:step] for stop in [-75, -25, 0, 25, 75] for step in [-5, -1, 1, 5]) assert all(slt[start::step] == lst[start::step] for start in [-75, -25, 0, 25, 75] for step in [-5, -1, 1, 5]) assert all(slt[start:stop:step] == lst[start:stop:step] for start in [-75, -25, 0, 25, 75] for stop in [-75, -25, 0, 25, 75] for step in [-5, -1, 1, 5])
def _get_split_point_candidates(self): splits = SortedList() min_value = math.inf max_value = -math.inf for class_val, att_estimator in self._class_lookup.items(): min_val_observed_for_class_val = self._min_val_observed_per_class.get(class_val, None) if min_val_observed_for_class_val is not None: if min_val_observed_for_class_val < min_value: min_value = min_val_observed_for_class_val max_val_observed_for_class_val = self._max_val_observed_per_class.get(class_val) if max_val_observed_for_class_val > max_value: max_value = max_val_observed_for_class_val if min_value < math.inf: new_bin = max_value - min_value new_bin /= (self._num_bins + 1) for i in range(self._num_bins): split = min_value + (new_bin * (i + 1)) if split > min_value and split < max_value: splits.add(split) return splits
class Rational: def __init__(self): self.numerator = SortedList() self.denominator = SortedList() def multiply_by(self,f): self.numerator.update(primefac.primefac(f)) def divide_by(self,d): self.denominator.update(primefac.primefac(d)) def value(self): if len(self.numerator) == 0 or len(self.denominator) == 0: return None numerator_index = 0 denominator_index = 0 while numerator_index < len(self.numerator) and denominator_index < len(self.denominator): if self.numerator[numerator_index] == self.denominator[denominator_index]: del self.numerator[numerator_index] del self.denominator[denominator_index] elif self.numerator[numerator_index] < self.denominator[denominator_index]: numerator_index += 1 else: denominator_index += 1 self.numerator.add(1) self.denominator.add(1) num_product = reduce(lambda x, y: mpfr(x)*y, self.numerator) den_product = reduce(lambda x, y: mpfr(x)*y, self.denominator) if num_product <= 0 or den_product <= 0: return 0 val = num_product/den_product if val > 1: return 1 return val
def __core_distance(self): if len(self.neighbour_coordinates()) < min_pts: return -1 else: stop_on_next_it = False for search_radius in range(1, eps + 2): visited = SortedList([]) x_range = range(max(0, self.coordinates[0] - search_radius), min(mask.shape[0], self.coordinates[0] + search_radius)) y_range = range(max(0, self.coordinates[1] - search_radius), min(mask.shape[1], self.coordinates[1] + search_radius)) z_range = range(max(0, self.coordinates[2] - search_radius), min(mask.shape[2], self.coordinates[2] + search_radius)) for x in x_range: for y in y_range: for z in z_range: if mask_data[(x,y,z)] > 0: if m.fabs(data[self.coordinates] - data[(x,y,z)]) <= max_difference: visited.add(distance(self.coordinates, (x,y,z))) if stop_on_next_it: return visited[min_pts - 1] if len(visited) >= min_pts: stop_on_next_it = True
class ChatThread(object): """ Represents a chat thread between the owner of the history and a list of participants. Messages are stored in sorted order. """ def __init__(self, participants): self.participants = list(participants) self.participants.sort() self.messages = SortedList() def add_message(self, message): """ Adds a message to the chat thread. message -- the message to add """ self.messages.add(message) def __lt__(self, other): return len(self.messages) < len(other.messages) def __len__(self): return len(self.messages)
def test_add(): random.seed(0) slt = SortedList() for val in range(1000): slt.add(val) slt._check() slt = SortedList() for val in range(1000, 0, -1): slt.add(val) slt._check() slt = SortedList() for val in range(1000): slt.add(random.random()) slt._check()
def test_copy(): alpha = SortedList(range(100), load=7) beta = alpha.copy() alpha.add(100) assert len(alpha) == 101 assert len(beta) == 100
class Solver(object): def __init__(self, start: Tree, goal: Tree, rules=DEFAULT_RULES): self.heuristic = SimpleNodeAmountHeuristic self.start = start self.goal = goal cost = 0.0 h = self.heuristic.estimate(start, goal) self.nr_steps_made: int = 0 solution = SolutionStep(start, cost, h, self.nr_steps_made) self.steps = SortedList([solution]) self.solution_hash_dict = {solution.root_hash: solution} self.rules = sorted( rules, key=lambda x: x.cost, ) self.stats = { "operations_checked": 0, "operations_applied": 0, } self.history: List[Tuple[str, str, float, float, float]] = [] def solve(self) -> Tree: if self.start == self.goal: return self.start while True: current_step, current_rule, current_nid = self._suggest_next_step() if current_step is None: raise UnableToFindSolutionCalculatorError( "We tried all possible operations and could not find solution." ) current_tree = current_step.tree self.stats["operations_applied"] += 1 current_hash = hash(current_tree.root) new_tree, operation_cost, typical_mistakes = current_rule.apply(current_tree, current_nid) new_tree_cost = current_step.cost + operation_cost assert hash(current_tree.root) == current_hash # Added to ensure that trees are not mutable self._update(new_tree, new_tree_cost, current_tree, current_rule, current_nid, typical_mistakes) if new_tree.root.equals(self.goal.root): return new_tree def get_best_path(self, tree: Tree) -> List[PathStep]: """get path (as lit of operations) from beginning to a given tree in the forest""" step = self.solution_hash_dict[hash(tree.root)] operations: List[PathStep] = [] while step.step_before: tree_before = step.step_before assert step.rule_before is not None assert step.nid_before is not None operations.append( PathStep( step.tree, tree_before.tree, step.rule_before.__class__, step.nid_before, step.cost, step.h, step.typical_mistakes, ) ) step = tree_before operations.reverse() return operations def get_solution_steps(self, tree: Tree) -> SimplificationResult: path = self.get_best_path(tree) steps: List[SimplificationResultStep] = [] for step in path: typical_mistakes = [ SimplificationResultMistake(type=m.name, value=printer.do_print(m.full_error_expr)) for m in step.typical_mistakes ] steps.append( SimplificationResultStep( rule_name=step.rule.__name__, formula=printer.do_print(step.tree_current.root), typical_mistakes=typical_mistakes, ) ) return SimplificationResult(steps=steps) def _suggest_next_step( self, ) -> Tuple[ SolutionStep, BaseSolverRule, int ]: # TODO: change to prefer nodes around nodes that were modified in the previous step for step in self.steps: if step.fully_visited: continue for rule in self.rules: rule_name = type(rule).__name__ if rule_name not in step.visited_operations: step.visited_operations[rule_name] = [] else: continue for nid, node in step.tree.expand_tree(): node_type = type(node) if node_type == rule.root_operation and nid not in step.visited_operations[rule_name]: step.visited_operations[rule_name].append(nid) self.stats["operations_checked"] = self.stats["operations_checked"] + 1 if rule.check_condition(step.tree, nid): return step, rule, nid step.fully_visited = True raise UnableToFindSolutionCalculatorError("We tried all possible operations and could not find solution.") def _update( self, new_tree: Tree, new_tree_cost: float, tree_before: Tree, rule_before: BaseSolverRule, nid_before: int, typical_mistakes: List[TypicalMistake], ) -> None: """Add new calculated element to the path or, if it already exists update the cost""" root_hash = hash(new_tree.root) h = self.heuristic.estimate(new_tree, self.goal) if root_hash in self.solution_hash_dict: existing_step = self.solution_hash_dict[root_hash] if existing_step.priority <= new_tree_cost + h: return None raise UnableToFindSolutionCalculatorError("Finding solution was too difficult.") history_entry = ( printer.do_print(new_tree.root), rule_before.__class__.__name__, new_tree_cost, h, new_tree_cost + h, ) self.history.append(history_entry) step_before = self.solution_hash_dict[hash(tree_before.root)] self.nr_steps_made += 1 step = SolutionStep( new_tree, new_tree_cost, h, self.nr_steps_made, step_before, rule_before, nid_before, typical_mistakes ) self.steps.add(step) self.solution_hash_dict[step.root_hash] = step
class ColorTransferFunctionViewer(DataViewer): def __init__(self, data: ColorTransferFunction = None): super().__init__(data) self.chart = ColorTransferFunctionChart(data) # self.chart.legend().hide() self.chart_rect_f = QRectF() self.axis_x = QtCharts.QValueAxis() # self.axis_x.setLabelFormat('%d') self.axis_x.setLabelFormat('%.1f') self.axis_x.setTitleText('Intensity') self.chart.addAxis(self.axis_x, Qt.AlignBottom) self.axis_y = QtCharts.QValueAxis() # self.axis_y.setTickCount(10) self.axis_y.setLabelFormat('%.2f') # self.axis_y.setTitleText('Magnitude') self.chart.addAxis(self.axis_y, Qt.AlignLeft) self.axis_x.setRange(0, self.data.points[-1].x) self.axis_y.setRange(0, 1) # Add an empty series, else |chart.mapToPosition| will no work self.series = self.add_series() self.chart_view = QtCharts.QChartView(self.chart) # self.chart_view.setRubberBand(QtCharts.QChartView.RectangleRubberBand) self.chart_view.setRenderHint(QPainter.Antialiasing) self.scene = self.chart_view.scene() self._interval_views = SortedList() self._point_views = [] if self.data is not None: self._add_interval_views() self._add_point_views() self.data.point_added.connect(self._on_point_added) grid_layout = QGridLayout() grid_layout.setContentsMargins(0, 0, 0, 0) grid_layout.addWidget(self.chart_view) self.setLayout(grid_layout) def _add_interval_views(self): for point, next_point in pairwise(self.data.points): if next_point is not None: self._add_interval_view(point, next_point) def _add_interval_view(self, begin_point: ColorTransferFunctionPoint, end_point: ColorTransferFunctionPoint) \ -> ColorTransferFunctionIntervalView: interval_view = ColorTransferFunctionIntervalView( self, begin_point, end_point, self.chart) interval_view.setZValue(10) # To display item on top of chart grid self.scene.addItem(interval_view) self._interval_views.add(interval_view) return interval_view def _add_point_views(self): for point in self.data.points: self._add_point_view(point) def _add_point_view( self, point: ColorTransferFunctionPoint ) -> ColorTransferFunctionPointView: point_view = ColorTransferFunctionPointView(self, point, self.chart) point_view.setZValue( 11) # To display item on top of chart grid and interval views self.scene.addItem(point_view) self._point_views.append(point_view) return point_view def _update_chart_size(self): top_left_pos = self.chart.mapToPosition( QPointF(self.axis_x.min(), self.axis_y.max())) bottom_right_pos = self.chart.mapToPosition( QPointF(self.axis_x.max(), self.axis_y.min())) self.chart_rect_f = QRectF(top_left_pos, bottom_right_pos) def _on_point_added(self, point: ColorTransferFunctionPoint): self._add_point_view(point) # Update existed interval views point_index = self.data.points.index(point) if 0 < point_index < len( self.data.points ) - 1: # Only if point was added between other existed points # Exclude cases, where point was added before first and after last points before_point_index = point_index - 1 before_point_interval = self._interval_views[before_point_index] before_point_interval.end_point = point before_point_interval.update() # Add new interval view if point_index == len( self.data.points ) - 1: # If was added last point (point after last interval) new_interval_view = self._add_interval_view( self.data.point_before(point), point) new_interval_view.update() else: new_interval_view = self._add_interval_view( point, self.data.point_after(point)) new_interval_view.update() def resizeEvent(self, resize_event: QResizeEvent): self._update_chart_size() # min_tick_count = self.axis_x.max() - self.axis_x.min() + 1 # tick_count = min(min_tick_count, self.width() / 50) tick_count = self.chart_rect_f.width() / 50 self.axis_x.setTickCount(round(tick_count)) self.axis_y.setTickCount(round(self.chart_rect_f.height() / 20)) for interval_view in self._interval_views: interval_view.update() for point_view in self._point_views: point_view.update_size() point_view.update_pos() def add_series(self): series = QtCharts.QLineSeries() series.setName('Color Transfer Function') self.chart.addSeries(series) series.attachAxis(self.axis_x) series.attachAxis(self.axis_y) return series
class Book: """ Represent bid / ask book. * Book only allows one user order at a time * User order will not affect book statistics like quote and volume """ def __init__(self, side: str, key_func: Optional[Callable[[int], int]]) -> None: self.side = side self.key_func = key_func if key_func else lambda x: x # We need this because price levels follow price priority not time priority (which dict alone can provides) self.prices = SortedList(key=key_func) # Sorted prices self.price_levels: Dict[int, PriceLevel] = {} # Price to level map self.order_pool: Dict[int, PriceLevel] = {} # Order ID to level map # Store order price and PriceLevel. We do not need ID since there is only one order self.user_order_info: Optional[Tuple[int, PriceLevel]] = None self._front_idx: Optional[int] = None def reset(self): self.prices.clear() self.price_levels.clear() self.order_pool.clear() self.user_order_info = None self._front_idx = None # ========== Order Operations ========== def add_limit_order(self, order: LimitOrder) -> None: """ Add limit order to the correct price level """ if order.id in self.order_pool: raise RuntimeError(f'LimitOrder {order.id} already exists') self.order_pool[order.id] = self._get_price_level(order.price, force_index=True).add_limit_order(order) def match_limit_order(self, market_order: MarketOrder) -> Tuple[bool, Optional[Execution]]: """ Match environment order against limit order. Remove empty price level where needed """ # Sometime environment order may not follow time priority. We should follow the referenced order ID in this case user_order = None target_price_level = self.order_pool[market_order.id] # User orders may create price levels that do not exist in the real market. Need to match against those first if target_price_level.price != self.prices[0]: top_level = self.price_levels[self.prices[0]] if top_level.shares > 0: # Shares > 0 means that there are real LimitOrder exists in the top level raise RuntimeError('Market order being matched against levels not in the front') user_order = top_level.pop_user_order() self._remove_price_level_if_empty(top_level) # Now get the user orders that are in front of the matched real LimitOrder price_level, exhausted, executed_order = target_price_level.match_limit_order(market_order) self._remove_price_level_if_empty(price_level) # It can be that both order are None if executed_order is not None: user_order = executed_order # Whether the matching limit order is already exhausted if exhausted: del self.order_pool[market_order.id] # Update user order pool and return executions return exhausted, self._handle_matched_user_limit_order(user_order) if user_order else None def cancel_order(self, order: CancelOrder) -> None: """ Cancel (partial) shares of a LimitOrder """ self.order_pool[order.id].cancel_order(order) def delete_order(self, order: DeleteOrder): """ Delete the whole LimitOrder """ price_level = self.order_pool[order.id].delete_order(order) del self.order_pool[order.id] self._remove_price_level_if_empty(price_level) # ========== User Order Operation ========== def add_user_limit_order(self, order: UserLimitOrder) -> None: """ Add user limit order to the correct price level * Remove the old user order if exists * We do not want to deal with time priority because * This simplifies the flow * Last action's effect will spill over to the current one """ if self.user_order_info: original_id, price_level = self.user_order_info price_level.pop_user_order() # Only one user order is allowed self._remove_price_level_if_empty(price_level) self.user_order_info = order.price, self._get_price_level(order.price).add_user_limit_order(order) def match_limit_order_for_user(self, order: UserMarketOrder) -> Execution: """ Match LimitOrder for UserMarketOrder """ if self.user_order_info: raise RuntimeError('Cannot execute MarketOrder on the side that also has user LimitOrder') total_value = 0 shares = 0 # Recall that we are not actually matching the LimitOrders. No need to remove the executed LimitOrder. for price in self.prices: executed = order.shares - self.price_levels[price].match_limit_order_for_user(order) total_value += price * executed shares += executed if order.shares == 0: break if order.shares > 0: raise RuntimeError('User market order cannot be fully executed') return Execution(order.id, int(total_value / shares), shares if order.side == 'B' else -shares) def delete_user_order(self): """ Remove user order """ if self.user_order_info: _, price_level = self.user_order_info price_level.pop_user_order() self._remove_price_level_if_empty(price_level) self.user_order_info = None def resolve_book_crossing_on_user_order(self, price: int) -> Optional[Execution]: """ User orders may be placed inside the real market, in which case the newly added real order may cross with the user orders. When this happens, we assume that the user orders are executed """ signed_price = self.key_func(price) quote = self.quote if quote and self.key_func(quote) <= signed_price: raise RuntimeError('Real order crosses real order') if self.user_order_info and self.key_func(self.user_order_info[0]) <= signed_price: price_level = self.price_levels[self.user_order_info[0]] execution = self._handle_matched_user_limit_order(price_level.pop_user_order()) # Must be empty self._remove_price_level_if_empty(price_level) return execution return None # ========== Private Methods ========== def _get_price_level(self, price: int, force_index=False) -> PriceLevel: """ Return price level indicated by price. Price level will be added if not already exists """ level = self.price_levels.get(price, None) # shares == 0 means that the PriceLevel was previously occupied by user order only if level is None: self.prices.add(price) level = PriceLevel(price) self.price_levels[price] = level # force_index is used when we are adding a new price level for real order. Order is not added at this point # and shares will be 0. Therefore, we need to force it # On the other hand, we still need to run update_front_index for user order because it may change the # ordering self._update_front_index(force_index, price) elif level.shares == 0: self._update_front_index(force_index, price) return level def _remove_price_level_if_empty(self, price_level: PriceLevel): """ Remove PriceLevel if empty """ if price_level.empty: del self.price_levels[price_level.price] # "remove" will raise ValueError if not exists self.prices.remove(price_level.price) if price_level.shares == 0: # Separate from the logic above because we run be in the situation where real orders are exhausted # but at least one user order is waiting. In this case, this price level is technically gone self._update_front_index() def _update_front_index(self, force_index=False, target_price=None) -> None: """ Find out the first price level that has real order """ if not self.prices: self._front_idx = None else: price = self.prices[0] if self.price_levels[price].shares > 0 or (force_index and price == target_price): self._front_idx = 0 else: self._front_idx = 1 if len(self.prices) > 1 else None def _handle_matched_user_limit_order(self, order: UserLimitOrder) -> Execution: """ Book-keeping actions for UserLimitOrder execution """ self.user_order_info = None return Execution(order.id, order.price, order.shares if self.side == 'B' else -order.shares) # ========== Properties ========== # These statistics should not include user orders. Otherwise, we may end up being our own market @property def quote(self) -> Optional[int]: """ Return the front price without user orders """ if self._front_idx is not None: return self.prices[self._front_idx] return None @property def volume(self) -> Optional[int]: """ Return the volume at the front without user orders """ if self._front_idx is not None: return self.price_levels[self.quote].shares return None def get_depth(self, num_levels: int) -> List[Tuple[int, int]]: """ Return the top n price levels without user orders """ if self._front_idx is not None: return [(price, self.price_levels[price].shares) for price in self.prices[self._front_idx: self._front_idx + num_levels]] return [] @property def empty(self) -> bool: if len(self.order_pool) == 0: if self.user_order_info is None: return len(self.prices) == 0 return len(self.prices) == 1 and self.price_levels[self.prices[0]].shares == 0 return False @property def user_order_price(self) -> Optional[int]: if self.user_order_info: return self.user_order_info[0] return None
# Read Previous Results, if any if AUTH_OUT == AUTH_PUBS: AuthCount = 0 PubCount = 0 try: with fileinput.input(files=(AUTH_PUBS), openhook=fileinput.hook_encoded("utf-8")) as f: print("Processing Previous Results") for line in f: pub = json.loads(line) PubCount += 1 if pub['auth_sid'] not in AuthIDs: AuthCount += 1 if not (AuthCount % 1000): print("Loaded {} Authors".format(AuthCount)) AuthIDs.add(pub['auth_sid']) print("{} Authors Processed".format(AuthCount)) print("{} Publications Processed".format(PubCount)) except: pass PubOut = Output(AUTH_OUT) # Fetch Publications for a given Author ID. This can be called in parallel for multiple IDs def fetch_pubs(auth): results = [] pubs = auth.get_pubs(view='STANDARD') for pub in pubs: results.append({ "auth_sid": auth.SID,
def test_len(): slt = SortedList() for val in range(10000): slt.add(val) assert len(slt) == (val + 1)
#stop_date = datetime.strptime(row['Stoptime'], '%m/%d/%y %H:%M') start_date = parser.parse(row['Starttime']) stop_date = parser.parse(row['Stoptime']) start_day = start_date.date() stop_day = stop_date.date() # store the start time of each bike in sorted order if bike_id in dates_time: new_list = [from_sta_id, to_sta_id, start_day] dates_time[bike_id].update({stop_date: new_list}) else: dates_time[bike_id] = { stop_date: [from_sta_id, to_sta_id, start_day] } if start_day not in date_as_column: date_as_column.add(start_day) if stop_day not in date_as_column: date_as_column.add(stop_day) # print(day,date) # check if the from station_id is valid # print(type(from_sta_id),from_sta_id) if not math.isnan(from_sta_id): if from_sta_id in station_records: if start_day in station_records[from_sta_id]: station_records[from_sta_id][start_day]['fromCNT'] += 1 else: new_day = {'fromCNT': 1, 'toCNT': 0, 'rebalCNT': 0} station_records[from_sta_id][start_day] = new_day else: station_records[from_sta_id] = { start_day: {
def choose_sense(self, sense_actions: List[Square], move_actions: List[chess.Move], seconds_left: float) -> \ Optional[Square]: allSqr = chess.SquareSet(chess.BB_ALL) for square in allSqr: if self.board.piece_at(square) is not None and self.board.piece_at( square).color == self.color: self.lastScanned[square] = 0 else: self.lastScanned[square] = self.lastScanned[square] + 1 self.moveNum = self.moveNum + 1 if self.panic: for square, piece in self.board.piece_map().items(): if piece.color == self.color: sense_actions.remove(square) return random.choice(sense_actions) if self.interestingTile: print("Scanning interesting tile") return self.interestingTile if self.expectedEnemy is not None: if self.board.piece_at(self.expectedEnemy.to_square) is None: print("Scanning expected move") return self.expectedEnemy.to_square m_king_square = self.board.king(self.color) if m_king_square: attackers = self.board.attackers(not self.color, m_king_square) if not attackers: potentialThreats = SortedList(key=lambda x: x[1]) tB = self.board.copy() bfs = [] bfs.append((tB, None, 0, -1)) while len(bfs) > 0: cS = bfs.pop(0) if cS[0].turn == self.color: cS[0].push(chess.Move.null()) m_king_square = cS[0].king(self.color) if m_king_square: attackers = cS[0].attackers(not self.color, m_king_square) if attackers: potentialThreats.add( (cS[1], self.lastScanned[cS[1]])) continue for mv in cS[0].legal_moves: if cS[2] >= 3: continue if cS[2] >= cS[3] and cS[3] != -1: continue if cS[0].piece_at(mv.to_square) is not None: continue if self.lastScanned[mv.from_square] <= 1: continue nB = cS[0].copy() nB.push(mv) if cS[3] != -1: bfs.append( (nB, cS[1] if cS[1] is not None else mv.to_square, cS[2] + 1, min(cS[3], cS[2] + self.lastScanned[mv.to_square]))) else: bfs.append( (nB, cS[1] if cS[1] is not None else mv.to_square, cS[2] + 1, self.lastScanned[mv.from_square])) if len(potentialThreats) > 0: pot = potentialThreats.pop(-1) print("Scanning threat: ", chess.SQUARE_NAMES[pot[0]]) return pot[0] self.nextMove = self.choose_move(move_actions, seconds_left) if self.nextMove is not None: print("Scanning next move") return self.nextMove.to_square for square, piece in self.board.piece_map().items(): if piece.color == self.color: sense_actions.remove(square) print("Scanning random") return random.choice(sense_actions)
def test(filename): """ run bentley ottmann """ adjuster, segments = load_segments(filename) #segments = sorted(segments, key=lambda x: min(x.endpoints[0].coordinates[1], # x.endpoints[0].coordinates[1])) events = SortedList() for s in segments: events.add((min(s.endpoints), "in", s)) events.add((max(s.endpoints), "out", s)) sweep = SweepLines() result = [] if DEBUG: print("Events (init):", events) print("\n========\n LOOP \n========\n\n ") while True: try: current, event_type, segment = events.pop(0) Segment.point = current if DEBUG: print("Current:", current, event_type, segment) if DEBUG: print("Events:", events) if DEBUG: print("SL:", len(sweep), sweep) tmp_sweep = SweepLines() for node in sweep: tmp_sweep.put(node.value) sweep = tmp_sweep if DEBUG: print("SL:", len(sweep), sweep) if event_type == "in": node = sweep.put(segment) left = node.predecessor() if left: left = left.value intrsctn = segment.intersection_with(left) if intrsctn is not None: intrsctn = adjuster.hash_point(intrsctn) if intrsctn.coordinates[1] <= current.coordinates[ 1] and intrsctn.coordinates[ 0] != current.coordinates[0]: events.add((intrsctn, "x", (left, segment))) right = node.successor() if right: right = right.value intrsctn = segment.intersection_with(right) if intrsctn is not None: intrsctn = adjuster.hash_point(intrsctn) if intrsctn.coordinates[1] <= current.coordinates[ 1] and intrsctn.coordinates[ 0] != current.coordinates[0]: events.add((intrsctn, "x", (segment, right))) elif event_type == "out": node = sweep.search(segment) left = node.predecessor() right = node.successor() if left and right: left = left.value right = right.value intrsctn = left.intersection_with(right) if intrsctn is not None: intrsctn = adjuster.hash_point(intrsctn) if intrsctn.coordinates[1] <= current.coordinates[ 1] and intrsctn.coordinates[ 0] != current.coordinates[0]: events.add((intrsctn, "x", (left, right))) sweep.delete(segment) else: #event_type == "x" result.append(current) u = sweep.search(segment[0]) right = u.successor() if right: intrsctn = u.value.intersection_with(right.value) if intrsctn is not None: intrsctn = adjuster.hash_point(intrsctn) if intrsctn.coordinates[1] <= current.coordinates[ 1] and intrsctn.coordinates[ 0] != current.coordinates[0]: events.add((intrsctn, "x", (u.value, right.value))) v = sweep.search(segment[1]) left = v.predecessor() if left: intrsctn = v.value.intersection_with(left.value) if intrsctn is not None: intrsctn = adjuster.hash_point(intrsctn) if intrsctn.coordinates[1] <= current.coordinates[ 1] and intrsctn.coordinates[ 0] != current.coordinates[0]: events.add((intrsctn, "x", (left.value, v.value))) if DEBUG: print("Events:", events) if DEBUG: print("SL:", len(sweep), sweep) #tycat(segments, result, current) #input("Press [ENTER] to continue...\n") except IndexError: break print("\n\n=========\n THE END\n=========") if DEBUG: print("Events:", events) print("SL:", sweep) print("IL:", result) tycat(segments, result)
def bentley_ottmann(filename, nodisp=False, noinfo=False): """ Fonction principale de notre projet """ global COUPE COUPE = 0 y_cour = None adjuster, segments = load_segments(filename) actifs = SortedList() evenements = [ ] #liste de nos evenements, valeurs des y, que lon transformera en Tas ensuite pt_inter = { } #dictionnaire que lon retournera a la fin, associant les segments interseptés index = 0 cache_inters = {} #cache qui nous dira si on a deja compare 2 seg intersections = [] #liste contenant tous nos points dintersections for seg in segments: #initialisation de nos evenements (x_0, y_0) = seg.endpoints[0].coordinates (x_1, y_1) = seg.endpoints[1].coordinates Segment.y_cour = [x_0, y_0] if y_0 < y_1: #Segments croissant suivant les y evenements.append([y_0, -x_0, seg, 'D']) evenements.append([y_1, -x_1, seg, 'F']) elif y_0 > y_1: #Segments decroissant suivant les y: evenements.append([y_0, -x_0, seg, 'F']) evenements.append([y_1, -x_1, seg, 'D']) else: #Cas d'un segment horizontal evenements.append([y_1, -min(x_0, x_1), seg, max(x_0, x_1)]) pt_inter[seg] = [] #Initialisation du dictionnaire cache_inters[seg] = [] heapify( evenements ) #Tas des evenement,3 types, 'D' 'F' 'I': Debut, fin, intersection #trié en fonction des y croissant, puis des x décroissants. def indice(seg): """ Retourne l'indice de seg dans la liste actifs, None si le segment n'est pas présent. Cette fonction auxiliaire est implémentée suite aux problèmes majeurs rencontrés avec la méthode index de la classe SortedList """ for i, elmt in enumerate(actifs): if seg is elmt: return i def intersection(seg, seg_2): """ Fonction qui va légitimer et gérer l'intersection entre 2 segments donnés. """ global COUPE if seg_2 not in cache_inters[seg]: #On ne compare pas deux segments #déja comparés intersection = seg.intersection_with(seg_2) cache_inters[seg].append(seg_2) cache_inters[seg_2].append(seg) if intersection is not None: intersection = adjuster.hash_point(intersection) #Ajustement if intersection not in seg.endpoints or intersection not in seg_2.endpoints: #Le point nest pas lextrémitié des deux segments pt_inter[seg].append(seg_2) pt_inter[seg_2].append(seg) heappush(evenements, [ intersection.coordinates[1], -intersection.coordinates[0], seg, 'I', seg_2 ]) #L'ordre dans le tuple est important: il permet de savoir #qui est à gauche ou à droite if intersection not in intersections: intersections.append(intersection) COUPE += 1 return while evenements: #Boucle traitant tous les évènements tant que notre tas #n'est pas vide. y_cour = heappop(evenements) if y_cour[3] == 'D': #evenement de debut de segment Segment.y_cour = [-y_cour[1], y_cour[0]] actifs = SortedList(actifs) #Mise à jour de actifs seg = y_cour[2] actifs.add(seg) #Ajout du nouveau segment aux actifs if len(actifs ) > 1: #Si un seul segment dans actifs: on ne fait rien try: index = actifs.index(seg) except ValueError: index = indice(seg) if index != len(actifs) - 1: seg_2 = actifs[index + 1] intersection(seg, seg_2) if index != 0: seg_2 = actifs[index - 1] intersection(seg_2, seg) elif y_cour[3] == 'F': #evenement de fin de segment Segment.y_cour = [-y_cour[1], y_cour[0]] actifs = SortedList(actifs) #Mise à jour de actifs seg = y_cour[2] try: index = actifs.index(seg) except ValueError: index = indice(seg) actifs.pop(index) actifs = SortedList(actifs) #Mise à jour de actifs if len(actifs) > 1: if 0 < index < len(actifs): #On n'enleve pas le seg le plus à #droite/gauche seg = actifs[index] seg_2 = actifs[index - 1] intersection(seg, seg_2) elif y_cour[3] == 'I': #evenement de point d'intersection seg, seg_2 = y_cour[2], y_cour[4] try: actifs.remove(seg) except ValueError: index = indice(seg) if index is not None: #Renvoie parfois une erreur: #"segment not in actifs" del actifs[index] try: actifs.remove(seg_2) except ValueError: index_2 = indice(seg_2) if index_2 is not None: del actifs[index_2] Segment.y_cour = [-y_cour[1], y_cour[0] + 0.00000000001] #Cf. convention: A une intersection, on se situe #au dessus de l'intersection actifs = SortedList(actifs) #Mise à jour de actifs actifs.add(seg) #Une fois changés de place l'intersection passée, #on remet nos deux segments dans actifs actifs.add(seg_2) try: index = actifs.index(seg) #Indice du seg a droite une fois #l'intersection faite except ValueError: index = indice(seg) if len(actifs ) > 2: #On teste les nouvelles intersections possibles if index < len( actifs) - 1: #Cas de l'extrémité droite de actifs seg_2 = actifs[index + 1] intersection(seg, seg_2) if index - 1 != 0: #Cas de l'extrémité gauche seg_2 = actifs[index - 2] intersection(seg_2, y_cour[4]) else: #Cas dun segment horizontal seg_h = y_cour[2] for seg in actifs: inter = seg_h.intersection_with(seg) if inter: inter = adjuster.hash_point(inter) if inter not in seg_h.endpoints or inter not in seg.endpoints: #Le point n'est pas l'extrémité ds deux segments pt_inter[seg_h].append(seg) pt_inter[seg].append(seg_h) if inter not in intersections: intersections.append(inter) COUPE += 1 if nodisp and noinfo: return pt_inter, intersections if noinfo: tycat(segments, intersections) return pt_inter, intersections if nodisp: print( "Le nombre d'intersections (= le nombre de points differents) est : ", len(intersections)) print("Le nombre de coupes est : ", COUPE) return pt_inter, intersections print( "le nombre d'intersections (= le nombre de points differents) est : ", len(intersections)) print("le nombre de coupes est : ", COUPE)
def choose_move(self, move_actions: List[chess.Move], seconds_left: float) -> Optional[chess.Move]: self.approxTime = seconds_left self.response = None if self.nextMove is not None: mv = self.nextMove self.nextMove = None return mv if self.board.turn is not self.color: self.board.push(chess.Move.null()) e_king_square = self.board.king(not self.color) if e_king_square: attackers = self.board.attackers(self.color, e_king_square) for attack in attackers: m = chess.Move(attack, e_king_square) if m in move_actions: return m # m_king_square = self.board.king(self.color) # if m_king_square: # attackers = self.board.attackers(not self.color, m_king_square) # if attackers: # legalMoves = self.board.legal_moves # for move in legalMoves: # tempBoard = self.board.copy() # tempBoard.push(move) # attackers = tempBoard.attackers(not self.color, tempBoard.king(self.color)) # if (attackers is None or len(attackers) is 0) and move in move_actions: # return move class State: def __init__(self, board, iM): self.brd = board self.score = 0 self.initialMove = iM self.responses = SortedList(key=lambda x: x.score) def eval_board(board) -> float: score = 0 for square, piece in board.piece_map().items(): mult = 1 attackers = board.attackers(not piece.color, square) defenders = board.attackers(piece.color, square) val = 0 if piece.color != self.color: mult = -1 if piece.piece_type == chess.PAWN: val = 5 elif piece.piece_type == chess.KNIGHT: val = 10 elif piece.piece_type == chess.BISHOP: val = 10 elif piece.piece_type == chess.ROOK: val = 10 elif piece.piece_type == chess.QUEEN: val = 20 elif piece.piece_type == chess.KING: val = 100 #if len(attackers) != 0 and len(defenders) == 0 and piece.color == self.color: # val = val / 10 #if len(defenders) != 0 and piece.piece_type != chess.KING and piece.color != self.color: # val = val * 2 score = score + mult * val return score self.score = eval_board(self.board) trialsPer = 20 depth = 10 wasKing = False if self.board.king(not self.color): wasKing = True self.panic = False else: wasKing = False self.panic = True potentials = SortedList(key=lambda x: (x.score)) for mv in self.board.legal_moves: startState = State(self.board.copy(), mv) startState.brd.push(mv) count = 0 for resp in startState.brd.legal_moves: res = State(startState.brd.copy(), resp) res.brd.push(resp) scr = 0 for i in range(trialsPer): tempBoard = res.brd.copy() tscr = 0 for j in range(depth): if tempBoard.turn == self.color: e_king_square = tempBoard.king(not self.color) if e_king_square: attackers = tempBoard.attackers( self.color, e_king_square) if attackers: tscr = 1000 break m_king_square = tempBoard.king(self.color) if m_king_square: attackers = tempBoard.attackers( not self.color, m_king_square) if attackers: tscr = -1000 break if tempBoard.turn != self.color: m_king_square = tempBoard.king(self.color) if m_king_square: attackers = tempBoard.attackers( not self.color, m_king_square) if attackers: tscr = -1000 break e_king_square = tempBoard.king(not self.color) if e_king_square: attackers = tempBoard.attackers( self.color, e_king_square) if attackers: tscr = 1000 break potMoves = [] for pM in tempBoard.legal_moves: potMoves.append(pM) if len(potMoves) != 0: tempBoard.push(random.choice(potMoves)) else: tscr = -1000 break if tscr == 0: tscr = eval_board(tempBoard) scr = scr + tscr scr = scr / trialsPer res.score = scr startState.responses.add(res) startState.score = startState.score + scr count = count + 1 if count != 0: startState.score = startState.score / count else: startState.score = eval_board(startState.brd) potentials.add(startState) while len(potentials) > 0: top = potentials.pop(-1) if top.initialMove in move_actions: if len(top.responses) > 0: self.expectedEnemy = top.responses.pop(0).initialMove self.expectedScore = top.score return top.initialMove return random.choice(move_actions + [None])
def _all_segment_intersections_no_horizontal(segments): # noqa # Must be unique assert len(set(segments)) == len(segments) segments = list(segments) # Must not be degenerate for segment in segments: assert segment[0] != segment[1] # Use the convention from the book: sweep on Y axis def event_key(pt): return (pt[1], pt[0]) # From point to list of segments event_queue = SortedDict(event_key) def add_event(pt, segment_key=None): if pt not in event_queue: event_queue[pt] = [] if segment_key is not None: event_queue[pt].append(segment_key) for i, segment in enumerate(segments): if event_key(segment[0]) < event_key(segment[1]): add_event(segment[0], _SweepKey(segment, segment[0])) add_event(segment[1], None) else: add_event(segment[0], None) add_event(segment[1], _SweepKey(segment, segment[1])) active = SortedList() y = -math.inf while len(event_queue) > 0: v = event_queue.popitem(0) pt, segstarts = v # Can't be > since while there are no horizontal segments, # there can still be points in horizontal relation to one another assert pt[1] >= y y = pt[1] # Find all segments within the event point fake_segment = ((pt[0], pt[1]), (pt[0], pt[1] + 1)) fake_key = _SweepKey(fake_segment, pt) touches = [] # The next lower / higher keys, respectively, to enter new events for neighbours = [] if _extra_checks: _assert_fully_sorted(list(active), y) # Iterate on both sides for it in ( active.irange(None, fake_key, inclusive=(True, True), reverse=True), active.irange(fake_key, None, inclusive=(False, True)), ): neighbour = None for sweep_key in it: if sweep_key.at_y(y) != pt[0]: neighbour = sweep_key break touches.append(sweep_key) neighbours.append(neighbour) # Remove the old sweep keys for touch in touches: active.remove(touch) segments_at_pt = [ sweep_key.segment for sweep_key in touches + segstarts ] if len(segments_at_pt) > 1: yield (pt, tuple(segments_at_pt)) # Create new _SweepKeys, automatically sorts # according to order after point sweep_keys = [] for segment in segments_at_pt: # Is this segment still relevant? if max(segment[0][1], segment[1][1]) <= pt[1]: continue sweep_keys.append(_SweepKey(segment, pt)) sweep_keys = list(sorted(sweep_keys)) # Add new events for neighbours if len(sweep_keys) == 0: # If we just removed stuff, the neighbours might now meet... if neighbours[0] is not None and neighbours[1] is not None: ipt = _nonparallel_intersection_point(neighbours[0].segment, neighbours[1].segment) if ipt and ipt[1] > pt[1]: add_event(ipt) continue if neighbours[0] is not None: ipt = _nonparallel_intersection_point(sweep_keys[0].segment, neighbours[0].segment) # hyp.note(fstr('IPTL', ipt, pt)) if ipt and ipt[1] > pt[1]: add_event(ipt) if neighbours[1] is not None: ipt = _nonparallel_intersection_point(sweep_keys[-1].segment, neighbours[1].segment) # hyp.note(fstr('IPTR', ipt, pt)) if ipt and ipt[1] > pt[1]: add_event(ipt) # Add them in and continue for sweep_key in sweep_keys: active.add(sweep_key)
class SweepStatus: def __init__(self): self.lines_list = SortedList() self.num_lines = 0 # all lines as seen by the sweep line def find_line(self, l): # can replace this with a binary search # return binary_search(self.lines_list,SweepStatusEntry(l,-1)) for i in range(len(self.lines_list)): if self.lines_list[i].l == l: return i return None def check_intersection(self, l_index, direction): res = [] print("Looking for intersection at index", l_index, " direction =", direction) cur = l_index #the index of the line which we are checking if direction == "left": cur = l_index - 1 while cur >= 0: p = intersection_point(self.lines_list[l_index].l, self.lines_list[cur].l) if p is not None: res.append( IntersectionPoint(p, self.lines_list[cur].l, self.lines_list[l_index].l)) else: break cur -= 1 else: # direction is "right" cur = l_index + 1 while cur <= len(self.lines_list) - 1: p = intersection_point(self.lines_list[l_index].l, self.lines_list[cur].l) if p is not None: res.append( IntersectionPoint(p, self.lines_list[l_index].l, self.lines_list[cur].l)) else: break cur += 1 s = "Found Intersection Point(s): " for l in res: s += (str(l) + '\n') s += "%%%%%%%%%%" print(s) return res def add_line(self, new_l): # return an iterable print("Adding line ", new_l) res = [] # insert new one in place new_entry = SweepStatusEntry(new_l, -1) new_index = bisect.bisect(self.lines_list, new_entry) print(new_index) new_key = self.num_lines + 1 self.num_lines += 1 self.lines_list.add(SweepStatusEntry(new_l, new_key)) # search is independent of the key # need the key to keep the line rooted in place print("lines list after adding") for i in self.lines_list: print(i) print() res.extend(self.check_intersection(new_index, "left")) res.extend(self.check_intersection(new_index, "right")) return res def remove_line(self, l): print("Removing line ", l) res = [] l_index = self.find_line(l) if l_index is None: return res # remove the line found self.lines_list.pop(l_index) # check for new intersections if (len(self.lines_list) != 0): res.extend(self.check_intersection(l_index - 1, "right")) # new l_index is the one that was on the right return res def intersection_point(self, ipt): print("Reached intersection point ", ipt) res = [] l1_index = self.find_line(ipt.l1) if (l1_index == None): return res l2_index = self.find_line(ipt.l2) if (l2_index == None): return res print("Swapping indices: ", l1_index, l2_index) self.lines_list[l1_index].l = ipt.l2 self.lines_list[l2_index].l = ipt.l1 print("Lines after swapping: ") print(self.lines_list[l1_index]) print(self.lines_list[l2_index]) if l1_index > l2_index: temp = l1_index l1_index = l2_index l2_index = temp # now l1_index is the lower index res.extend(self.check_intersection(l1_index, "left")) res.extend(self.check_intersection(l2_index, "right")) return res def __str__(self): s = "--- Sweep Line Status: ---\n" for i in self.lines_list: s += str(i) s += " ;\n" s += "-----------------" return s
return ''.join(chars) def pron_strs(prons): return ' '.join(map(lambda x: pron_str(x), prons)) for pron1 in prons: closest_prons = [] dists = SortedList() for pron2 in prons: if pron1 != pron2: dist = Levenshtein.distance(pron1, pron2) if len(dists) > 0 and dist < dists[0]: closest_prons = [] dists.add(dist) if dist == dists[0]: closest_prons.append(pron2) best1_dists.append(dists[0]) best5_dists.append(np.mean(dists[:5])) print("%s - %s (%.2f)" % (pron_str(pron1), pron_strs(closest_prons), dists[0])) best1_dists = np.array(best1_dists) best5_dists = np.array(best5_dists) print() print("For %.1f %% of pronuncations the shortest Levenshtein distance is 1" % (100 * (best1_dists == 1).sum() / len(dists))) print("1-best average Levenshtein distance: %.2f" % best1_dists.mean()) print("5-best average Levenshtein distance: %.2f" % best5_dists.mean())
class UrlClusterizedQueue: def __init__(self): self._features_count_list = SortedList() # list of pairs (feature_count, feature_name) self._features_count_dict = dict() self._clusterizer = DBSCAN(metric='jaccard') self._min_freq = 0.1 self._max_freq = 0.9 self._urls = dict() self._urls_keys = [] self._index = -1 self._min_urls_count = 50 self._subqueue = Queue() self._subqueue_len = 4 # constants self._i_is_used = 1 self._i_features = 0 self._i_list_for_unused = 0 self._i_list_for_used = 1 self._i_list_total = 2 def _return_url(self, url): self._urls[url][1] = True return url def _next_queue_fallback(self): self._index += 1 url = self._urls_keys[self._index] return self._return_url(url) def _run_clustering(self): print("try to use clustering") # here we need to run clustering # first of all we need to choose features max_feature_count = int(self._max_freq * len(self._urls)) min_feature_count = int(self._min_freq * len(self._urls)) start_index = bisect.bisect_left(self._features_count_list, (min_feature_count, '')) end_index = bisect.bisect_right(self._features_count_list, (max_feature_count, 'ZZZ')) if start_index >= end_index: print("not enough features") return self._next_queue_fallback() chosen_features = SortedSet() for i in range(start_index, end_index): chosen_features.add(self._features_count_list[i][1]) # then we need to build features matrix X = np.empty((len(self._urls), len(chosen_features))) for i in range(len(self._urls)): features = self._urls[self._urls_keys[i]][self._i_features] for j, fname in enumerate(chosen_features): if fname in features: X[i][j] = 1 else: X[i][j] = 0 # now we can run clustering y = self._clusterizer.fit_predict(X) # and we need to create uniform distributed queue def get_list_of_2_sets(): return [set(), set(), 0] # 0 is for used urls, # 1 is for unused # 3 is for total count url_in_cluster = defaultdict(get_list_of_2_sets) for i in range(len(y)): url = self._urls_keys[i] if self._urls[url][self._i_is_used]: url_in_cluster[y[i]][self._i_list_for_used].add(url) else: url_in_cluster[y[i]][self._i_list_for_unused].add(url) url_in_cluster[y[i]][self._i_list_total] += 1 limit = self._subqueue_len cluster_keys = SortedKeyList(url_in_cluster.keys(), key=lambda x: -len(url_in_cluster[x][self._i_list_for_used])) while limit > 0: # Todo: optimize if len(cluster_keys) > 0: less_index = cluster_keys.pop() unused_urls = url_in_cluster[less_index][self._i_list_for_unused] if len(unused_urls) > 0: url = unused_urls.pop() self._subqueue.put(url) limit -= 1 if len(unused_urls) > 0: url_in_cluster[less_index][self._i_list_for_used].add(url) cluster_keys.add(less_index) else: break async def _run_and_wait_clustering(self): t = threading.Thread(target=UrlClusterizedQueue._run_clustering, args=(self,)) t.run() while t.is_alive(): await asyncio.sleep(0.3) async def get(self): if len(self._urls) < self._min_urls_count: return self._next_queue_fallback() else: if self._subqueue.empty(): await self._run_and_wait_clustering() # if self._subqueue.qsize() == 1: # asyncio.create_task(self._run_and_wait_clustering()) return self._return_url(self._subqueue.get()) async def empty(self): if self._index + 1 >= len(self._urls): return True else: return False async def put(self, url): if url in self._urls: return features = url_features.extract(url) for fname in features: if fname in self._features_count_dict: fcount = self._features_count_dict[fname] del self._features_count_list[self._features_count_list.index((fcount, fname))] else: fcount = 0 fcount += 1 self._features_count_dict[fname] = fcount self._features_count_list.add((fcount, fname)) self._urls[url] = [features, False] # False is for used self._urls_keys.append(url)
def test_len(): slt = SortedList() for val in range(10000): slt.add(val) assert len(slt) == (val + 1)
class SegmentedBuffer: __slots__ = ('buffer_size', 'buffer_bits', 'data_bits', 'data_size', 'method', 'sorted_data_count', 'remaining_bits', 'remaining_size', 'remaining') def __init__(self): self.buffer_bits: int = 0 self.buffer_size: int = 0 self.data_bits: int = 0 self.data_size: int = 0 self.method: Method = Method.UNDEFINED self.sorted_data_count: SortedList[DataCount] = SortedList() self.remaining_bits: int = 0 self.remaining_size: int = 0 self.remaining: DataType = 0 # check & print values def check_values(self, file: AnyFile = sys.stderr): max_data = bits_mask(self.data_bits) for data_count in self.sorted_data_count: if data_count.data > max_data: print(f'!\tinvalid data: {data_count.data:0>b}', file=file) def print_values(self, file: AnyFile = sys.stdout): _print_bits_size('buffer', self.buffer_bits, self.buffer_size, file=file) _print_bits_size('data', self.data_bits, self.data_size, file=file) if self.method == Method.INT: def print_data_count(item: DataCount): return print( f'-\t{item.data:0>{self.data_bits}b}: {item.count}', file=file) else: raise ValueError(f'unsupported method: {self.method}') for data_count in self.sorted_data_count: print_data_count(data_count) _print_bits_size('remaining', self.remaining_bits, self.remaining_size, file=file) if self.method == Method.INT: print(f'-\t{self.remaining:0>{self.remaining_bits}b}') # write, read def _write_sorted_data_count(self, wrapper: FileWrapper, size_of_size: int): wrapper.write_unsigned_int(len(self.sorted_data_count), size_of_size) if self.method == Method.INT: for data_count in self.sorted_data_count: wrapper.write_unsigned_int(data_count.data, self.data_size) wrapper.write_unsigned_int(data_count.count, size_of_size) else: raise ValueError(f'unsupported method: {self.method}') def _read_sorted_data_count(self, wrapper: FileWrapper, size_of_size: int): length = wrapper.read_unsigned_int(size_of_size) self.sorted_data_count = SortedList() if self.method == Method.INT: for step in range(length): data = wrapper.read_unsigned_int(self.data_size) count = wrapper.read_unsigned_int(size_of_size) self.sorted_data_count.add(DataCount(data, count)) else: raise ValueError(f'unsupported method: {self.method}') return SortedList() def _write_remaining(self, wrapper: FileWrapper, size_of_size: int): if self.method == Method.INT: wrapper.write_unsigned_int(self.remaining, self.remaining_size) elif self.method == Method.BYTE: wrapper.write_bytes(self.remaining) else: raise ValueError(f'unsupported method: {self.method}') def _read_remaining(self, wrapper: FileWrapper): if self.method == Method.INT: self.remaining = wrapper.read_unsigned_int(self.remaining_size) elif self.method == Method.BYTE: self.remaining = wrapper.read_bytes(self.remaining_size) else: raise ValueError(f'unsupported method: {self.method}') def write(self, wrapper: FileWrapper, size_of_size: int = 4): wrapper.write_unsigned_int(size_of_size, 1) # size of size wrapper.write_unsigned_int(self.buffer_bits, size_of_size) # buffer bits wrapper.write_unsigned_int(self.buffer_size, size_of_size) # buffer size wrapper.write_unsigned_int(self.data_bits, size_of_size) # data bits wrapper.write_unsigned_int(self.data_size, size_of_size) # data size wrapper.write_unsigned_int(self.remaining_bits, size_of_size) # remaining bits wrapper.write_unsigned_int(self.remaining_size, size_of_size) # remaining size self.method.write(wrapper) # method # sorted data count & remaining self._write_sorted_data_count(wrapper, size_of_size) self._write_remaining(wrapper, size_of_size) def read(self, wrapper: FileWrapper): size_of_size = wrapper.read_unsigned_int(1) # size of size self.buffer_bits = wrapper.read_unsigned_int( size_of_size) # buffer bits self.buffer_size = wrapper.read_unsigned_int( size_of_size) # buffer size self.data_bits = wrapper.read_unsigned_int(size_of_size) # data bits self.data_size = wrapper.read_unsigned_int(size_of_size) # data size self.remaining_bits = wrapper.read_unsigned_int( size_of_size) # remaining bits self.remaining_size = wrapper.read_unsigned_int( size_of_size) # remaining size self.method = Method.read(wrapper) # method # sorted data count & remaining self._read_sorted_data_count(wrapper, size_of_size) self._read_remaining(wrapper) @staticmethod def static_read(wrapper: FileWrapper): instance = SegmentedBuffer() instance.read(wrapper) return instance @staticmethod def static_write(wrapper: FileWrapper, instance: 'SegmentedBuffer', size_of_size: int = 4): return instance.write(wrapper, size_of_size=size_of_size) # scan & sort data count def _scan_data_count(self, buffer: bytes): if self.method == Method.BYTE: if self.data_bits % 8 == 0: max_index = self.buffer_size - self.remaining_size count_dict = defaultdict(lambda: 1) for index in range(0, max_index, self.data_size): count_dict[buffer[index:index + self.data_size]] += 1 return count_dict, buffer[max_index:] else: raise ValueError( f'{self.data_bits} data bits is not supported in {self.method}' ) elif self.method == Method.INT: count_dict = defaultdict(lambda: 0) if self.data_bits == 8: for value in buffer: count_dict[value] += 1 return count_dict, 0 elif self.data_bits % 8 == 0: max_index = self.data_size - self.remaining_size for index in range(0, max_index, self.data_size): count_dict[from_bytes(buffer[index:index + self.data_size])] += 1 return count_dict, from_bytes(buffer[max_index:]) else: bits_io = BitsIO(buffer, self.data_bits) for value in bits_io: count_dict[value] += 1 return count_dict, bits_io.remaining() @staticmethod def _sort_count_dict(count_dict: Dict): return SortedList( (DataCount(key, value) for key, value in count_dict.items()), key=lambda item: item.count) @staticmethod def scan_buffer(data_bits: int, buffer: bytes, method: Method = Method.INT): if data_bits <= 1: raise ValueError(f'invalid data bits: {data_bits}') elif data_bits > 1024: raise ValueError(f'data bits is too big: {data_bits}') result = SegmentedBuffer() result.buffer_size = len(buffer) result.buffer_bits = result.buffer_size * 8 result.data_bits = data_bits result.data_size = get_bytes_per_bits(data_bits) result.remaining_bits = result.buffer_bits % result.data_bits result.remaining_size = get_bytes_per_bits(result.remaining_bits) result.method = method count_dict, result.remaining = result._scan_data_count(buffer) result.sorted_data_count = SegmentedBuffer._sort_count_dict(count_dict) return result def convert(self, data_bits: int): if self.data_bits % data_bits != 0: raise ValueError() result = SegmentedBuffer() result.buffer_size = self.buffer_size result.buffer_bits = self.buffer_bits result.data_bits = data_bits result.data_size = get_bytes_per_bits(data_bits) result.remaining_bits = result.buffer_bits % result.data_bits result.remaining_size = get_bytes_per_bits(result.remaining_bits) raise BaseException('incomplete code!')
class PlatformBatchLightSystem: """Batch light system for platforms.""" __slots__ = [ "dirty_lights", "dirty_schedule", "clock", "is_sequential_function", "update_task", "update_callback", "sort_function", "update_hz", "max_batch_size" ] # pylint: disable-msg=too-many-arguments def __init__(self, clock, sort_function, is_sequential_function, update_callback, update_hz, max_batch_size): """Initialise light system.""" self.dirty_lights = SortedSet( key=sort_function) # type: Set[PlatformBatchLight] self.dirty_schedule = SortedList( key=lambda x: x[0] + sort_function(x[1])) self.is_sequential_function = is_sequential_function self.sort_function = sort_function self.update_task = None self.clock = clock self.update_callback = update_callback self.update_hz = update_hz self.max_batch_size = max_batch_size def start(self): """Start light system.""" self.update_task = self.clock.loop.create_task(self._send_updates()) self.update_task.add_done_callback(self._done) def stop(self): """Stop light system.""" if self.update_task: self.update_task.cancel() self.update_task = None @staticmethod def _done(future): try: future.result() except asyncio.CancelledError: pass async def _send_updates(self): while True: while self.dirty_schedule and self.dirty_schedule[0][ 0] <= self.clock.get_time(): self.dirty_lights.add(self.dirty_schedule[0][1]) del self.dirty_schedule[0] sequential_lights = [] for light in list(self.dirty_lights): if not sequential_lights: # first light sequential_lights = [light] elif self.is_sequential_function(sequential_lights[-1], light): # lights are sequential sequential_lights.append(light) else: # sequence ended await self._send_update_batch(sequential_lights) # this light is a new sequence sequential_lights = [light] if sequential_lights: await self._send_update_batch(sequential_lights) self.dirty_lights.clear() await asyncio.sleep(.001, loop=self.clock.loop) async def _send_update_batch(self, sequential_lights): sequential_brightness_list = [] common_fade_ms = None current_time = self.clock.get_time() for light in sequential_lights: brightness, fade_ms, done = light.get_fade_and_brightness( current_time) if not done: self.dirty_schedule.add( (current_time + (fade_ms / 1000), light)) if common_fade_ms is None: common_fade_ms = fade_ms if common_fade_ms == fade_ms and len( sequential_brightness_list) < self.max_batch_size: sequential_brightness_list.append( (light, brightness, common_fade_ms)) else: await self.update_callback(sequential_brightness_list) # start new list current_time = self.clock.get_time() common_fade_ms = fade_ms sequential_brightness_list = [(light, brightness, common_fade_ms)] if sequential_brightness_list: await self.update_callback(sequential_brightness_list) def mark_dirty(self, light: "PlatformBatchLight"): """Mark as dirty.""" self.dirty_lights.add(light) self.dirty_schedule = SortedList( [x for x in self.dirty_schedule if x[1] != light], key=lambda x: x[0] + self.sort_function(x[1]))
class PriorityDict(MutableMapping): """ A PriorityDict provides the same methods as a dict. Additionally, a PriorityDict efficiently maintains its keys in value sorted order. Consequently, the keys method will return the keys in value sorted order, the popitem method will remove the item with the highest value, etc. """ def __init__(self, *args, **kwargs): """ A PriorityDict provides the same methods as a dict. Additionally, a PriorityDict efficiently maintains its keys in value sorted order. Consequently, the keys method will return the keys in value sorted order, the popitem method will remove the item with the highest value, etc. If the first argument is the boolean value False, then it indicates that keys are not comparable. By default this setting is True and duplicate values are tie-breaked on the key. Using comparable keys improves the performance of the PriorityDict. An optional *iterable* argument provides an initial series of items to populate the PriorityDict. Each item in the sequence must itself contain two items. The first is used as a key in the new dictionary, and the second as the key's value. If a given key is seen more than once, the last value associated with it is retained in the new dictionary. If keyword arguments are given, the keywords themselves with their associated values are added as items to the dictionary. If a key is specified both in the positional argument and as a keyword argument, the value associated with the keyword is retained in the dictionary. For example, these all return a dictionary equal to ``{"one": 2, "two": 3}``: * ``SortedDict(one=2, two=3)`` * ``SortedDict({'one': 2, 'two': 3})`` * ``SortedDict(zip(('one', 'two'), (2, 3)))`` * ``SortedDict([['two', 3], ['one', 2]])`` The first example only works for keys that are valid Python identifiers; the others work with any valid keys. Note that this constructor mimics the Python dict constructor. If you're looking for a constructor like collections.Counter(...), see PriorityDict.count(...). """ self._dict = dict() if len(args) > 0 and isinstance(args[0], bool): if args[0]: self._list = SortedList() else: self._list = SortedListWithKey(key=lambda tup: tup[0]) else: self._list = SortedList() self.iloc = _IlocWrapper(self) self.update(*args, **kwargs) def clear(self): """Remove all elements from the dictionary.""" self._dict.clear() self._list.clear() def clean(self, value=0): """ Remove all items with value less than or equal to `value`. Default `value` is 0. """ _list, _dict = self._list, self._dict pos = self.bisect_right(value) for key in (key for value, key in _list[:pos]): del _dict[key] del _list[:pos] def __contains__(self, key): """Return True if and only if *key* is in the dictionary.""" return key in self._dict def __delitem__(self, key): """ Remove ``d[key]`` from *d*. Raises a KeyError if *key* is not in the dictionary. """ value = self._dict[key] self._list.remove((value, key)) del self._dict[key] def __getitem__(self, key): """ Return the priority of *key* in *d*. Raises a KeyError if *key* is not in the dictionary. """ return self._dict[key] def __iter__(self): """ Create an iterator over the keys of the dictionary ordered by the value sort order. """ return iter(key for value, key in self._list) def __reversed__(self): """ Create an iterator over the keys of the dictionary ordered by the reversed value sort order. """ return iter(key for value, key in reversed(self._list)) def __len__(self): """Return the number of (key, value) pairs in the dictionary.""" return len(self._dict) def __setitem__(self, key, value): """Set `d[key]` to *value*.""" if key in self._dict: old_value = self._dict[key] self._list.remove((old_value, key)) self._list.add((value, key)) self._dict[key] = value def copy(self): """Create a shallow copy of the dictionary.""" result = PriorityDict() result._dict = self._dict.copy() result._list = self._list.copy() result.iloc = _IlocWrapper(result) return result def __copy__(self): """Create a shallow copy of the dictionary.""" return self.copy() @classmethod def fromkeys(cls, iterable, value=0): """ Create a new dictionary with keys from `iterable` and values set to `value`. The default *value* is 0. """ return PriorityDict((key, value) for key in iterable) def get(self, key, default=None): """ Return the value for *key* if *key* is in the dictionary, else *default*. If *default* is not given, it defaults to ``None``, so that this method never raises a KeyError. """ return self._dict.get(key, default) def has_key(self, key): """Return True if and only in *key* is in the dictionary.""" return key in self._dict def pop(self, key, default=_NotGiven): """ If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is not in the dictionary, a KeyError is raised. """ if key in self._dict: value = self._dict[key] self._list.remove((value, key)) return self._dict.pop(key) else: if default == _NotGiven: raise KeyError else: return default def popitem(self, index=-1): """ Remove and return item at *index* (default: -1). Raises IndexError if dict is empty or index is out of range. Negative indices are supported as for slice indices. """ value, key = self._list.pop(index) del self._dict[key] return key, value def setdefault(self, key, default=0): """ If *key* is in the dictionary, return its value. If not, insert *key* with a value of *default* and return *default*. *default* defaults to ``0``. """ if key in self._dict: return self._dict[key] else: self._dict[key] = default self._list.add((default, key)) return default def elements(self): """ Return an iterator over elements repeating each as many times as its count. Elements are returned in value sort-order. If an element’s count is less than one, elements() will ignore it. """ values = (repeat(key, value) for value, key in self._list) return chain.from_iterable(values) def most_common(self, count=None): """ Return a list of the `count` highest priority elements with their priority. If `count` is not specified, `most_common` returns *all* elements in the dict. Elements with equal counts are ordered by key. """ _list, _dict = self._list, self._dict if count is None: return [(key, value) for value, key in reversed(_list)] end = len(_dict) start = end - count return [(key, value) for value, key in reversed(_list[start:end])] def subtract(self, elements): """ Elements are subtracted from an iterable or from another mapping (or counter). Like dict.update() but subtracts counts instead of replacing them. Both inputs and outputs may be zero or negative. """ self -= Counter(elements) def tally(self, *args, **kwargs): """ Elements are counted from an iterable or added-in from another mapping (or counter). Like dict.update() but adds counts instead of replacing them. Also, the iterable is expected to be a sequence of elements, not a sequence of (key, value) pairs. """ self += Counter(*args, **kwargs) @classmethod def count(self, *args, **kwargs): """ Consume `args` and `kwargs` with a Counter and use that mapping to initialize a PriorityDict. """ return PriorityDict(Counter(*args, **kwargs)) def update(self, *args, **kwargs): """ Update the dictionary with the key/value pairs from *other*, overwriting existing keys. *update* accepts either another dictionary object or an iterable of key/value pairs (as a tuple or other iterable of length two). If keyword arguments are specified, the dictionary is then updated with those key/value pairs: ``d.update(red=1, blue=2)``. """ _list, _dict = self._list, self._dict if len(args) == 1 and len(kwargs) == 0 and isinstance(args[0], Mapping): items = args[0] else: items = dict(*args, **kwargs) if (10 * len(items)) > len(_dict): _dict.update(items) _list.clear() _list.update((value, key) for key, value in iteritems(_dict)) else: for key, value in iteritems(items): old_value = _dict[key] _list.remove((old_value, key)) _dict[key] = value _list.add((value, key)) def index(self, key): """ Return the smallest *i* such that `d.iloc[i] == key`. Raises KeyError if *key* is not present. """ value = self._dict[key] return self._list.index((value, key)) def bisect_left(self, value): """ Similar to the ``bisect`` module in the standard library, this returns an appropriate index to insert *value* in PriorityDict. If *value* is already present in PriorityDict, the insertion point will be before (to the left of) any existing entries. """ return self._list.bisect_left((value,)) def bisect(self, value): """Same as bisect_left.""" return self._list.bisect((value,)) def bisect_right(self, value): """ Same as `bisect_left`, but if *value* is already present in PriorityDict, the insertion point will be after (to the right of) any existing entries. """ return self._list.bisect_right((value, _Biggest)) def __iadd__(self, that): """Add values from `that` mapping.""" _list, _dict = self._list, self._dict if len(_dict) == 0: _dict.update(that) _list.update((value, key) for key, value in iteritems(_dict)) elif len(that) * 3 > len(_dict): _list.clear() for key, value in iteritems(that): if key in _dict: _dict[key] += value else: _dict[key] = value _list.update((value, key) for key, value in iteritems(_dict)) else: for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _list.remove((old_value, key)) value = old_value + value _dict[key] = value _list.add((value, key)) return self def __isub__(self, that): """Subtract values from `that` mapping.""" _list, _dict = self._list, self._dict if len(_dict) == 0: _dict.clear() _list.clear() elif len(that) * 3 > len(_dict): _list.clear() for key, value in iteritems(that): if key in _dict: _dict[key] -= value _list.update((value, key) for key, value in iteritems(_dict)) else: for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _list.remove((old_value, key)) value = old_value - value _dict[key] = value _list.add((value, key)) return self def __ior__(self, that): """Or values from `that` mapping (max(v1, v2)).""" _list, _dict = self._list, self._dict if len(_dict) == 0: _dict.update(that) _list.update((value, key) for key, value in iteritems(_dict)) elif len(that) * 3 > len(_dict): _list.clear() for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _dict[key] = old_value if old_value > value else value else: _dict[key] = value _list.update((value, key) for key, value in iteritems(_dict)) else: for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _list.remove((old_value, key)) value = old_value if old_value > value else value _dict[key] = value _list.add((value, key)) return self def __iand__(self, that): """And values from `that` mapping (min(v1, v2)).""" _list, _dict = self._list, self._dict if len(_dict) == 0: _dict.clear() _list.clear() elif len(that) * 3 > len(_dict): _list.clear() for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _dict[key] = old_value if old_value < value else value _list.update((value, key) for key, value in iteritems(_dict)) else: for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _list.remove((old_value, key)) value = old_value if old_value < value else value _dict[key] = value _list.add((value, key)) return self def __add__(self, that): """Add values from this and `that` mapping.""" result = PriorityDict() _list, _dict = result._list, result._dict _dict.update(self._dict) for key, value in iteritems(that): if key in _dict: _dict[key] += value else: _dict[key] = value _list.update((value, key) for key, value in iteritems(_dict)) return result def __sub__(self, that): """Subtract values in `that` mapping from this.""" result = PriorityDict() _list, _dict = result._list, result._dict _dict.update(self._dict) for key, value in iteritems(that): if key in _dict: _dict[key] -= value _list.update((value, key) for key, value in iteritems(_dict)) return result def __or__(self, that): """Or values from this and `that` mapping.""" result = PriorityDict() _list, _dict = result._list, result._dict _dict.update(self._dict) for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _dict[key] = old_value if old_value > value else value else: _dict[key] = value _list.update((value, key) for key, value in iteritems(_dict)) return result def __and__(self, that): """And values from this and `that` mapping.""" result = PriorityDict() _list, _dict = result._list, result._dict _dict.update(self._dict) for key, value in iteritems(that): if key in _dict: old_value = _dict[key] _dict[key] = old_value if old_value < value else value _list.update((value, key) for key, value in iteritems(_dict)) return result def __eq__(self, that): """Compare two mappings for equality.""" if isinstance(that, PriorityDict): that = that._dict return self._dict == that def __ne__(self, that): """Compare two mappings for inequality.""" if isinstance(that, PriorityDict): that = that._dict return self._dict != that def __lt__(self, that): """Compare two mappings for less than.""" if isinstance(that, PriorityDict): that = that._dict _dict = self._dict return (_dict != that and self <= that) def __le__(self, that): """Compare two mappings for less than equal.""" if isinstance(that, PriorityDict): that = that._dict _dict = self._dict return (len(_dict) <= len(that) and all(_dict[key] <= that[key] if key in that else False for key in _dict)) def __gt__(self, that): """Compare two mappings for greater than.""" if isinstance(that, PriorityDict): that = that._dict _dict = self._dict return (_dict != that and self >= that) def __ge__(self, that): """Compare two mappings for greater than equal.""" if isinstance(that, PriorityDict): that = that._dict _dict = self._dict return (len(_dict) >= len(that) and all(_dict[key] >= that[key] if key in _dict else False for key in that)) def isdisjoint(self, that): """ Return True if no key in `self` is also in `that`. This doesn't check that the value is greater than zero. To remove keys with value less than or equal to zero see *clean*. """ return not any(key in self for key in that) def items(self): """ Return a list of the dictionary's items (``(key, value)`` pairs). Items are ordered by their value from least to greatest. """ return list((key, value) for value, key in self._list) def iteritems(self): """ Return an iterable over the items (``(key, value)`` pairs) of the dictionary. Items are ordered by their value from least to greatest. """ return iter((key, value) for value, key in self._list) @not26 def viewitems(self): """ In Python 2.7 and later, return a new `ItemsView` of the dictionary's items. Beware iterating the `ItemsView` as items are unordered. In Python 2.6, raise a NotImplementedError. """ if hexversion < 0x03000000: return self._dict.viewitems() else: return self._dict.items() def keys(self): """ Return a list of the dictionary's keys. Keys are ordered by their corresponding value from least to greatest. """ return list(key for value, key in self._list) def iterkeys(self): """ Return an iterable over the keys of the dictionary. Keys are ordered by their corresponding value from least to greatest. """ return iter(key for value, key in self._list) @not26 def viewkeys(self): """ In Python 2.7 and later, return a new `KeysView` of the dictionary's keys. Beware iterating the `KeysView` as keys are unordered. In Python 2.6, raise a NotImplementedError. """ if hexversion < 0x03000000: return self._dict.viewkeys() else: return self._dict.keys() def values(self): """ Return a list of the dictionary's values. Values are ordered from least to greatest. """ return list(value for value, key in self._list) def itervalues(self): """ Return an iterable over the values of the dictionary. Values are iterated from least to greatest. """ return iter(value for value, key in self._list) @not26 def viewvalues(self): """ In Python 2.7 and later, return a `ValuesView` of the dictionary's values. Beware iterating the `ValuesView` as values are unordered. In Python 2.6, raise a NotImplementedError. """ if hexversion < 0x03000000: return self._dict.viewvalues() else: return self._dict.values() def __repr__(self): """Return a string representation of PriorityDict.""" return 'PriorityDict({0})'.format(repr(dict(self))) def _check(self): self._list._check() assert len(self._dict) == len(self._list) assert all(key in self._dict and self._dict[key] == value for value, key in self._list)
def create_group_diagram(entities, entities_times, all_time_stamps, d, time_shift, first, last, birds, save_path): print "this code is used" # Initialize kml-file to store the GD representatives kml = simplekml.Kml() # find common start and end time stamps of the trajectories start_times = SortedList() end_times = SortedList() for entityTime in entities_times: start_times.add(entityTime[0]) end_times.add(entityTime[-1]) common_start_time = start_times[-1] common_end_time = end_times[0] all_common_time_stamps = SortedList(all_time_stamps[all_time_stamps.index( common_start_time):all_time_stamps.index(common_end_time)]) # COMPUTING ALL SPLIT AND MERGE EVENTS event_count = 0 event_times = SortedList() print "number of all common time stamps:", len(all_common_time_stamps) # loop through all time stamps and compute the events for time_index, timestamp in enumerate(all_common_time_stamps[:-1]): for index1 in range(0, len(entities)): start_segment_1 = entities_times[index1].index(timestamp) end_segment_1 = start_segment_1 + 1 for index2 in range(0, len(entities)): start_segment_2 = entities_times[index2].index(timestamp) end_segment_2 = start_segment_2 + 1 new_events = compare_two_segments( entities[index1][start_segment_1][1], entities[index1][end_segment_1][1], entities[index2][start_segment_2][1], entities[index2][end_segment_2][1], entities[index1][start_segment_1][2], entities[index1][end_segment_1][2], entities[index2][start_segment_2][2], entities[index2][end_segment_2][2], time_index + first, all_common_time_stamps, d) # print new_events if len(new_events) > 0: for event in new_events: event_times.add(event) event_count += 1 # print event_count # propagate event time stamps to all trajectories adding_count = 0 print "Number of events:", len(event_times) for timestamp in event_times: if timestamp not in all_common_time_stamps: all_common_time_stamps.add(timestamp) adding_count += 1 for index, entity in enumerate(entities): length = len(entities_times[index]) # check if the current entity has this timestamp, if not add it and interpolate the # coordinates but only if it is in the range of the trajectory. Otherwise interpolation is not possible if timestamp not in entities_times[index] and entities_times[ index].bisect(timestamp) > 0 and entities_times[ index].bisect(timestamp) < length: entity.add( interpolate(entity, entities_times[index], timestamp)) entities_times[index].add(timestamp) # solve set cover instance for every time stamp, return the segments of the solution and draw them universe = set(range(0, len(entities))) cover_size = [] cover_numbers = [] # for each entity initialize all_together = 0 all_together_time = 0 number_of_representations = 0 start = time.time() for time_index, timestamp in enumerate(all_common_time_stamps[first:last]): if timestamp in event_times or timestamp == all_common_time_stamps[ first]: set_list = [] start_segment_common = all_common_time_stamps.index(timestamp) end_segment_common = start_segment_common + 1 for index in range(0, len(entities)): current_set = set() start_segment = entities_times[index].index(timestamp) end_segment = start_segment + 1 segment = [[ entities[index][start_segment][1], entities[index][start_segment][2] ], [ entities[index][end_segment][1], entities[index][end_segment][2] ]] for number, entity in enumerate(entities): if within_alpha_similar_time_distance( segment, timestamp, all_common_time_stamps[end_segment_common], entity, entities_times[number], time_shift, d, 0): current_set.add(number) set_list.append(current_set) result = set_cover(universe, set_list) # print time_index, result cover_size.append(len(result[0])) number_of_representations += len(result[0]) if len(result[0]) == 1: all_together += 1 all_together_time += all_common_time_stamps[time_index + 1] - timestamp cover_numbers.append(result[1]) for indexSet, entity in enumerate(result[1]): points = [ (entities[entity][entities_times[entity].index(timestamp)][1], entities[entity][entities_times[entity].index(timestamp)][2]), (entities[entity][entities_times[entity].index(timestamp) + 1][1], entities[entity][entities_times[entity].index(timestamp) + 1][2]) ] lin = kml.newlinestring(coords=points) # needs to be adjusted for groups with more than 4 entities if entity == 0: lin.style.linestyle.color = simplekml.Color.navy elif entity == 1: lin.style.linestyle.color = simplekml.Color.darkgreen elif entity == 2: lin.style.linestyle.color = simplekml.Color.darkorange elif entity == 3: lin.style.linestyle.color = simplekml.Color.darkred lin.style.linestyle.width = result[2][indexSet] * 2 end = time.time() print "time to construct and solve Set Cover", d, time_shift, end - start all_together = 1.0 * all_together / (len(all_common_time_stamps)) number_of_representations = 1.0 * number_of_representations / ( len(all_common_time_stamps)) size = os.path.join( save_path, "outputSize_" + str(d) + '_' + str(time_shift) + ".txt") numbers = os.path.join( save_path, "outputNumber_" + birds + str(d) + '_' + str(time_shift) + ".txt") kml_save = os.path.join( save_path, 'groupDiagram2D' + birds + str(d) + '_' + str(time_shift) + '.kml') file1 = open(size, "w") simplejson.dump(cover_size, file1) file1.close() file2 = open(numbers, 'w') simplejson.dump(cover_numbers, file2) file2.close() kml.save(kml_save) return all_together, number_of_representations
class BarGridKernel(Kernel): ''' Store a kernel of n dimensions as a list of bars in the space of dimension (n-1). A bar is given by its start and its end coordinates, and corresponds to the hull of the viable points in the last dimension for each coordinates in the space of dimension (n-1). The order of the dimensions may have been changed, and the last dimension of these data may not correspond to the last dimension of the viability problem. Therefore the attribute ``permutation`` give a matrix describing the permutation of the dimension. ''' def __init__(self, originCoords, oppositeCoords, intervalNumberperaxis, permutation=None, kernelMinPoint=None, kernelMaxPoint=None,data=[], metadata={}): super(BarGridKernel, self).__init__(metadata) self.originCoords = np.array(originCoords, float) self.oppositeCoords = np.array(oppositeCoords, float) self.intervalNumberperaxis = np.array(intervalNumberperaxis, int) self.bars = SortedList(data) if permutation is None: self.permutation = np.eye(len(originCoords),dtype = int) else: self.permutation = permutation if kernelMinPoint is None: self.kernelMinPoint = [] for i in range(len(originCoords)): self.kernelMinPoint.append(intervalNumberperaxis[i]) else : self.kernelMinPoint = kernelMinPoint if kernelMaxPoint is None: self.kernelMaxPoint = [-1] * len(originCoords) else : self.kernelMaxPoint = kernelMaxPoint self.metadata.update(self.getDataAttributes()) @staticmethod def getFormatCode(): return "bars" def getDataAttributes(self): da = super(BarGridKernel, self).getDataAttributes() da['origin'] = self.originCoords da['opposite'] = self.oppositeCoords da['intervals'] = self.intervalNumberperaxis da['permutation']= self.permutation da['maxPoint']= self.kernelMaxPoint da['minPoint']= self.kernelMinPoint return da @classmethod def initFromHDF5(cls, metadata, attrs, data): ''' Create an object of class BarGridKernel from attributes and data loaded from an HDF5 file. This method is intended to be used by the method hdf5common.readKernel ''' return cls( originCoords=attrs['origin'], oppositeCoords=attrs['opposite'], intervalNumberperaxis=attrs['intervals'], permutation=attrs['permutation'], kernelMinPoint=attrs['minPoint'], kernelMaxPoint=attrs['maxPoint'], data=data.tolist(), metadata=metadata ) def getData(self): return np.array(list(self.bars), dtype='int64') def getIntervalSizes(self): ''' Give the coordinates of the point of the grid with minimal coordinates ''' intervalsizes = [] intervalsizes = list((self.oppositeCoords-self.originCoords)/self.intervalNumberperaxis) return intervalsizes def getMinFrameworkBounds(self): return list(self.originCoords-np.array(self.getIntervalSizes())/2) def getMaxFrameworkBounds(self): return list(self.oppositeCoords+np.array(self.getIntervalSizes())/2) def getMinBounds(self): ''' Give the coordinates of the point of the vino with minimal coordinates ''' minbounds = [] intervalSizes = np.array(self.getIntervalSizes()) permutOriginCoords = np.dot(self.permutation, self.originCoords) permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords) permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis) minbounds = list(np.dot(np.transpose(self.permutation),permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*self.kernelMinPoint/permutIntervalNumberperaxis)) minbounds = minbounds - intervalSizes/2 return minbounds def getMaxBounds(self): ''' Give the coordinates of the point of the vino with maximal coordinates ''' maxbounds = [] intervalSizes = np.array(self.getIntervalSizes()) permutOriginCoords = np.dot(self.permutation, self.originCoords) permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords) permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis) maxbounds = list(np.dot(np.transpose(self.permutation),permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*self.kernelMaxPoint/permutIntervalNumberperaxis)) maxbounds = maxbounds + intervalSizes/2 return maxbounds def getDataToPlot(self): data = [] permutOriginCoords = np.dot(self.permutation, self.originCoords) permutOppositeCoords = np.dot(self.permutation, self.oppositeCoords) permutIntervalNumberperaxis = np.dot(self.permutation, self.intervalNumberperaxis) for i in range(len(self.bars)): data.append([ permutOriginCoords+(permutOppositeCoords-permutOriginCoords)*np.array(self.bars[i][:-1])/permutIntervalNumberperaxis, permutOriginCoords[-1]+(permutOppositeCoords[-1]-permutOriginCoords[-1])*self.bars[i][-1]/permutIntervalNumberperaxis[-1] ]) perm = np.dot(self.permutation,np.arange(len(self.originCoords))) data = [self.getMinFrameworkBounds()+self.getMaxFrameworkBounds()+self.getIntervalSizes()+[perm], data] return data def getTotalPointNumber(self): return sum([elt[-1] - elt[-2] + 1 for elt in self.bars]) def toRegularGridKernel(self): ''' Convert the kernel to the regular grid representation. Returns an instance of RegularGridKernel. The returned grid is trimed to not include empty portion of grid. ''' minPoint = np.array(self.kernelMinPoint) maxPoint = np.array(self.kernelMaxPoint) dimensionsExtents = maxPoint - minPoint + 1 grid = RegularGridKernel(self.originCoords, self.intervalNumberperaxis, dimensionsExtents, metadata=self.metadata) for bar in self.bars: barPosition = (bar[:-2]-minPoint[:-1]).tolist() grid.grid[tuple(barPosition)].put(list(range(bar[-2], bar[-1] + 1)), True) return grid def intersectionwithBarGridKernel(self,othergrid): ''' Returns an instance of BarGridKernel which is the intersection of two BarGridKernels with the same underlying grid characteristics ''' data = [] grid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,self.permutation,None,None,data,self.metadata) barsindex = 0 otherbarsindex = 0 while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)): actualbarposition = self.bars[barsindex][:-2] # print("actualbarposition[0] ::%d " %actualbarposition[0]) while (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] < self.bars[barsindex][:-2]): otherbarsindex = otherbarsindex + 1 while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] == self.bars[barsindex][:-2]): if othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-2]: otherbarsindex = otherbarsindex + 1 elif othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-1]: barsindex = barsindex + 1 elif othergrid.bars[otherbarsindex][-1] > self.bars[barsindex][-1]: grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), self.bars[barsindex][-1]) barsindex = barsindex + 1 elif othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-1]: grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), othergrid.bars[otherbarsindex][-1]) otherbarsindex = otherbarsindex + 1 else: grid.addBar(self.bars[barsindex][:-2], max(othergrid.bars[otherbarsindex][-2], self.bars[barsindex][-2]), othergrid.bars[otherbarsindex][-1]) otherbarsindex = otherbarsindex + 1 barsindex = barsindex + 1 while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] > self.bars[barsindex][:-2]): barsindex = barsindex + 1 return grid def MinusBarGridKernel(self,othergrid): ''' Returns an instance of BarGridKernel which is the element of the BarGridKernels which are not in the other one. The Bargridkernel have the same underlying grid characteristics ''' data = [] grid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,self.permutation,None,None,data,self.metadata) barsindex = 0 otherbarsindex = 0 while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)): while (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] < self.bars[barsindex][:-2]): otherbarsindex = otherbarsindex + 1 alreadycut = False while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] == self.bars[barsindex][:-2]): if othergrid.bars[otherbarsindex][-1] < self.bars[barsindex][-2]: otherbarsindex = otherbarsindex + 1 elif othergrid.bars[otherbarsindex][-1] >= self.bars[barsindex][-1]: if othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-2]: if alreadycut: grid.addBar(self.bars[barsindex][:-2], remember, othergrid.bars[otherbarsindex][-2]-1) else : grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], othergrid.bars[otherbarsindex][-2]-1) barsindex = barsindex + 1 alreadycut = False else : if othergrid.bars[otherbarsindex][-2] > self.bars[barsindex][-2]: if alreadycut: grid.addBar(self.bars[barsindex][:-2], remember, othergrid.bars[otherbarsindex][-2]-1) else : grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], othergrid.bars[otherbarsindex][-2]-1) alreadycut = True remember = othergrid.bars[otherbarsindex][-1]+1 otherbarsindex = otherbarsindex + 1 while (barsindex < len(self.bars)) and (otherbarsindex < len(othergrid.bars)) and (othergrid.bars[otherbarsindex][:-2] > self.bars[barsindex][:-2]): if alreadycut: grid.addBar(self.bars[barsindex][:-2], remember, self.bars[barsindex][-1]) barsindex = barsindex + 1 alreadycut = False else : grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], self.bars[barsindex][-1]) barsindex = barsindex + 1 if (otherbarsindex >= len(othergrid.bars)): while (barsindex < len(self.bars)): if alreadycut: grid.addBar(self.bars[barsindex][:-2], remember, self.bars[barsindex][-1]) barsindex = barsindex + 1 alreadycut = False else : grid.addBar(self.bars[barsindex][:-2], self.bars[barsindex][-2], self.bars[barsindex][-1]) barsindex = barsindex + 1 return grid def toBarGridKernel(self, newOriginCoords, newOppositeCoords, newIntervalNumberperaxis): ''' Convert a BarGridKernel to another BarGridKernel with another underlying grid. Returns an instance of BarGridKernel. ''' dimension = len(self.originCoords) actualbarposition = np.zeros(dimension-1,int) barsindex = 0 # converting to numpy arrays newOriginCoords = np.array(newOriginCoords,float) newIntervalNumberperaxis = np.array(newIntervalNumberperaxis,float) permutnewOriginCoords = np.dot(self.permutation, newOriginCoords) # permuting coordinates permutnewIntervalNumberperaxis = np.dot(self.permutation, newIntervalNumberperaxis) permutnewpas = np.dot(self.permutation,(np.array(newOppositeCoords,float) - newOriginCoords) / newIntervalNumberperaxis) permutOriginCoords = np.dot(self.permutation, self.originCoords) permutinversepas = np.dot(self.permutation, self.intervalNumberperaxis / (self.oppositeCoords - self.originCoords)) data = [] grid = BarGridKernel(newOriginCoords,newOppositeCoords,newIntervalNumberperaxis,self.permutation,None,None,data,self.metadata) # oups = 0 # while (oups < 1) : # oups = 1 while(actualbarposition[0]<permutnewIntervalNumberperaxis[0]+1): realpoint = permutnewOriginCoords[:-1] + actualbarposition * permutnewpas[:-1] intpoint = (realpoint-permutOriginCoords[:-1]) * permutinversepas[:-1] intpoint = [int(e+0.5) for e in intpoint] while (barsindex < len(self.bars)) and (self.bars[barsindex][:2] < intpoint): barsindex = barsindex+1 barinprocess = False # print intpoint while (barsindex < len(self.bars)) and (self.bars[barsindex][:-2] == intpoint): inf = self.bars[barsindex][-2] realinf = inf/permutinversepas[-1] +permutOriginCoords[-1] intinf = int((realinf-permutnewOriginCoords[-1])/permutnewpas[-1]+0.5) sup = self.bars[barsindex][-1] realsup = sup/permutinversepas[-1] +permutOriginCoords[-1] intsup = int((realsup-permutnewOriginCoords[-1])/permutnewpas[-1]+0.5) # print realinf # print realsup # print intinf # print intsup if (intinf<permutnewIntervalNumberperaxis[-1]+1) or (intsup >=0): if barinprocess == True : if intinf == grid.bars[-1][-1]: grid.bars[-1][-1] = min(intsup,permutnewIntervalNumberperaxis[-1]+1) else : grid.addBar(actualbarposition.tolist(), max(intinf,0), min(intsup,permutnewIntervalNumberperaxis[-1]+1)) else : grid.addBar(actualbarposition.tolist(), max(intinf,0), min(intsup,permutnewIntervalNumberperaxis[-1]+1)) barinprocess = True barsindex = barsindex+1 for i in range(dimension-1): if ((i == dimension - 2) or (actualbarposition[dimension-2-i]<permutnewIntervalNumberperaxis[dimension-2-i])): actualbarposition[dimension-2-i] = actualbarposition[dimension-2-i]+1 break else : actualbarposition[dimension-2-i] = 0 return grid def addBar(self, coords, inf, sup): # First, we collect the bars already present at the position 'coords' # and we merge the bar to add with the existing ones # two bars will be merged if at least they touch themselves # "touch" means that a lower bound of one bar is equals to the (upper bound of the other one) + 1 insertion_point = self.bars.bisect(coords) merged = False mergedBarsToRemove=[] rightExpanded = False while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==coords: if inf > self.bars[insertion_point][-1] + 1: # the new bar doesn't touch the right of the current one # we should test if it isn't equals to the upper bound + 1 to ensure that it doesn't touch insertion_point += 1 continue; if rightExpanded: # a previous bar has been modified, we check that it doesn't cross or touch the current bar if self.bars[insertion_point][-2] <= self.bars[insertion_point-1][-1] + 1: # the previous bar now intersects the lower bound of the current one # so let's merge the two bars self.bars[insertion_point][-2] = self.bars[insertion_point-1][-2] if self.bars[insertion_point][-2] <= self.bars[insertion_point-1][-2]: # the previous bar completly overlaps the current one self.bars[insertion_point][-2] = self.bars[insertion_point-1][-2] mergedBarsToRemove.append(insertion_point-1) rightExpanded = True elif inf >= self.bars[insertion_point][-2] and inf <= self.bars[insertion_point][-1] + 1: # the lower bound of the inserted bar is inside the current one merged = True if sup > self.bars[insertion_point][-1]: # the upper bound is outside the current bar, so we update the upper bound self.bars[insertion_point][-1] = sup rightExpanded = True elif inf < self.bars[insertion_point][-2]: # the lower bound of the inserted bar is before the current one if sup >= self.bars[insertion_point][-2] - 1: # the inserted bar crosses or touches the current bar, so we update the lower bound self.bars[insertion_point][-2] = inf merged = True if sup > self.bars[insertion_point][-1]: # the inserted bound is globally bigger than the current one, so we update also the upper bound self.bars[insertion_point][-1] = sup rightExpanded = True insertion_point += 1 for index in reversed(mergedBarsToRemove): del self.bars[index] if not merged: self.bars.add(coords[:] + [inf,sup]) self.kernelMinPoint[:-1] = [min(x) for x in zip(self.kernelMinPoint[:-1],coords)] self.kernelMinPoint[-1] = min(self.kernelMinPoint[-1], inf) self.kernelMaxPoint[:-1] = [max(x) for x in zip(self.kernelMaxPoint[:-1],coords)] self.kernelMaxPoint[-1] = max(self.kernelMaxPoint[-1], sup) def getBars(self): return self.bars def isInSet(self, point): ''' Returns if point belongs to the BarGridKernel. This method will find the cell where to lookup a bar containing the point. If not, the point is not considered in the set. Technical details: In a BarGrid, each cell of the (n-1) dimensional space is stored by its index inside a matrix between two opposite points and the number of cells in each dimension. First, this method will compute the index(es) of the cell where to lookup for bars, then it will look for the last dimension to check if the point is inside a cell covered by a bar in the selected cell of the (n-1) dimensional space. Note that if the point is exactly between several cells (on a vertex or on an edge), the method will check the bars of all the touching cells. Let's consider s(i) the size of a cell (the step size) on the dimension i, p(i) the coordinates of the point requested on the dimension i, and c(i) the coordinates of the center of a cell of the BarGrid in the dimension i, thus we consider that the point p is inside the cell c if: - p(i)>=c(i)-s(i)/2 (after the left side of the cell) - p(i)<=c(i)+s(i)/2 (before the right side of the cell) - for i belongs to [0;n-1] ''' result = False point = np.array(point,float) # first we need to project the point into the cells coordinate system point_int = self.intervalNumberperaxis * (point - self.originCoords)/(self.oppositeCoords - self.originCoords) point_int = np.dot(self.permutation, np.transpose(point_int)) points = [point_int] for i,coord in enumerate(point_int): new_points = [] for p in points: if (coord%1)==0.5: # the point is exactly between two cells on the current dimension left = [pp for pp in p] left[i] = int(math.floor(left[i])) right = [pp for pp in p] right[i] = int(math.ceil(right[i])) new_points.extend([left, right]) else: # we just need to round to the nearest cell center new_point = [pp for pp in p] new_point[i] = int(round(new_point[i])) new_points.append(new_point) points=new_points l = len(point) # we will look at each bar if they are positioned in the coordinates # in (n-1) dimensions space than our point for point in points: candidateBar = False for bar in self.bars: if point[:-1] == bar[:-2]: # we have reached the interesting zone candidateBar = True # is our point in the bar? if (point[l-1] >= bar[l-1]-0.5) and (point[l-1] <= bar[l] + 0.5): result = True break elif candidateBar: # we have passed the position in (n-1) dimensions space, so we can't find candidates anymore break return result def permute(self,permutation): ''' Create a BarGrid corresponding to the same data as the initial one but with a different permutation of the variables : np.dot(np.transpose(permutation),self.permutation) instead of self.permutation ''' griddata = [] unitbars = [] dimension = len(self.originCoords) matid = np.identity(dimension,dtype = int) b = False for i in range(dimension): for j in range(dimension): if permutation[i][j] != matid[i][j]: b = True if b: permutegrid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,np.dot(np.transpose(permutation),self.permutation),np.dot(np.transpose(permutation),self.kernelMinPoint),np.dot(np.transpose(permutation),self.kernelMaxPoint),griddata,self.metadata) if permutation[dimension-1][dimension-1] == 0: barposition = [0]*(dimension-1) increment = [0]*len(barposition) increment.append(1) oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1] oldindex = oldincrement.index(1) newincrement = list(np.dot(np.transpose(permutation),np.array(increment,int)))[:-1] newindex = newincrement.index(1) NmaxUsefullBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[oldindex] barposition.append(0) permutnewIntervalNumberperaxis = np.dot(permutegrid.permutation, permutegrid.intervalNumberperaxis) if (newindex == 0): indexbar = 1 else : indexbar = 0 if dimension == 2: # print "barposition" # print barposition usefuloldbars = [] newbars = [] oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1] for i in range(NmaxUsefullBars+1): oldbarposition[oldindex] = i # print oldbarposition insertion_point = self.bars.bisect(oldbarposition) while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition: usefuloldbars.append(self.bars[insertion_point]) insertion_point = insertion_point+1 # print usefuloldbars for oldbar in usefuloldbars: # print "newoldbar" # print oldbar level = oldbar[oldindex] unitbar = barposition[:-1] + [level,level] mini = oldbar[-2] maxi = oldbar[-1] newbartoupdateindex = mini if newbars: k = 0 while k <len(newbars): newbar = newbars[k] # print "ole" # print newbars # print newbar # print k if (newbar[newindex] > maxi): break elif (newbar[newindex] >= mini): # print newbar[newindex] if (newbar[newindex] > newbartoupdateindex): for l in range(newbartoupdateindex,newbar[newindex]): unitbar[newindex] = l newbars.insert(k,copy.copy(unitbar)) k=k+1 newbartoupdateindex = newbar[newindex] if (newbar[-1] == level-1): newbar[-1] = level newbartoupdateindex = newbar[newindex]+1 k = k+1 for l in range(newbartoupdateindex,maxi+1): unitbar[newindex] = l newbars.insert(k,copy.copy(unitbar)) k = k+1 else : for l in range(mini,maxi+1): unitbar[newindex] = l newbars.append(copy.copy(unitbar)) # print "newbars" # print newbars permutegrid.bars.update(newbars) else: while(barposition[indexbar]<permutnewIntervalNumberperaxis[indexbar]+1): # print "barposition" # print barposition usefuloldbars = [] newbars = [] oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1] for i in range(NmaxUsefullBars+1): oldbarposition[oldindex] = i # print oldbarposition insertion_point = self.bars.bisect(oldbarposition) while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition: usefuloldbars.append(self.bars[insertion_point]) insertion_point = insertion_point+1 # print usefuloldbars for oldbar in usefuloldbars: # print "newoldbar" # print oldbar level = oldbar[oldindex] unitbar = barposition[:-1] + [level,level] mini = oldbar[-2] maxi = oldbar[-1] newbartoupdateindex = mini if newbars: k = 0 while k <len(newbars): newbar = newbars[k] # print "ole" # print newbars # print newbar # print k if (newbar[newindex] > maxi): break elif (newbar[newindex] >= mini): # print newbar[newindex] if (newbar[newindex] > newbartoupdateindex): for l in range(newbartoupdateindex,newbar[newindex]): unitbar[newindex] = l newbars.insert(k,copy.copy(unitbar)) k=k+1 newbartoupdateindex = newbar[newindex] if (newbar[-1] == level-1): newbar[-1] = level newbartoupdateindex = newbar[newindex]+1 k = k+1 for l in range(newbartoupdateindex,maxi+1): unitbar[newindex] = l newbars.insert(k,copy.copy(unitbar)) k = k+1 else : for l in range(mini,maxi+1): unitbar[newindex] = l newbars.append(copy.copy(unitbar)) # print "newbars" # print newbars permutegrid.bars.update(newbars) for i in range(dimension-1): if ((dimension-2-i) != newindex): if ((i == dimension - 2- indexbar) or (barposition[dimension-2-i]<permutnewIntervalNumberperaxis[dimension-2-i])): barposition[dimension-2-i] = barposition[dimension-2-i]+1 break else : barposition[dimension-2-i] = 0 else: for bar in self.bars : tpermutation = np.transpose(permutation) permutegrid.bars.add(list(np.dot(tpermutation,bar[:-1]))+[bar[-1]]) else: permutegrid = BarGridKernel(self.originCoords,self.oppositeCoords,self.intervalNumberperaxis,np.dot(np.transpose(permutation),self.permutation),np.dot(np.transpose(permutation),self.kernelMinPoint),np.dot(np.transpose(permutation),self.kernelMaxPoint),list(self.bars),self.metadata) return permutegrid def buildNewBars(self,barposition,permutation,data): newdata = [] unitbar = [] increment = [0]*len(barposition) increment.append(1) newincrement = list(np.dot(np.transpose(permutation),np.array(increment,int)))[:-1] newindex = newincrement.index(1) oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1] oldindex = oldincrement.index(1) NmaxNewBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[-1] # print newincrement # print newindex # print oldincrement # print oldindex # print NmaxNewBars for oldbar in data: level = oldbar[oldindex] unitbar = barposition + [level,level] mini = oldbar[-2] maxi = oldbar[-1] newbartoupdateindex = mini if newdata: for k in range(len(newdata)): # print "ole" # print newdata # print k newbar = newdata[k] if (newbar[newindex] >= mini) and (newbar[newindex] <= maxi): # print newbar[newindex] if (newbar[newindex] > newbartoupdateindex): for l in range(newbartoupdateindex,newbar[newindex]): unitbar[newindex] = l newdata.insert(k,copy.copy(unitbar)) k=k+1 if (newbar[-1] == level-1): newbar[-1] = level newbartoupdateindex = newbar[newindex]+1 for l in range(newbartoupdateindex,maxi+1): unitbar[newindex] = l newdata.insert(len(newdata),copy.copy(unitbar)) else : for l in range(mini,maxi+1): unitbar[newindex] = l newdata.append(copy.copy(unitbar)) return newdata def findUsefullBars(self,barposition,permutation): data = [] increment = [0]*len(barposition) barposition.append(0) increment.append(1) oldincrement = list(np.dot(permutation,np.array(increment,int)))[:-1] index = oldincrement.index(1) oldbarposition = list(np.dot(permutation,np.array(barposition,int)))[:-1] NmaxUsefullBars = list(np.dot(self.permutation, self.intervalNumberperaxis))[index] # print oldbarposition # print oldincrement # print index # print NmaxUsefullBars for i in range(NmaxUsefullBars+1): oldbarposition[index] = i insertion_point = self.bars.bisect(oldbarposition) # print oldbarposition # print insertion_point while insertion_point<len(self.bars) and self.bars[insertion_point][:-2]==oldbarposition: data.append(self.bars[insertion_point]) insertion_point = insertion_point+1 return data
sig_buf.add_signal_fragment(record.signal[i], record.leads[i]) annots = mit.read_annotations(args.r + '.' + args.a) #Delineation of every QRS annotation. for ann in (a for a in annots if mit.is_qrs_annotation(a)): #Temporal window btime = ann.time beg = btime-C.QRS_BANN_DMAX end = btime+C.QRS_EANN_DMAX #Path simplification and signal characterization siginfo = _characterize_signal(beg, end) try: #QRS delineation qrs = delineate_qrs(siginfo) #Output annotations bann = mit.MITAnnotation() bann.code = mit.ECGCodes.WFON bann.time = int(beg+qrs.start) peak = mit.MITAnnotation() peak.code = mit.ECGCodes.NORMAL peak.time = int(beg+qrs.peak) eann = mit.MITAnnotation() eann.code = mit.ECGCodes.WFOFF eann.time = int(beg+qrs.end) output.add(bann) output.add(peak) output.add(eann) except InconsistencyError: warnings.warn('Annotation [{0}] could not be delineated'.format( ann)) mit.save_annotations(output, args.r + '.' + args.o)
class Chain: def __init__(self, anchor): self.anchors = SortedList([anchor]) def align_locally_to_graph(self, graphs, sequence_graphs, linear_ref_nodes, read_sequence, n_mismatches_allowed=7, k=21, print_debug=False): ##logging.debug("Trying to align chain %s locally" % self) # Attempts to align this chain locally to the graph by aligning from first anchor hit in both directions numeric_sequence = letter_sequence_to_numeric(read_sequence) first_anchor = self.anchors[0] sequence_after_first_anchor = read_sequence[ first_anchor.read_offset + k - 1:] #numeric_sequence[first_anchor.read_offset+k-1:] # Forward chromosome = first_anchor.chromosome graph = graphs[str(chromosome)] sequence_graph = sequence_graphs[str(chromosome)] linear_ref_nodes_this_chr = linear_ref_nodes[str(chromosome)] #print("ALIGNING ANCHOR with node %d, offset %d" % (first_anchor.node, first_anchor.offset)) """ aligner = SingleSequenceAligner(graph, sequence_graph, first_anchor.node, first_anchor.offset, sequence_after_first_anchor, n_mismatches_allowed=n_mismatches_allowed, print_debug=debug_read) """ #aligner = LocalGraphAligner(graph, sequence_graph, sequence_after_first_anchor, linear_ref_nodes_this_chr, first_anchor.node, first_anchor.offset) #aligner = Aligner(graph, sequence_graph, int(first_anchor.node), sequence_after_first_anchor) # Align whole sequence at one if print_debug: logging.debug("First anchor read offset: %d" % first_anchor.read_offset) aligner = Aligner(graph, sequence_graph, int(first_anchor.node), read_sequence, n_bp_to_traverse_left=first_anchor.read_offset + 32, n_bp_to_traverse_right=len(read_sequence) + 20) alignment_after, score_after = aligner.align() if print_debug: logging.debug("Alignment after: %s, score: %d" % (alignment_after, score_after)) #print(alignment_after, score_after) if not alignment_after: return Alignment([], [], 0, False, chromosome, first_anchor.position) alignment_before = [] score_before = 0 return Alignment(alignment_before, alignment_after, score_before + score_after, True, chromosome, first_anchor.position) def add_anchor(self, anchor): self.anchors.add(anchor) def anchor_fits(self, anchor): if self.anchors[-1].fits_to_left_of(anchor): return True else: ##logging.debug(" Does not fit to left of %s" % self.anchors[-1]) return False def anchor_is_smaller(self, anchor): return anchor < self.anchors[-1] def __len__(self): return len(self.anchors) def __lt__(self, other): return self.anchors[-1] < other.anchors[-1] def __compare__(self, other): return self.anchors[-1].__compare__(other) def __str__(self): return "-".join(str(anchor) for anchor in self.anchors) def __repr__(self): return self.__str__() def __eq__(self, other): print("Running eq %s == %s" % (self, other)) return str(self) == str(other)
class Image(object): """Manage images. An image is a collection of :class:`Section`s and meta-data. Parameters ---------- sections: iteratable (typically `list`or `tuple` The sections the image should initialized with. meta: object Arbitrary meta-data. """ def __init__(self, sections=None, join=True, meta=None): if meta is None: meta = {} if not sections: sections = [] elif isinstance(sections, Section) or hasattr(sections, "__iter__"): sections = list(sections) else: raise TypeError( "Argument section is of wrong type '{}'".format(sections)) self._sections = SortedList(sections, key=attrgetter("start_address")) self._join = join if join: self.join_sections() _validate_sections(self._sections) self.address = 0 #if meta and not isinstance(meta, MetaRecord): # raise TypeError("meta-data must be of instance 'MetaRecord'") self.meta = meta def __repr__(self): result = [] for segment in self.sections: result.append(repr(segment)) return '\n'.join(result) __str__ = __repr__ def __len__(self): return len(self.sections) def __iter__(self): return iter(self.sections) def __getitem__(self, idx): return self.sections[idx] def __eq__(self, other): if len(self.sections) == len(other.sections): return all(eq(l, r) for l, r in zip(self.sections, other.sections)) else: return False def __ne__(self, other): return not (self == other) def __contains__(self, addr): return any(addr in sec for sec in self.sections) def hexdump(self, fp=sys.stdout): """ """ for idx, section in enumerate(self.sections): print("\nSection #{0:04d}".format(idx), file=fp) print("-" * 13, file=fp) section.hexdump(fp) def _call_address_function(self, func_name, addr, *args): for section in self.sections: if addr in section: func = getattr(section, func_name) return func(addr, *args) raise InvalidAddressError( "Address 0x{:08x} not in range.".format(addr)) def read(self, addr, length): """Read bytes from image. Parameters ---------- addr: int Startaddress. length: int Number of bytes to read. Returns ------- bytes Raises ------ :class:`InvalidAddressError` if `addr` is out of range Note ---- if `addr` + `len` is out of range, result is silently truncated, i.e. without raising an exception. """ return self._call_address_function("read", addr, length) def write(self, addr, length, data): """Write bytes to image. Parameters ---------- addr: int Startaddress. length: int Number of bytes to write. data: bytes Raises ------ :class:`InvalidAddressError` if `addr` is out of range Note ---- if `addr` + `len` is out of range, result is silently truncated, i.e. without raising an exception. """ self._call_address_function("write", addr, length, data) def read_numeric(self, addr, dtype): """ """ return self._call_address_function("readNumeric", addr, dtype) def write_numeric(self, addr, value, dtype): """ """ self._call_address_function("writeNumeric", addr, value, dtype) def read_string(self, addr, encoding="latin1", length=-1): """ """ return self._call_address_function("readString", addr, encoding, length) def write_string(self, addr, value, encoding="latin1"): """ """ self._call_address_function("writeString", addr, value, encoding) def _address_contained(self, address, length): """Check if address space exists. Parameters ---------- address: int length: int Returns ------- bool """ return address in self or (address + length - 1) in self def insert_section(self, data, start_address=None, dont_join=False): """Insert/add a new section to image. Parameters ---------- data: convertible to bytearray() -- s. :func:`_data_converter`. Bytes making up the section. start_address: int dont_join: bool Don't join/merge adjacent section. Raises ------ :class:`InvalidAddressError` Notes ----- Overlapping sections are not supported. To relace a section use :meth:`update_section`. """ start_address = start_address if start_address is not None else self.address # If Address omitted, create continuous address space. if self._address_contained(start_address, len(data)): raise InvalidAddressError("Overlapping address-space") if isinstance(data, str): data = [ord(x) for x in data] # array.array('B',data) self._sections.add(Section(start_address, data)) if self._join: self.join_sections() self.address = start_address + len(data) @property def sections(self): return self._sections def get_section(self, address): """Get :class:`Section` containing `address`. Parameters ---------- address: int Returns ------- :class:`Section` Raises ------ :class:`InvalidAddressError` """ if not address in self: raise InvalidAddressError("Address not in range") result = self._sections.bisect_right(SearchType(address)) return self._sections[result - 1] def update_section(self, data, start_address=None): """ """ if not self._address_contained(start_address, len(data)): raise InvalidAddressError("Address-space not in range") def delete_section(self, start_address=None): """ """ def join_sections(self): """ """ self._sections = join_sections(self._sections) def split(self, at=None, equal_parts=None, remap=None): print("SPLIT-IMAGE", at, equal_parts, remap)
def from_scratch(self): doc = SortedList() doc.add(self.empty_start) doc.add(self.empty_end) return doc
avg_j = np.mean(list(ratings_j.values())) dev_j = { user: (rating - avg_j) for user, rating in ratings_j.items() } dev_j_values = np.array(list(dev_j.values())) sigma_j = np.sqrt(dev_j_values.dot(dev_j_values)) # calculate correlation coefficient numerator = sum(dev_i[m] * dev_j[m] for m in common_users) w_ij = numerator / (sigma_i * sigma_j) # insert into sorted list and truncate # negate weight, because list is sorted ascending # maximum value (1) is "closest" sl.add((-w_ij, j)) if len(sl) > K: del sl[-1] # store the neighbors neighbors.append(sl) # print out useful things if i % 1 == 0: print(i) # using neighbors, calculate train and test MSE def predict(i, u): # calculate the weighted sum of deviations
class TTLCache(object): """A key/value cache implementation where each entry has its own TTL""" def __init__(self, cache_name, timer=time.time): # map from key to _CacheEntry self._data = {} # the _CacheEntries, sorted by expiry time self._expiry_list = SortedList() self._timer = timer def set(self, key, value, ttl): """Add/update an entry in the cache :param key: Key for this entry. :param value: Value for this entry. :param paramttl: TTL for this entry, in seconds. :type paramttl: float """ expiry = self._timer() + ttl self.expire() e = self._data.pop(key, SENTINEL) if e != SENTINEL: self._expiry_list.remove(e) entry = _CacheEntry(expiry_time=expiry, key=key, value=value) self._data[key] = entry self._expiry_list.add(entry) def get(self, key, default=SENTINEL): """Get a value from the cache :param key: The key to look up. :param default: default value to return, if key is not found. If not set, and the key is not found, a KeyError will be raised. :returns a value from the cache, or the default. """ self.expire() e = self._data.get(key, SENTINEL) if e == SENTINEL: if default == SENTINEL: raise KeyError(key) return default return e.value def get_with_expiry(self, key): """Get a value, and its expiry time, from the cache :param key: key to look up :returns The value from the cache, and the expiry time. :rtype: Tuple[Any, float] Raises: KeyError if the entry is not found """ self.expire() try: e = self._data[key] except KeyError: raise return e.value, e.expiry_time def pop(self, key, default=SENTINEL): """Remove a value from the cache If key is in the cache, remove it and return its value, else return default. If default is not given and key is not in the cache, a KeyError is raised. :param key: key to look up :param default: default value to return, if key is not found. If not set, and the key is not found, a KeyError will be raised :returns a value from the cache, or the default """ self.expire() e = self._data.pop(key, SENTINEL) if e == SENTINEL: if default == SENTINEL: raise KeyError(key) return default self._expiry_list.remove(e) return e.value def __getitem__(self, key): return self.get(key) def __delitem__(self, key): self.pop(key) def __contains__(self, key): return key in self._data def __len__(self): self.expire() return len(self._data) def expire(self): """Run the expiry on the cache. Any entries whose expiry times are due will be removed """ now = self._timer() while self._expiry_list: first_entry = self._expiry_list[0] if first_entry.expiry_time - now > 0.0: break del self._data[first_entry.key] del self._expiry_list[0]
# 315. Count of Smaller Numbers After Self # https://leetcode.com/problems/count-of-smaller-numbers-after-self from sortedcontainers import SortedList class Solution: def countSmaller(self, nums: List[int]) -> List[int]: n = len(nums) res = [0] * n sl = SortedList([nums[-1]]) for i in range(n - 2, -1, -1): index = sl.bisect(nums[i] - 1) res[i] = index sl.add(nums[i]) return res
def match_candidates(segment_kwargs, directory=None, match_all=False): for s in streams.segments: if not original_video(s, directory): continue if s.youtube and s.official and not match_all: continue if not s.official and segment_kwargs.get('official') is False: print(f'Skipping segment {s.hash} ' '(both videos are unofficial)', file=sys.stderr) continue subrefs = SortedList( [subref for ref in s.references for subref in ref.subrefs], key=lambda x: x.abs_start) if len(subrefs) == 0: print(f'Skipping segment {s.hash} (no subrefs)', file=sys.stderr) continue tmp_subref = None if subrefs[0].abs_start != 0: tmp_subref = SubReference(name='Начало', parent=s.references[0]) subrefs.add(tmp_subref) tmp_stream = Stream(data=[], key=s.stream.twitch) tmp_segment = Segment(stream=tmp_stream, **segment_kwargs) for subref in subrefs: tmp_segment.offset = subref.abs_start covered, partial, _ = refs_coverage(s.stream, tmp_segment) c_refs = SortedList([ subref for ref in flat([covered, partial]) for subref in ref.subrefs if subref._coverage >= 50 ], key=lambda x: x.abs_start) if len(c_refs) == 0: print( f'Skipping segment {s.hash} ' '(would not cover any subrefs)', file=sys.stderr) continue time_range = Timecode(c_refs[0].abs_start) time_range.end = c_refs[-1].abs_end scan_range = Timecode(time_range) - s.offset() # Expand scan range to cover all possible offsets overscan = tmp_segment.duration - time_range.duration if overscan > 0: scan_range.start -= overscan scan_range.end += overscan # Cut the end of the scan range that can't be matched anyway scan_range.end -= tmp_segment.duration - MATCH_OFFSET - MATCH_CHUNK # Keep range in stream bounds if scan_range.start < 0: scan_range.start = T + 0 if scan_range.end < 0: scan_range.end = T + MATCH_OFFSET + MATCH_CHUNK yield s, time_range, scan_range if tmp_subref: tmp_subref.parent.subrefs.remove(tmp_subref)
def longestRepeating(self, s: str, queryCharacters: str, queryIndices: List[int]) -> List[int]: from sortedcontainers import SortedList A = SortedList() B = SortedList() sz, last = 1, 0 ss = list((ord(x) - ord('a')) for x in s) for i in range(1, len(ss)): if ss[i] == ss[last]: sz += 1 else: A.add((last, last + sz - 1, ss[last])) B.add(sz) last = i sz = 1 A.add((last, last + sz - 1, ss[last])) B.add(sz) # very very error-prone. quite hard to write correct code. def update(p, c): # locate which range `p` is in. index = A.bisect((p,)) if index >= len(A): index -= 1 x, y, code = A[index] if x > p: index -= 1 x, y, code = A[index] size = (y - x + 1) assert code == ss[p] assert x <= p <= y # split that range to two segments. del A[index] B.remove(size) t0 = x, p - 1, ss[p] t1 = p + 1, y, ss[p] insert(t0) insert(t1) # generate new segment. t = p, p, c insert(t) def insert(t): x, y, code = t size = (y - x + 1) if size == 0: return index = A.bisect((x,)) index -= 1 # try to merge with neighbours. del_vec = [] # previous segment. if 0 <= index < len(A) and A[index][-1] == code: z0, z1, _ = A[index] # z0, z1, x, y if (z1 + 1) == x: x = z0 del_vec.append(A[index]) # next segment. if 0 <= (index + 1) < len(A) and A[index + 1][-1] == code: z0, z1, _ = A[index + 1] # x, y, z0, z1 if (y + 1) == z0: y = z1 del_vec.append(A[index + 1]) if del_vec: for t in del_vec: z0, z1, _t = t size = (z1 - z0 + 1) A.remove(t) B.remove(size) A.add((x, y, code)) size = (y - x + 1) B.add(size) ans = [] for c, i in zip(queryCharacters, queryIndices): c = ord(c) - ord('a') if c != ss[i]: update(i, c) ss[i] = c # print(A, B) sz = B[-1] ans.append(sz) return ans
class Kutana: """ Main class for kutana application :ivar ~.storage: Storage for things like states, e.t.c. :ivar ~.config: Application's configuration """ def __init__( self, concurrent_handlers_count=3000, storage=None, loop=None, ): self._plugins = [] self._backends = [] self._loop = loop or asyncio.new_event_loop() self._sem = asyncio.Semaphore(value=concurrent_handlers_count, loop=self._loop) self._routers = None self.storage = storage self.config = { "prefixes": (".", "/"), } def get_loop(self): """Return application's asyncio loop.""" return self._loop def add_plugin(self, plugin): """Add plugin to the application.""" if plugin in self._plugins: raise RuntimeError("Plugin already added") self._plugins.append(plugin) def add_plugins(self, plugins): """Add every plugin in passed iterable to the application.""" for plugin in plugins: self.add_plugin(plugin) def get_plugins(self): """Return list of added plugins.""" return self._plugins def add_backend(self, backend): """Add backend to the application.""" if backend in self._backends: raise RuntimeError("Backend already added") self._backends.append(backend) def get_backends(self): return self._backends async def _on_start(self, queue): for backend in self._backends: await backend.on_start(self) async def perform_updates_request(backend): def submit_update(update): return queue.put((update, backend)) while True: await backend.perform_updates_request(submit_update) await asyncio.sleep(0) asyncio.ensure_future(perform_updates_request(backend), loop=self._loop) for plugin in self._plugins: if plugin._on_start: await plugin._on_start(self) async def _main_loop(self): if self.storage is None: self.storage = NaiveMemory() queue = asyncio.Queue(maxsize=32, loop=self._loop) await self._on_start(queue) while True: await self._sem.acquire() update, backend = await queue.get() ctx = await Context.create( app=self, config=self.config, update=update, backend=backend, ) backend.prepare_context(ctx) task = asyncio.ensure_future(self._handle_update_with_logger( update, ctx), loop=self._loop) task.add_done_callback(lambda t: self._sem.release()) def _init_routers(self): self._routers = SortedList([], key=lambda r: -r.priority) def _add_router(new_router): for router in self._routers: if router.alike(new_router): router.merge(new_router) return self._routers.add(new_router) for plugin in self._plugins: for router in plugin._routers: _add_router(router) async def _handle_update_with_logger(self, update, ctx): logger.debug("Processing update %s", update) try: return await self._handle_update(update, ctx) except asyncio.CancelledError: pass except Exception as exc: logger.exception("Exception while handling the update") for plugin in self._plugins: if plugin._on_exception: await plugin._on_exception(update, ctx, exc) async def _handle_update(self, update, ctx): if self._routers is None: self._init_routers() for plugin in self._plugins: if plugin._on_before: await plugin._on_before(update, ctx) for router in self._routers: if await router.handle(update, ctx) != hr.SKIPPED: ctx._result = hr.COMPLETE break else: ctx._result = hr.SKIPPED for plugin in self._plugins: if plugin._on_after: await plugin._on_after(update, ctx, ctx._result) return ctx._result async def _shutdown(self): logger.info("Gracecfully shutting application down...") # Clean up tasks = [] for backend in self._backends: tasks.append(backend.on_shutdown(self)) for plugin in self._plugins: if plugin._on_shutdown: tasks.append(plugin._on_shutdown(self)) await asyncio.gather(*tasks, loop=self._loop, return_exceptions=True) # Cancel everything else tasks = [] for task in asyncio.Task.all_tasks(loop=self._loop): if task is not asyncio.Task.current_task(): task.cancel() tasks.append(task) await asyncio.gather(*tasks, loop=self._loop, return_exceptions=True) self._loop.stop() def run(self): """Start the application.""" logger.info("Starting application...") try: asyncio.ensure_future(self._main_loop(), loop=self._loop) self._loop.run_forever() except KeyboardInterrupt: asyncio.ensure_future(self._shutdown(), loop=self._loop) self._loop.run_forever() finally: self._loop.close() logger.info("Stopped application") def stop(self): """Stop the application.""" return asyncio.ensure_future(self._shutdown(), loop=self._loop)
def create_group_diagram_water_land(entities, entities_times, all_time_stamps, d, time_shift, first, last, birds, save_path): gmap = gmplot.GoogleMapPlotter(47, 9, 6) over_water = 0 over_land = 0 over_water_rep = 0 over_land_rep = 0 all_together_water = 0 all_together_land = 0 # start and end with same time start_times = SortedList() end_times = SortedList() for entityTime in entities_times: start_times.add(entityTime[0]) end_times.add(entityTime[-1]) common_start_time = start_times[-1] common_end_time = end_times[0] all_common_time_stamps = SortedList(all_time_stamps[all_time_stamps.index( common_start_time):all_time_stamps.index(common_end_time)]) # computing all events event_count = 0 event_times = SortedList() for time_index, timestamp in enumerate(all_common_time_stamps[:-1]): coordinates = [list() for i in xrange(len(entities))] for index, entity in enumerate(entities): start_segment = entities_times[index].index(timestamp) end_segment = start_segment + 1 coordinates[index] = [ entities[index][start_segment][1], entities[index][start_segment][2], entities[index][end_segment][1], entities[index][end_segment][2] ] for index1 in range( 0, len(coordinates)): # , coordinates1 in enumerate(coordinates): for index2 in range(0, len(coordinates)): new_events = compare_two_segments( coordinates[index1][0], coordinates[index1][2], coordinates[index2][0], coordinates[index2][2], coordinates[index1][1], coordinates[index1][3], coordinates[index2][1], coordinates[index2][3], time_index + first, all_common_time_stamps, d) if len(new_events) > 0: for event in new_events: event_times.add(event) event_count += 1 print event_count # propagate event time stamps to all trajectories adding_count = 0 for timestamp in event_times: if timestamp not in all_common_time_stamps: all_common_time_stamps.add(timestamp) adding_count += 1 for index, entity in enumerate(entities): length = len(entities_times[index]) # check if the current entity has this timestamp, if not add it and interpolate the # coordinates but only if it is in the range of the trajectory. Otherwise interpolation is not possible if timestamp not in entities_times[index] and entities_times[ index].bisect(timestamp) > 0 and entities_times[ index].bisect(timestamp) < length: entity.add( interpolate_surface(entity, entities_times[index], timestamp)) entities_times[index].add(timestamp) # solve set cover instance for every time stamp, return the segments of the solution and draw them universe = set(range(0, len(entities))) cover_size = [] cover_numbers = [] color_of_entities = ['red', 'blue', 'green', 'orange', 'black'] # for each entity initialize average_representation_strength = [0 for i in xrange(len(entities))] all_together = 0 number_of_representations = 0 for time_index, timestamp in enumerate(all_common_time_stamps[:-200]): set_list = [] start_segment_common = all_common_time_stamps.index(timestamp) end_segment_common = start_segment_common + 1 for index in range(0, len(entities)): current_set = set() start_segment = entities_times[index].index(timestamp) end_segment = start_segment + 1 segment = [[ entities[index][start_segment][1], entities[index][start_segment][2] ], [ entities[index][end_segment][1], entities[index][end_segment][2] ]] for number, entity in enumerate(entities): if within_alpha_similar_time_distance( segment, timestamp, all_common_time_stamps[end_segment_common], entity, entities_times[number], time_shift, d, 0): current_set.add(number) set_list.append(current_set) average_representation_strength[index] += len(current_set) result = set_cover(universe, set_list) number_of_representations += len(result[0]) if len(result[0]) == 1: all_together += 1 cover_numbers.append(result[1]) landmarks_start = [ entities[i][entities_times[i].index(timestamp)][3] for i in range(0, len(entities)) ] landmarks_end = [ entities[i][entities_times[i].index(timestamp)][3] for i in range(0, len(entities)) ] if 210 in landmarks_start or 210 in landmarks_end: water = True else: water = False cover_size.append([(len(result[0])), water]) # get if water: over_water += 1 over_water_rep += len(result[0]) if len(result[0]) == 1: all_together_water += 1 else: over_land += 1 over_land_rep += len(result[0]) if len(result[0]) == 1: all_together_land += 1 for indexSet, entity in enumerate(result[1]): # print entities[3][entities_times[3].index(timestamp)][2], entities[3][entities_times[3].index(timestamp)][1] # print entities[3][entities_times[3].index(timestamp)][3] # print color # print "\n" color = color_of_entities[entity] gmap.plot([ entities[entity][entities_times[entity].index(timestamp)][2], entities[entity][entities_times[entity].index(timestamp) + 1][2] ], [ entities[entity][entities_times[entity].index(timestamp)][1], entities[entity][entities_times[entity].index(timestamp) + 1][1] ], color=color, edge_width=result[2][indexSet]) all_together = 1.0 * all_together / (len(all_common_time_stamps) - 200 ) # ((last + adding_count) - first) number_of_representations = 1.0 * number_of_representations / ( len(all_common_time_stamps) - 200) # ((last + adding_count) - first) print "Together Test:", d, time_shift, all_together print "Average Number Representatives:", d, time_shift, number_of_representations # average_representation_strength = [1.0 * entry / ((last + len(event_times)) - first) - 1 for entry in # average_representation_strength] average_representation_strength = [ 1.0 * entry / (len(all_common_time_stamps) - 200) - 1 for entry in average_representation_strength ] size = os.path.join( save_path, "outputSize_" + str(d) + '_' + str(time_shift) + ".csv") numbers = os.path.join( save_path, "outputNumber_" + birds + str(d) + '_' + str(time_shift) + ".txt") representation = os.path.join( save_path, "averageRepresentationStrength" + birds + str(d) + '_' + str(time_shift) + ".txt") html = os.path.join( save_path, 'groupDiagram_water_land' + birds + str(d) + '_' + str(time_shift) + '.html') # file1 = open(size, "w") # simplejson.dump(cover_size, file1) # file1.close() with open(size, "wb") as f: writer = csv.writer(f) writer.writerows(cover_size) file2 = open(numbers, 'w') simplejson.dump(cover_numbers, file2) file2.close() file3 = open(representation, 'w') simplejson.dump(average_representation_strength, file3) file3.close() gmap.draw(html) print "average representatives over land:", 1.0 * over_land_rep / over_land print "average representatives over water:", 1.0 * over_water_rep / over_water print "time together over land:", 1.0 * all_together_land / over_land print "time together over water:", 1.0 * all_together_water / over_water return { d: [ all_together, number_of_representations, 1.0 * all_together_land / over_land, 1.0 * over_land_rep / over_land, 1.0 * all_together_water / over_water, 1.0 * over_water_rep / over_water ] }