class StockPrice(object): def __init__(self): self.timeToPrice = SortedDict() #time will be sorted self.priceToTime = SortedDict() #the price will be sorted def update(self, timestamp, price): if timestamp in self.timeToPrice: prevPrice = self.timeToPrice[timestamp] self.priceToTime[prevPrice].remove(timestamp) if len(self.priceToTime[prevPrice])==0: self.priceToTime.pop(prevPrice) #[0] if price not in self.priceToTime: self.priceToTime[price] = set() #initialized self.priceToTime[price].add(timestamp) self.timeToPrice[timestamp] = price def current(self): return self.timeToPrice.peekitem(-1)[1] def maximum(self): return self.priceToTime.peekitem(-1)[0] def minimum(self): return self.priceToTime.peekitem(0)[0]
class StockPrice: # 45%, 12% def __init__(self): # 两个TreeMap self.timeToPrice = SortedDict() # 时间->价格,key已排序 self.priceToTimes = SortedDict() # 价格->Set<该价格的时间>,key已排序 def update(self, timestamp: int, price: int) -> None: if timestamp in self.timeToPrice: # 时间已存在,是更新价格 oldPrice = self.timeToPrice[timestamp] self.priceToTimes[oldPrice].remove(timestamp) # 去掉老的价格中的时间,因为已失效 if not self.priceToTimes[oldPrice]: # 如果老价格不再对应任何时间,去除key self.priceToTimes.pop(oldPrice) self.timeToPrice[timestamp] = price # 更新/新建 时间->价格 if not price in self.priceToTimes: # 更新/新建 价格->时间们 self.priceToTimes[price] = set() self.priceToTimes[price].add(timestamp) def current(self) -> int: return self.timeToPrice.peekitem(-1)[1] # 找最大的时间key,得到其对应价格 def maximum(self) -> int: return self.priceToTimes.peekitem(-1)[0] # 找最大的价格key,得到key def minimum(self) -> int: return self.priceToTimes.peekitem(0)[0] # 找最小的价格key,得到key
def oddEvenJumps(self, arr: List[int]) -> int: N = len(arr) odd = [False] * N even = [False] * N odd[-1] = True even[-1] = True sd = SortedDict() sd[arr[N - 1]] = N - 1 for i in range(N - 2, -1, -1): if arr[i] in sd: odd[i] = even[sd[arr[i]]] even[i] = odd[sd[arr[i]]] else: # greatest smaller floor_idx = sd.bisect_left(arr[i]) - 1 if floor_idx != -1: even[i] = odd[sd.peekitem(floor_idx)[1]] # smallest greater ceiling_idx = sd.bisect_left(arr[i]) if ceiling_idx != len(sd): odd[i] = even[sd.peekitem(ceiling_idx)[1]] sd[arr[i]] = i return odd.count(True)
def longestSubarray(self, nums: List[int], limit: int) -> int: n = len(nums) w = SortedDict() start = 0 end = 0 def add(value): if value not in w: w[value] = 0 w[value] += 1 def delete(value): if w[value] == 1: del w[value] else: w[value] -= 1 while end < n: add(nums[end]) largest = w.peekitem(index=-1) smallest = w.peekitem(index=0) if largest[0] - smallest[0] > limit: delete(nums[start]) start += 1 end += 1 return end - start
class StockPrice0: ## not using sorteddict for time_to_prices def __init__(self): self.time_to_prices = {} self.rec = SortedDict() self.cur_time = -1 def update(self, timestamp: int, price: int) -> None: self.cur_time = max(self.cur_time, timestamp) if timestamp in self.time_to_prices: prev_price = self.time_to_prices[timestamp] self.rec[prev_price].remove(timestamp) if len(self.rec[prev_price]) == 0: self.rec.pop(prev_price) if not price in self.rec: self.rec[price] = set() self.rec[price].add(timestamp) self.time_to_prices[timestamp] = price def current(self) -> int: # return self.time_to_prices.peekitem(-1)[1] return self.time_to_prices[self.cur_time] def maximum(self) -> int: return self.rec.peekitem(-1)[0] def minimum(self) -> int: return self.rec.peekitem(0)[0]
def oddEvenJumps(self, nums) -> int: if len(nums) == 0: return 0 if len(nums) == 1: return 1 n = len(nums) isOkEven = [False] * len(nums) isOkOdd = [False] * len(nums) isOkEven[n - 1] = True isOkOdd[n - 1] = True tree = SortedDict() tree[nums[n - 1]] = n - 1 for i in range(n - 2, -1, -1): val = nums[i] if val in tree: isOkEven[i] = isOkOdd[tree[val]] isOkOdd[i] = isOkEven[tree[val]] else: smallestP = tree.bisect_left(val) largestP = tree.bisect_left(val) - 1 isOkOdd[i] = True if smallestP != len(tree) and isOkEven[ tree.peekitem(smallestP)[1]] else False isOkEven[i] = True if largestP != -1 and isOkOdd[tree.peekitem( largestP)[1]] else False tree[val] = i res = 0 for e in isOkOdd: if e: res += 1 return res
class StockPrice: def __init__(self): self.tpdict = SortedDict() self.ptdict = SortedDict() def update(self, timestamp: int, price: int) -> None: if timestamp not in self.tpdict: self.tpdict[timestamp] = price if price not in self.ptdict: self.ptdict[price] = set() self.ptdict[price].add(timestamp) else: prevp = self.tpdict[timestamp] self.tpdict[timestamp] = price self.ptdict[prevp].remove(timestamp) if not self.ptdict[prevp]: del self.ptdict[prevp] if price not in self.ptdict: self.ptdict[price] = set() self.ptdict[price].add(timestamp) def current(self) -> int: return self.tpdict.peekitem(-1)[1] def maximum(self) -> int: return self.ptdict.peekitem(-1)[0] def minimum(self) -> int: return self.ptdict.peekitem(0)[0]
class MaxStack: def __init__(self): self.map = SortedDict() self.head = Node(-1) self.tail = Node(-1) self.head.next, self.tail.prev = self.tail, self.head def push(self, x: int) -> None: node = Node(x) self.insert(node) def pop(self) -> int: node = self.tail.prev self.remove(node) return node.val def top(self) -> int: return self.tail.prev.val def peekMax(self) -> int: return self.map.peekitem()[0] def popMax(self) -> int: key, nodes = self.map.peekitem() self.remove(nodes[-1]) return key def insert(self, node) -> None: prev, nxt = self.tail.prev, self.tail node.prev, node.next = prev, nxt prev.next, nxt.prev = node, node if node.val in self.map: self.map[node.val].append(node) else: self.map[node.val] = [node] def remove(self, node) -> None: prev, nxt = node.prev, node.next prev.next, nxt.prev = nxt, prev for i in range(len(self.map[node.val]) - 1, -1, -1): if self.map[node.val][i] == node: del self.map[node.val][i] if not self.map[node.val] or not len(self.map[node.val]): del self.map[node.val] return # Your MaxStack object will be instantiated and called as such: # obj = MaxStack() # obj.push(x) # param_2 = obj.pop() # param_3 = obj.top() # param_4 = obj.peekMax() # param_5 = obj.popMax()
def top_view(self): sorted_endpoints = [] for segment_ in self.segments: sorted_endpoints.append(Endpoint(True, segment_)) sorted_endpoints.append(Endpoint(False, segment_)) sorted_endpoints.sort( key=lambda end_point: end_point.value()) # O n(log n) resulted_segments = [] # sweeper = SortedDict(key=lambda ep: ep.height) sweeper = SortedDict() print(sweeper) for endpoint_ in sorted_endpoints: if sweeper.__len__() == 0: sweeper[endpoint_] = endpoint_.segment elif not endpoint_.is_left: # right of some segment left_ep = Endpoint(True, endpoint_.segment) if left_ep.__eq__( sweeper.peekitem(0)[0]): # "top most" is ending now sweeper.pop(sweeper.peekitem(0)[0]) prev_seg = resulted_segments[resulted_segments.__len__() - 1] resulted_segments.append( Segment(prev_seg.right, endpoint_.segment.right, endpoint_.segment.height, endpoint_.segment.name)) else: # some segment "not at top" needs to end now sweeper.pop(left_ep) elif sweeper.peekitem( 0 )[0].segment.height < endpoint_.segment.height: # crossover, a higher segment found current_top = sweeper.peekitem(0)[0] if resulted_segments.__len__() > 0: last_segment = resulted_segments[ resulted_segments.__len__() - 1] resulted_segments.append( Segment(last_segment.right, endpoint_.segment.left, current_top.segment.height, current_top.segment.name)) else: resulted_segments.append( Segment(current_top.segment.left, endpoint_.segment.left, current_top.segment.height, current_top.segment.name)) sweeper[endpoint_] = endpoint_.segment elif sweeper.peekitem( 0 )[0].segment.height > endpoint_.segment.height: # non contributing end found sweeper[endpoint_] = endpoint_.segment return resulted_segments
class SortedSlidingDic: def __init__(self, stime): self.flow_dic = {} self.ts_dic = SortedDict() self.stime = float(self.unified_ts(stime)) def unified_ts(self, ts): return round(ts, 10) def update(self, ts): ts = self.unified_ts(ts) while len(self.ts_dic) > 0 and ts - self.ts_dic.peekitem(0)[0] > self.stime: del self.flow_dic[self.ts_dic.peekitem(0)[1]] self.ts_dic.popitem(0) def add(self, flow, ts): ts = self.unified_ts(ts) # Remove the previous timestamp for this flow and the new one instead self.remove(flow) self.flow_dic[flow] = ts if ts in self.ts_dic: del self.flow_dic[self.ts_dic[ts]] self.ts_dic[ts] = flow assert len(self.flow_dic) == len(self.ts_dic) def remove(self, flow): if flow in self.flow_dic: ts = self.flow_dic[flow] try: del self.flow_dic[flow] self.ts_dic.pop(ts) except KeyError: print 'KeyError ', flow, ts def __str__(self): res = '' for k in self.ts_dic: res += str("%.10f" % k)+'\t'+str(self.ts_dic[k])+'\n' print res
class BisectNodePolicy(BaseNodePolicy): def __init__(self, hash_class=defaultHashClass): self.ring = SortedDict() super(BisectNodePolicy, self).__init__(hash_class=hash_class) def add_node(self, node=None, vnode_count=None): for i in range_(int(vnode_count)): self.ring[self._gen_key(node, i)] = node def remove_node(self, node=None): keys = list(self.ring.keys()) for key in keys: if self.ring[key] == node: self.ring.pop(key) def get_proper_node(self, key): key, _ = self.ring.peekitem(self._find_proper_pos(key)) return self.ring[key] def _find_proper_pos(self, key): key = self._gen_key(key) pos = self.ring.bisect(key) # if object_hash == node_hash, return node index if key in self.ring: return pos - 1 # embodies the concept of the ring. if pos == len(self.ring): return 0 return pos
def canAttendMeetings(self, intervals: List[List[int]]) -> bool: points = SortedDict() for start, end in intervals: # print(start, end) # print(points) i_start = points.bisect_right(start) i_end = points.bisect_left(end) # print("i_start", i_start) # print("i_end", i_end) if i_end != i_start: return False if i_start > 0 and points.peekitem(i_start-1)[1] == 1: return False if points.get(start) == -1: del points[start] else: points[start] = 1 if points.get(end) == 1: del points[end] else: points[end] = -1 return True
def getSkyline(self, buildings: List[List[int]]) -> List[List[int]]: def update_heights_count(typ, sd, height): if typ == 1: if height not in sd: sd[height] = 1 else: sd[height] += 1 else: if sd[height] == 1: del sd[height] else: sd[height] -= 1 res = [] d = defaultdict(list) prev_h = 0 sd = SortedDict() for l, r, h in buildings: d[l].append((h, 1)) d[r].append((h, -1)) for x in sorted(d.keys()): for height, typ in d[x]: update_heights_count(typ, sd, height) curr_h = sd.peekitem(-1)[0] if sd else 0 if curr_h != prev_h: res.append([x, curr_h]) prev_h = curr_h return res
def find_closest_elements_in_sorted_arrays(sorted_arrays): diff = float('inf') tree = SortedDict() for idx, array in enumerate(sorted_arrays): i = iter(array) nxt = next(i) tree[(nxt, idx)] = i while True: min_val, min_idx = tree.peekitem(0)[0] max_val = tree.peekitem()[0][0] diff = min(diff, max_val - min_val) i = tree.popitem(0)[1] nxt = next(i, None) if nxt is None: return diff tree[(nxt, min_idx)] = i
def find_closest_elements_in_sorted_arrays(sorted_arrays): # TODO - you fill in here. min_diff = float('inf') chosen_set = [] iters = SortedDict() for idx, s in enumerate(sorted_arrays): it = iter(s) first_min = next(it, None) assert first_min is not None, pdb.set_trace() if first_min is not None: iters[first_min] = (idx, it) while True: max_val = iters.keys()[-1] min_val, (min_idx, it) = iters.peekitem(0) if min_diff > max_val - min_val: min_diff = max_val - min_val chosen_set = sorted(list((v, i) for v, (i, _) in iters.items()), key=lambda x: x[1]) iters.popitem(0) next_min = next(it, None) if next_min == None: print(chosen_set) return min_diff iters[next_min] = (min_idx, it) assert len(iters) == len(sorted_arrays), pdb.set_trace()
class ClientsCreditsInfo: def __init__(self): self.offset = 0 self.clients = {} self.credits = SortedDict() def insert(self, client_id, c): self.remove(client_id) self.clients[client_id] = c - self.offset self.credits[c - self.offset] = client_id def remove(self, client_id): if client_id not in self.clients: return False credit = self.clients[client_id] if not self.credits[credit]: del self.credits[credit] del self.clients[client_id] return True def lookup(self, client_id): if client_id not in self.clients: return -1 credit = self.clients[client_id] return credit + self.offset def add_all(self, C): self.offset += C def max(self): if not self.credits: return '' return self.credits.peekitem(-1)[1]
class Book(object): def __init__(self): self._bid = SortedDict() self._ask = SortedDict() self._info = {} def random_init(self): for px, count in zip( np.arange(1, 11), np.histogram(np.clip( np.random.poisson(4, 1000), 1, 10, ), bins=10)[0]): self._ask[px] = Level("ASK", px, [Order() for _ in range(count)]) for px, count in zip( np.arange(0, -10, -1), np.histogram(np.clip( np.random.poisson(4, 1000), 1, 10, ), bins=10)[0]): self._bid[px] = Level("BID", px, [Order() for _ in range(count)]) return self # def is_crossed(self) -> bool: # if def imshow(self): fig, ax = plt.subplots(figsize=(16, 10)) ax.barh(self._ask.keys(), [level.qty for level in self._ask.values()], color='red') ax.barh(self._bid.keys(), [-level.qty for level in self._bid.values()], color='blue') ax.grid() ax.set_yticks( range(self._bid.peekitem(0)[0], self._ask.peekitem(-1)[0] + 1)) plt.show() def __repr__(self): ask_str = '\n'.join(f'\t{level}' for level in self._ask.values()) bid_str = '\n'.join(f'\t{level}' for level in self._bid.values()) return f'Book {self._info}' + ' {\n' + (ask_str + bid_str).replace( '\n', '\n\t') + '\n}'
class TimerScheduler: def __init__(self): self._epoch = datetime.now() self._timers_by_expire_time = SortedDict() def now(self): absolute_now = datetime.now() time_since_epoch = absolute_now - self._epoch return time_since_epoch.total_seconds() def schedule(self, timer): expire_time = timer.expire_time() assert expire_time is not None if expire_time in self._timers_by_expire_time: self._timers_by_expire_time[expire_time].append(timer) else: self._timers_by_expire_time[expire_time] = [timer] def unschedule(self, timer): expire_time = timer.expire_time() assert expire_time is not None assert expire_time in self._timers_by_expire_time timers_with_matching_expire = self._timers_by_expire_time[expire_time] assert timer in timers_with_matching_expire timers_with_matching_expire.remove(timer) if timers_with_matching_expire == []: self._timers_by_expire_time.pop(expire_time) def trigger_all_expired_timers(self): # Trigger all expired timers and return time until next expire now = self.now() while True: if not self._timers_by_expire_time: return None next_expire_time = self._timers_by_expire_time.peekitem(0)[0] if next_expire_time > now: return next_expire_time - now expired_timers = self._timers_by_expire_time.popitem(0)[1] for timer in expired_timers: timer.trigger_expire() def stop_all_timers(self): while self._timers_by_expire_time: timers = self._timers_by_expire_time.peekitem(0)[1] for timer in timers: timer.stop()
def find_closest_elements(sorted_arrays): min_distance_so_far = float('inf') iters = SortedDict() for idx, sorted_array in enumerate(sorted_arrays): it = iter(sorted_array) first_min = next(it, None) if first_min is not None: iters.setdefault((first_min, idx), default=it) while True: min_value, min_idx = iters.peekitem(index=0)[0] max_value = iters.peekitem()[0][0] min_distance_so_far = min(max_value - min_value, min_distance_so_far) it = iters.popitem(index=0)[1] next_min = next(it, None) if next_min is None: return min_distance_so_far iters.setdefault((next_min, min_idx), it)
class MenuBar(QMenuBar): def __init__(self, menu_order: Tuple[MainMenu] = (FileMenu, ViewMenu, ToolsMenu, AlgorithmsMenu, WindowsMenu, HelpMenu)): super().__init__() self._menu_order = menu_order self._menus_order_indexes = { menu_type: i for (i, menu_type) in enumerate(self._menu_order) } self._ordered_added_menus = SortedDict( ) # {order_index: MainMenu class} def add_menu(self, menu_type: Type[MainMenu]) -> MainMenu: menu = menu_type() menu_order_index = self._menu_order_index(menu_type) self._ordered_added_menus[menu_order_index] = menu menu_index_in_ordered_added_menus = self._ordered_added_menus.index( menu_order_index) # If the menu is the last one if menu_index_in_ordered_added_menus == len( self._ordered_added_menus) - 1: self.addMenu(menu) else: next_menu_index_in_ordered_added_menus = menu_index_in_ordered_added_menus + 1 next_menu = self._ordered_added_menus.peekitem( next_menu_index_in_ordered_added_menus)[1] self.insertMenu(next_menu.menuAction(), menu) return menu def menu(self, menu_type: Type[MainMenu], add_nonexistent: bool = True) -> Optional[MainMenu]: menu = self._ordered_added_menus.get(self._menu_order_index(menu_type)) if menu is None and add_nonexistent: menu = self.add_menu(menu_type) return menu def add_menu_action(self, menu_type: Type[MainMenu], action_name, method, shortcut=None) -> QAction: return self.menu(menu_type).addAction(action_name, method, shortcut) def _menu_order_index(self, menu_type: Type[MainMenu]) -> int: return self._menus_order_indexes[menu_type]
class QtDictListModel(QAbstractListModel): def __init__(self): QAbstractListModel.__init__(self) self._items = SortedDict() def role(self, item, role): return item def rowCount(self, parent): if parent.isValid(): return 0 return len(self._items) def from_index(self, index): if not index.isValid() or index.row() >= len(self._items): return None return self._items.peekitem(index.row())[1] def data(self, index, role): item = self.from_index(index) if item is None: return None return self.role(item, role) def _add(self, key, item): assert key not in self._items next_index = self._items.bisect_left(key) self.beginInsertRows(QModelIndex(), next_index, next_index) self._items[key] = item self.endInsertRows() # TODO - removal is O(n). def _remove(self, key): assert key in self._items item_index = self._items.index(key) self.beginRemoveRows(QModelIndex(), item_index, item_index) del self._items[key] self.endRemoveRows() def _clear(self): self.beginRemoveRows(QModelIndex(), 0, len(self._items) - 1) self._items.clear() self.endRemoveRows() # O(n). Rework if it's too slow. def _update(self, key, roles=None): item_index = self._items.index(key) index = self.index(item_index, 0) if roles is None: self.dataChanged.emit(index, index) else: self.dataChanged.emit(index, index, roles)
class StockPrice: def __init__(self): self.timestampToPrice = SortedDict() self.pricesCount = SortedDict() def update(self, timestamp: int, price: int) -> None: if timestamp in self.timestampToPrice: prevPrice = self.timestampToPrice[timestamp] self.pricesCount[prevPrice] -= 1 if self.pricesCount[prevPrice] == 0: del self.pricesCount[prevPrice] self.timestampToPrice[timestamp] = price self.pricesCount[price] = self.pricesCount.get(price, 0) + 1 def current(self) -> int: return self.timestampToPrice.peekitem(-1)[1] def maximum(self) -> int: return self.pricesCount.peekitem(-1)[0] def minimum(self) -> int: return self.pricesCount.peekitem(0)[0]
def lastStoneWeight(self, stones): stones_sorted_dict = SortedDict(Counter(stones)) # print(stones_sorted_dict) while len(stones_sorted_dict) != 1: last = None while len(stones_sorted_dict) > 0: q = stones_sorted_dict.popitem() if q[1] % 2: last = q[0] if last: break if len(stones_sorted_dict) == 0: if last is None: return 0 else: return last second_last = stones_sorted_dict.peekitem()[0] stones_sorted_dict[second_last] -= 1 if stones_sorted_dict[second_last] == 0: stones_sorted_dict.popitem() q = last - second_last if q in stones_sorted_dict: stones_sorted_dict[q] += 1 else: stones_sorted_dict[q] = 1 # print(stones_sorted_dict) if len(stones_sorted_dict) == 1: if stones_sorted_dict.peekitem()[1] % 2: return stones_sorted_dict.peekitem()[0] else: return 0
def extract_key_from_dict_of_dicts(dict_of_dicts, key): ret = SortedDict() for dk in dict_of_dicts: ret[dk] = SortedDict() c_dict = dict_of_dicts[dk] c_ret = ret[dk] if key in c_dict: c_ret[key] = c_dict[key] if len(ret.values()) == 1: # only one block, then just return this one as a SortedDict and not as a SortedDict of SortedDict's return ret.peekitem(0)[1].peekitem(0)[1] else: return ret
class BaseSimulator: def __init__(self): self.currentTime = 0.0 self.eventList = SortedDict() def addEvent(self, time, funcName, args): self.eventList[time] = [funcName, args] def deleteEvents(self, timeList): try: for key in timeList: self.eventList.pop(key) except KeyError: return def completeRun(self): try: while self.eventList.peekitem(0): self.currentTime, funcCall = self.eventList.peekitem(0) funcCall[0](funcCall[1]) self.eventList.popitem(0) except IndexError: return def run(self, timeLimit = 0): try: while self.eventList.peekitem(0): currentTime, funcCall = self.eventList.peekitem(0) if 'timeLimit' in locals(): if currentTime >= timeLimit: return funcCall[0](funcCall[1]) self.currentTime = currentTime self.eventList.popitem(0) except IndexError: return
def getSkyline(self, buildings: List[List[int]]) -> List[List[int]]: pts = [None] * (len(buildings) * 2) for i, v in enumerate(buildings): l, r, h = v pts[i] = (l, h, 's') pts[len(pts) - i - 1] = (r, h, 'e') pts.sort(key=lambda x: (x[0], -x[1] if x[2] == 's' else x[1])) sol = [] pq = SortedDict({0: 1}) for x, y, t in pts: prev, _ = pq.peekitem(index=-1) if t == 's': pq[y] = pq.get(y, 0) + 1 cur, _ = pq.peekitem(index=-1) if cur > prev: sol.append([x, y]) else: pq[y] = pq[y] - 1 if pq[y] == 0: del pq[y] cur, _ = pq.peekitem(index=-1) if cur < prev: sol.append([x, cur]) return sol
def solve(): n, k = map(int, raw_input().split()) cnts = SortedDict([(n, 1)]) i = 0 while i < k: sz, cnt = cnts.peekitem() ls = (sz-1) / 2 rs = sz / 2 if ls: cnts[ls] = cnts.get(ls, 0) + cnt if rs: cnts[rs] = cnts.get(rs, 0) + cnt del cnts[sz] i += cnt return "%d %d" % (rs, ls)
class StockPrice: def __init__(self): self.timeToPrices = SortedDict() self.rec = SortedDict() def update(self, timestamp: int, price: int) -> None: if timestamp in self.timeToPrices: prevPrice = self.timeToPrices[timestamp] self.rec[prevPrice].remove(timestamp) if len(self.rec[prevPrice]) == 0: self.rec.pop(prevPrice) if not price in self.rec: self.rec[price] = set() self.rec[price].add(timestamp) self.timeToPrices[timestamp] = price def current(self) -> int: return self.timeToPrices.peekitem(-1)[1] def maximum(self) -> int: return self.rec.peekitem(-1)[0] def minimum(self) -> int: return self.rec.peekitem(0)[0]
def longestSubarray(self, nums: List[int], limit: int) -> int: sd = SortedDict() l, r, res = 0, 0, 0 while r < len(nums): new_val = nums[r] r += 1 if new_val not in sd: sd[new_val] = 1 else: sd[new_val] += 1 while abs(sd.peekitem(0)[0] - sd.peekitem(-1)[0]) > limit: rm_val = nums[l] l += 1 sd[rm_val] -= 1 if sd[rm_val] == 0: del sd[rm_val] res = max(res, r - l) return res
class Events: def __init__(self): self.list_of_events = SortedDict() # for each time, a list of events def add_event(self, event): if not self.list_of_events.get(event.time): self.list_of_events[event.time] = [event] else: self.list_of_events[event.time].append(event) def count_arrivals_at(self, time): try: li = self.list_of_events[time] n = 0 for e in li: if li.type == 'arrival': n += 1 return n except: return 0 def pop_next_event(self): li = self.list_of_events.peekitem(0)[1] #if debug: # print(self.list_of_events) if (len(li) > 1): toret = li.pop(0) #if debug: # print(f'returning {toret}') return toret else: lil = self.list_of_events.popitem(0)[1] toret = lil[0] #if debug: # print(f'returning {toret}') return toret def count(self): return len(self.list_of_events) def __str__(self): s = '' for k in self.list_of_events.keys(): s += (str(k) + ": ") for e in self.list_of_events[k]: s += str(e) + ', ' return s
def test_peekitem2(): mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)] temp = SortedDict(mapping) temp.peekitem(index=100)
class OrderBook(WebsocketClient): def __init__(self, product_id='BTC-USD', log_to=None): super(OrderBook, self).__init__(products=product_id) self._asks = SortedDict() self._bids = SortedDict() self._client = PublicClient() self._sequence = -1 self._log_to = log_to if self._log_to: assert hasattr(self._log_to, 'write') self._current_ticker = None @property def product_id(self): ''' Currently OrderBook only supports a single product even though it is stored as a list of products. ''' return self.products[0] def on_open(self): self._sequence = -1 print("-- Subscribed to OrderBook! --\n") def on_close(self): print("\n-- OrderBook Socket Closed! --") def reset_book(self): self._asks = SortedDict() self._bids = SortedDict() res = self._client.get_product_order_book(product_id=self.product_id, level=3) for bid in res['bids']: self.add({ 'id': bid[2], 'side': 'buy', 'price': Decimal(bid[0]), 'size': Decimal(bid[1]) }) for ask in res['asks']: self.add({ 'id': ask[2], 'side': 'sell', 'price': Decimal(ask[0]), 'size': Decimal(ask[1]) }) self._sequence = res['sequence'] def on_message(self, message): if self._log_to: pickle.dump(message, self._log_to) sequence = message.get('sequence', -1) if self._sequence == -1: self.reset_book() return if sequence <= self._sequence: # ignore older messages (e.g. before order book initialization from getProductOrderBook) return elif sequence > self._sequence + 1: self.on_sequence_gap(self._sequence, sequence) return msg_type = message['type'] if msg_type == 'open': self.add(message) elif msg_type == 'done' and 'price' in message: self.remove(message) elif msg_type == 'match': self.match(message) self._current_ticker = message elif msg_type == 'change': self.change(message) self._sequence = sequence def on_sequence_gap(self, gap_start, gap_end): self.reset_book() print('Error: messages missing ({} - {}). Re-initializing book at sequence.'.format( gap_start, gap_end, self._sequence)) def add(self, order): order = { 'id': order.get('order_id') or order['id'], 'side': order['side'], 'price': Decimal(order['price']), 'size': Decimal(order.get('size') or order['remaining_size']) } if order['side'] == 'buy': bids = self.get_bids(order['price']) if bids is None: bids = [order] else: bids.append(order) self.set_bids(order['price'], bids) else: asks = self.get_asks(order['price']) if asks is None: asks = [order] else: asks.append(order) self.set_asks(order['price'], asks) def remove(self, order): price = Decimal(order['price']) if order['side'] == 'buy': bids = self.get_bids(price) if bids is not None: bids = [o for o in bids if o['id'] != order['order_id']] if len(bids) > 0: self.set_bids(price, bids) else: self.remove_bids(price) else: asks = self.get_asks(price) if asks is not None: asks = [o for o in asks if o['id'] != order['order_id']] if len(asks) > 0: self.set_asks(price, asks) else: self.remove_asks(price) def match(self, order): size = Decimal(order['size']) price = Decimal(order['price']) if order['side'] == 'buy': bids = self.get_bids(price) if not bids: return assert bids[0]['id'] == order['maker_order_id'] if bids[0]['size'] == size: self.set_bids(price, bids[1:]) else: bids[0]['size'] -= size self.set_bids(price, bids) else: asks = self.get_asks(price) if not asks: return assert asks[0]['id'] == order['maker_order_id'] if asks[0]['size'] == size: self.set_asks(price, asks[1:]) else: asks[0]['size'] -= size self.set_asks(price, asks) def change(self, order): try: new_size = Decimal(order['new_size']) except KeyError: return try: price = Decimal(order['price']) except KeyError: return if order['side'] == 'buy': bids = self.get_bids(price) if bids is None or not any(o['id'] == order['order_id'] for o in bids): return index = [b['id'] for b in bids].index(order['order_id']) bids[index]['size'] = new_size self.set_bids(price, bids) else: asks = self.get_asks(price) if asks is None or not any(o['id'] == order['order_id'] for o in asks): return index = [a['id'] for a in asks].index(order['order_id']) asks[index]['size'] = new_size self.set_asks(price, asks) tree = self._asks if order['side'] == 'sell' else self._bids node = tree.get(price) if node is None or not any(o['id'] == order['order_id'] for o in node): return def get_current_ticker(self): return self._current_ticker def get_current_book(self): result = { 'sequence': self._sequence, 'asks': [], 'bids': [], } for ask in self._asks: try: # There can be a race condition here, where a price point is removed # between these two ops this_ask = self._asks[ask] except KeyError: continue for order in this_ask: result['asks'].append([order['price'], order['size'], order['id']]) for bid in self._bids: try: # There can be a race condition here, where a price point is removed # between these two ops this_bid = self._bids[bid] except KeyError: continue for order in this_bid: result['bids'].append([order['price'], order['size'], order['id']]) return result def get_ask(self): return self._asks.peekitem(0)[0] def get_asks(self, price): return self._asks.get(price) def remove_asks(self, price): del self._asks[price] def set_asks(self, price, asks): self._asks[price] = asks def get_bid(self): return self._bids.peekitem(-1)[0] def get_bids(self, price): return self._bids.get(price) def remove_bids(self, price): del self._bids[price] def set_bids(self, price, bids): self._bids[price] = bids
def test_peekitem2(): mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)] temp = SortedDict(mapping) with pytest.raises(IndexError): temp.peekitem(index=100)
class Replica(HasActionQueue, MessageProcessor): def __init__(self, node: 'plenum.server.node.Node', instId: int, isMaster: bool = False): """ Create a new replica. :param node: Node on which this replica is located :param instId: the id of the protocol instance the replica belongs to :param isMaster: is this a replica of the master protocol instance """ super().__init__() self.stats = Stats(TPCStat) self.config = getConfig() routerArgs = [(ReqDigest, self._preProcessReqDigest)] for r in [PrePrepare, Prepare, Commit]: routerArgs.append((r, self.processThreePhaseMsg)) routerArgs.append((Checkpoint, self.processCheckpoint)) routerArgs.append((ThreePCState, self.process3PhaseState)) self.inBoxRouter = Router(*routerArgs) self.threePhaseRouter = Router( (PrePrepare, self.processPrePrepare), (Prepare, self.processPrepare), (Commit, self.processCommit) ) self.node = node self.instId = instId self.name = self.generateName(node.name, self.instId) self.outBox = deque() """ This queue is used by the replica to send messages to its node. Replica puts messages that are consumed by its node """ self.inBox = deque() """ This queue is used by the replica to receive messages from its node. Node puts messages that are consumed by the replica """ self.inBoxStash = deque() """ If messages need to go back on the queue, they go here temporarily and are put back on the queue on a state change """ self.isMaster = isMaster # Indicates name of the primary replica of this protocol instance. # None in case the replica does not know who the primary of the # instance is self._primaryName = None # type: Optional[str] # Requests waiting to be processed once the replica is able to decide # whether it is primary or not self.postElectionMsgs = deque() # PRE-PREPAREs that are waiting to be processed but do not have the # corresponding request digest. Happens when replica has not been # forwarded the request by the node but is getting 3 phase messages. # The value is a list since a malicious entry might send PRE-PREPARE # with a different digest and since we dont have the request finalised, # we store all PRE-PPREPARES self.prePreparesPendingReqDigest = {} # type: Dict[Tuple[str, int], List] # PREPAREs that are stored by non primary replica for which it has not # got any PRE-PREPARE. Dictionary that stores a tuple of view no and # prepare sequence number as key and a deque of PREPAREs as value. # This deque is attempted to be flushed on receiving every # PRE-PREPARE request. self.preparesWaitingForPrePrepare = {} # type: Dict[Tuple[int, int], deque] # COMMITs that are stored for which there are no PRE-PREPARE or PREPARE # received self.commitsWaitingForPrepare = {} # type: Dict[Tuple[int, int], deque] # Dictionary of sent PRE-PREPARE that are stored by primary replica # which it has broadcasted to all other non primary replicas # Key of dictionary is a 2 element tuple with elements viewNo, # pre-prepare seqNo and value is a tuple of Request Digest and time self.sentPrePrepares = {} # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]] # Dictionary of received PRE-PREPAREs. Key of dictionary is a 2 # element tuple with elements viewNo, pre-prepare seqNo and value is # a tuple of Request Digest and time self.prePrepares = {} # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]] # Dictionary of received Prepare requests. Key of dictionary is a 2 # element tuple with elements viewNo, seqNo and value is a 2 element # tuple containing request digest and set of sender node names(sender # replica names in case of multiple protocol instances) # (viewNo, seqNo) -> ((identifier, reqId), {senders}) self.prepares = Prepares() # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]] self.commits = Commits() # type: Dict[Tuple[int, int], # Tuple[Tuple[str, int], Set[str]]] # Set of tuples to keep track of ordered requests. Each tuple is # (viewNo, ppSeqNo) self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]] # Dictionary to keep track of the which replica was primary during each # view. Key is the view no and value is the name of the primary # replica during that view self.primaryNames = {} # type: Dict[int, str] # Holds msgs that are for later views self.threePhaseMsgsForLaterView = deque() # type: deque[(ThreePhaseMsg, str)] # Holds tuple of view no and prepare seq no of 3-phase messages it # received while it was not participating self.stashingWhileCatchingUp = set() # type: Set[Tuple] # Commits which are not being ordered since commits with lower view # numbers and sequence numbers have not been ordered yet. Key is the # viewNo and value a map of pre-prepare sequence number to commit self.stashedCommitsForOrdering = {} # type: Dict[int, # Dict[int, Commit]] self.checkpoints = SortedDict(lambda k: k[0]) self.stashingWhileOutsideWaterMarks = deque() # Low water mark self._h = 0 # type: int # High water mark self.H = self._h + self.config.LOG_SIZE # type: int self.lastPrePrepareSeqNo = self.h # type: int @property def h(self) -> int: return self._h @h.setter def h(self, n): self._h = n self.H = self._h + self.config.LOG_SIZE @property def requests(self): return self.node.requests def shouldParticipate(self, viewNo: int, ppSeqNo: int): # Replica should only participating in the consensus process and the # replica did not stash any of this request's 3-phase request return self.node.isParticipating and (viewNo, ppSeqNo) \ not in self.stashingWhileCatchingUp @staticmethod def generateName(nodeName: str, instId: int): """ Create and return the name for a replica using its nodeName and instanceId. Ex: Alpha:1 """ return "{}:{}".format(nodeName, instId) @staticmethod def getNodeName(replicaName: str): return replicaName.split(":")[0] @property def isPrimary(self): """ Is this node primary? :return: True if this node is primary, False otherwise """ return self._primaryName == self.name if self._primaryName is not None \ else None @property def primaryName(self): """ Name of the primary replica of this replica's instance :return: Returns name if primary is known, None otherwise """ return self._primaryName @primaryName.setter def primaryName(self, value: Optional[str]) -> None: """ Set the value of isPrimary. :param value: the value to set isPrimary to """ if not value == self._primaryName: self._primaryName = value self.primaryNames[self.viewNo] = value logger.debug("{} setting primaryName for view no {} to: {}". format(self, self.viewNo, value)) logger.debug("{}'s primaryNames for views are: {}". format(self, self.primaryNames)) self._stateChanged() def _stateChanged(self): """ A series of actions to be performed when the state of this replica changes. - UnstashInBox (see _unstashInBox) """ self._unstashInBox() if self.isPrimary is not None: # TODO handle suspicion exceptions here self.process3PhaseReqsQueue() # TODO handle suspicion exceptions here try: self.processPostElectionMsgs() except SuspiciousNode as ex: self.outBox.append(ex) self.discard(ex.msg, ex.reason, logger.warning) def _stashInBox(self, msg): """ Stash the specified message into the inBoxStash of this replica. :param msg: the message to stash """ self.inBoxStash.append(msg) def _unstashInBox(self): """ Append the inBoxStash to the right of the inBox. """ self.inBox.extend(self.inBoxStash) self.inBoxStash.clear() def __repr__(self): return self.name @property def f(self) -> int: """ Return the number of Byzantine Failures that can be tolerated by this system. Equal to (N - 1)/3, where N is the number of nodes in the system. """ return self.node.f @property def viewNo(self): """ Return the current view number of this replica. """ return self.node.viewNo def isPrimaryInView(self, viewNo: int) -> Optional[bool]: """ Return whether a primary has been selected for this view number. """ return self.primaryNames[viewNo] == self.name def isMsgForLaterView(self, msg): """ Return whether this request's view number is greater than the current view number of this replica. """ viewNo = getattr(msg, "viewNo", None) return viewNo > self.viewNo def isMsgForCurrentView(self, msg): """ Return whether this request's view number is equal to the current view number of this replica. """ viewNo = getattr(msg, "viewNo", None) return viewNo == self.viewNo def isMsgForPrevView(self, msg): """ Return whether this request's view number is less than the current view number of this replica. """ viewNo = getattr(msg, "viewNo", None) return viewNo < self.viewNo def isPrimaryForMsg(self, msg) -> Optional[bool]: """ Return whether this replica is primary if the request's view number is equal this replica's view number and primary has been selected for the current view. Return None otherwise. :param msg: message """ if self.isMsgForLaterView(msg): self.discard(msg, "Cannot get primary status for a request for a later " "view {}. Request is {}".format(self.viewNo, msg), logger.error) else: return self.isPrimary if self.isMsgForCurrentView(msg) \ else self.isPrimaryInView(msg.viewNo) def isMsgFromPrimary(self, msg, sender: str) -> bool: """ Return whether this message was from primary replica :param msg: :param sender: :return: """ if self.isMsgForLaterView(msg): logger.error("{} cannot get primary for a request for a later " "view. Request is {}".format(self, msg)) else: return self.primaryName == sender if self.isMsgForCurrentView( msg) else self.primaryNames[msg.viewNo] == sender def _preProcessReqDigest(self, rd: ReqDigest) -> None: """ Process request digest if this replica is not a primary, otherwise stash the message into the inBox. :param rd: the client Request Digest """ if self.isPrimary is not None: self.processReqDigest(rd) else: logger.debug("{} stashing request digest {} since it does not know " "its primary status". format(self, (rd.identifier, rd.reqId))) self._stashInBox(rd) def serviceQueues(self, limit=None): """ Process `limit` number of messages in the inBox. :param limit: the maximum number of messages to process :return: the number of messages successfully processed """ # TODO should handle SuspiciousNode here r = self.inBoxRouter.handleAllSync(self.inBox, limit) r += self._serviceActions() return r # Messages that can be processed right now needs to be added back to the # queue. They might be able to be processed later def processPostElectionMsgs(self): """ Process messages waiting for the election of a primary replica to complete. """ while self.postElectionMsgs: msg = self.postElectionMsgs.popleft() logger.debug("{} processing pended msg {}".format(self, msg)) self.dispatchThreePhaseMsg(*msg) def process3PhaseReqsQueue(self): """ Process the 3 phase requests from the queue whose view number is equal to the current view number of this replica. """ unprocessed = deque() while self.threePhaseMsgsForLaterView: request, sender = self.threePhaseMsgsForLaterView.popleft() logger.debug("{} processing pended 3 phase request: {}" .format(self, request)) # If the request is for a later view dont try to process it but add # it back to the queue. if self.isMsgForLaterView(request): unprocessed.append((request, sender)) else: self.processThreePhaseMsg(request, sender) self.threePhaseMsgsForLaterView = unprocessed @property def quorum(self) -> int: r""" Return the quorum of this RBFT system. Equal to :math:`2f + 1`. Return None if `f` is not yet determined. """ return self.node.quorum def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any: """ Create a three phase request to be handled by the threePhaseRouter. :param msg: the ThreePhaseMsg to dispatch :param sender: the name of the node that sent this request """ senderRep = self.generateName(sender, self.instId) if self.isPpSeqNoAcceptable(msg.ppSeqNo): try: self.threePhaseRouter.handleSync((msg, senderRep)) except SuspiciousNode as ex: self.node.reportSuspiciousNodeEx(ex) else: logger.debug("{} stashing 3 phase message {} since ppSeqNo {} is " "not between {} and {}". format(self, msg, msg.ppSeqNo, self.h, self.H)) self.stashingWhileOutsideWaterMarks.append((msg, sender)) def processReqDigest(self, rd: ReqDigest): """ Process a request digest. Works only if this replica has decided its primary status. :param rd: the client request digest to process """ self.stats.inc(TPCStat.ReqDigestRcvd) if self.isPrimary is False: self.dequeuePrePrepare(rd.identifier, rd.reqId) else: self.doPrePrepare(rd) def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str): """ Process a 3-phase (pre-prepare, prepare and commit) request. Dispatch the request only if primary has already been decided, otherwise stash it. :param msg: the Three Phase message, one of PRE-PREPARE, PREPARE, COMMIT :param sender: name of the node that sent this message """ # Can only proceed further if it knows whether its primary or not if self.isMsgForLaterView(msg): self.threePhaseMsgsForLaterView.append((msg, sender)) logger.debug("{} pended received 3 phase request for a later view: " "{}".format(self, msg)) else: if self.isPrimary is None: self.postElectionMsgs.append((msg, sender)) logger.debug("Replica {} pended request {} from {}". format(self, msg, sender)) else: self.dispatchThreePhaseMsg(msg, sender) def processPrePrepare(self, pp: PrePrepare, sender: str): """ Validate and process the PRE-PREPARE specified. If validation is successful, create a PREPARE and broadcast it. :param pp: a prePrepareRequest :param sender: name of the node that sent this message """ key = (pp.viewNo, pp.ppSeqNo) logger.debug("{} Receiving PRE-PREPARE{} at {} from {}". format(self, key, time.perf_counter(), sender)) if self.canProcessPrePrepare(pp, sender): if not self.node.isParticipating: self.stashingWhileCatchingUp.add(key) self.addToPrePrepares(pp) logger.info("{} processed incoming PRE-PREPARE{}". format(self, key)) def tryPrepare(self, pp: PrePrepare): """ Try to send the Prepare message if the PrePrepare message is ready to be passed into the Prepare phase. """ if self.canSendPrepare(pp): self.doPrepare(pp) else: logger.debug("{} cannot send PREPARE".format(self)) def processPrepare(self, prepare: Prepare, sender: str) -> None: """ Validate and process the PREPARE specified. If validation is successful, create a COMMIT and broadcast it. :param prepare: a PREPARE msg :param sender: name of the node that sent the PREPARE """ # TODO move this try/except up higher logger.debug("{} received PREPARE{} from {}". format(self, (prepare.viewNo, prepare.ppSeqNo), sender)) try: if self.isValidPrepare(prepare, sender): self.addToPrepares(prepare, sender) self.stats.inc(TPCStat.PrepareRcvd) logger.debug("{} processed incoming PREPARE {}". format(self, (prepare.viewNo, prepare.ppSeqNo))) else: # TODO let's have isValidPrepare throw an exception that gets # handled and possibly logged higher logger.warning("{} cannot process incoming PREPARE". format(self)) except SuspiciousNode as ex: self.node.reportSuspiciousNodeEx(ex) def processCommit(self, commit: Commit, sender: str) -> None: """ Validate and process the COMMIT specified. If validation is successful, return the message to the node. :param commit: an incoming COMMIT message :param sender: name of the node that sent the COMMIT """ logger.debug("{} received COMMIT {} from {}". format(self, commit, sender)) if self.isValidCommit(commit, sender): self.stats.inc(TPCStat.CommitRcvd) self.addToCommits(commit, sender) logger.debug("{} processed incoming COMMIT{}". format(self, (commit.viewNo, commit.ppSeqNo))) def tryCommit(self, prepare: Prepare): """ Try to commit if the Prepare message is ready to be passed into the commit phase. """ if self.canCommit(prepare): self.doCommit(prepare) else: logger.debug("{} not yet able to send COMMIT".format(self)) def tryOrder(self, commit: Commit): """ Try to order if the Commit message is ready to be ordered. """ canOrder, reason = self.canOrder(commit) if canOrder: logger.debug("{} returning request to node".format(self)) self.tryOrdering(commit) else: logger.trace("{} cannot return request to node: {}". format(self, reason)) def doPrePrepare(self, reqDigest: ReqDigest) -> None: """ Broadcast a PRE-PREPARE to all the replicas. :param reqDigest: a tuple with elements identifier, reqId, and digest """ if not self.node.isParticipating: logger.error("Non participating node is attempting PRE-PREPARE. " "This should not happen.") return if self.lastPrePrepareSeqNo == self.H: logger.debug("{} stashing PRE-PREPARE {} since outside greater " "than high water mark {}". format(self, (self.viewNo, self.lastPrePrepareSeqNo+1), self.H)) self.stashingWhileOutsideWaterMarks.append(reqDigest) return self.lastPrePrepareSeqNo += 1 tm = time.time()*1000 logger.debug("{} Sending PRE-PREPARE {} at {}". format(self, (self.viewNo, self.lastPrePrepareSeqNo), time.perf_counter())) prePrepareReq = PrePrepare(self.instId, self.viewNo, self.lastPrePrepareSeqNo, *reqDigest, tm) self.sentPrePrepares[self.viewNo, self.lastPrePrepareSeqNo] = (reqDigest.key, tm) self.send(prePrepareReq, TPCStat.PrePrepareSent) def doPrepare(self, pp: PrePrepare): logger.debug("{} Sending PREPARE {} at {}". format(self, (pp.viewNo, pp.ppSeqNo), time.perf_counter())) prepare = Prepare(self.instId, pp.viewNo, pp.ppSeqNo, pp.digest, pp.ppTime) self.send(prepare, TPCStat.PrepareSent) self.addToPrepares(prepare, self.name) def doCommit(self, p: Prepare): """ Create a commit message from the given Prepare message and trigger the commit phase :param p: the prepare message """ logger.debug("{} Sending COMMIT{} at {}". format(self, (p.viewNo, p.ppSeqNo), time.perf_counter())) commit = Commit(self.instId, p.viewNo, p.ppSeqNo, p.digest, p.ppTime) self.send(commit, TPCStat.CommitSent) self.addToCommits(commit, self.name) def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool: """ Decide whether this replica is eligible to process a PRE-PREPARE, based on the following criteria: - this replica is non-primary replica - the request isn't in its list of received PRE-PREPAREs - the request is waiting to for PRE-PREPARE and the digest value matches :param pp: a PRE-PREPARE msg to process :param sender: the name of the node that sent the PRE-PREPARE msg :return: True if processing is allowed, False otherwise """ # TODO: Check whether it is rejecting PRE-PREPARE from previous view # PRE-PREPARE should not be sent from non primary if not self.isMsgFromPrimary(pp, sender): raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp) # A PRE-PREPARE is being sent to primary if self.isPrimaryForMsg(pp) is True: raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp) # A PRE-PREPARE is sent that has already been received if (pp.viewNo, pp.ppSeqNo) in self.prePrepares: raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp) key = (pp.identifier, pp.reqId) if not self.requests.isFinalised(key): self.enqueuePrePrepare(pp, sender) return False # A PRE-PREPARE is sent that does not match request digest if self.requests.digest(key) != pp.digest: raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp) return True def addToPrePrepares(self, pp: PrePrepare) -> None: """ Add the specified PRE-PREPARE to this replica's list of received PRE-PREPAREs. :param pp: the PRE-PREPARE to add to the list """ key = (pp.viewNo, pp.ppSeqNo) self.prePrepares[key] = \ ((pp.identifier, pp.reqId), pp.ppTime) self.dequeuePrepares(*key) self.dequeueCommits(*key) self.stats.inc(TPCStat.PrePrepareRcvd) self.tryPrepare(pp) def hasPrepared(self, request) -> bool: return self.prepares.hasPrepareFrom(request, self.name) def canSendPrepare(self, request) -> bool: """ Return whether the request identified by (identifier, requestId) can proceed to the Prepare step. :param request: any object with identifier and requestId attributes """ return self.shouldParticipate(request.viewNo, request.ppSeqNo) \ and not self.hasPrepared(request) \ and self.requests.isFinalised((request.identifier, request.reqId)) def isValidPrepare(self, prepare: Prepare, sender: str) -> bool: """ Return whether the PREPARE specified is valid. :param prepare: the PREPARE to validate :param sender: the name of the node that sent the PREPARE :return: True if PREPARE is valid, False otherwise """ key = (prepare.viewNo, prepare.ppSeqNo) primaryStatus = self.isPrimaryForMsg(prepare) ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares # If a non primary replica and receiving a PREPARE request before a # PRE-PREPARE request, then proceed # PREPARE should not be sent from primary if self.isMsgFromPrimary(prepare, sender): raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare) # If non primary replica if primaryStatus is False: if self.prepares.hasPrepareFrom(prepare, sender): raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare) # If PRE-PREPARE not received for the PREPARE, might be slow network if key not in ppReqs: self.enqueuePrepare(prepare, sender) return False elif prepare.digest != self.requests.digest(ppReqs[key][0]): raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare) elif prepare.ppTime != ppReqs[key][1]: raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare) else: return True # If primary replica else: if self.prepares.hasPrepareFrom(prepare, sender): raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare) # If PRE-PREPARE was not sent for this PREPARE, certainly # malicious behavior elif key not in ppReqs: raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT, prepare) elif prepare.digest != self.requests.digest(ppReqs[key][0]): raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare) elif prepare.ppTime != ppReqs[key][1]: raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG, prepare) else: return True def addToPrepares(self, prepare: Prepare, sender: str): self.prepares.addVote(prepare, sender) self.tryCommit(prepare) def hasCommitted(self, request) -> bool: return self.commits.hasCommitFrom(ThreePhaseKey( request.viewNo, request.ppSeqNo), self.name) def canCommit(self, prepare: Prepare) -> bool: """ Return whether the specified PREPARE can proceed to the Commit step. Decision criteria: - If this replica has got just 2f PREPARE requests then commit request. - If less than 2f PREPARE requests then probably there's no consensus on the request; don't commit - If more than 2f then already sent COMMIT; don't commit :param prepare: the PREPARE """ return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \ self.prepares.hasQuorum(prepare, self.f) and \ not self.hasCommitted(prepare) def isValidCommit(self, commit: Commit, sender: str) -> bool: """ Return whether the COMMIT specified is valid. :param commit: the COMMIT to validate :return: True if `request` is valid, False otherwise """ primaryStatus = self.isPrimaryForMsg(commit) ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares key = (commit.viewNo, commit.ppSeqNo) if key not in ppReqs: self.enqueueCommit(commit, sender) return False if (key not in self.prepares and key not in self.preparesWaitingForPrePrepare): logger.debug("{} rejecting COMMIT{} due to lack of prepares". format(self, key)) # raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit) return False elif self.commits.hasCommitFrom(commit, sender): raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit) elif commit.digest != self.getDigestFor3PhaseKey(ThreePhaseKey(*key)): raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit) elif key in ppReqs and commit.ppTime != ppReqs[key][1]: raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG, commit) else: return True def addToCommits(self, commit: Commit, sender: str): """ Add the specified COMMIT to this replica's list of received commit requests. :param commit: the COMMIT to add to the list :param sender: the name of the node that sent the COMMIT """ self.commits.addVote(commit, sender) self.tryOrder(commit) def hasOrdered(self, viewNo, ppSeqNo) -> bool: return (viewNo, ppSeqNo) in self.ordered def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]: """ Return whether the specified commitRequest can be returned to the node. Decision criteria: - If have got just 2f+1 Commit requests then return request to node - If less than 2f+1 of commit requests then probably don't have consensus on the request; don't return request to node - If more than 2f+1 then already returned to node; don't return request to node :param commit: the COMMIT """ if not self.commits.hasQuorum(commit, self.f): return False, "no quorum: {} commits where f is {}".\ format(commit, self.f) if self.hasOrdered(commit.viewNo, commit.ppSeqNo): return False, "already ordered" if not self.isNextInOrdering(commit): viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo if viewNo not in self.stashedCommitsForOrdering: self.stashedCommitsForOrdering[viewNo] = {} self.stashedCommitsForOrdering[viewNo][ppSeqNo] = commit # self._schedule(self.orderStashedCommits, 2) self.startRepeating(self.orderStashedCommits, 2) return False, "stashing {} since out of order".\ format(commit) return True, None def isNextInOrdering(self, commit: Commit): viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo if self.ordered and self.ordered[-1] == (viewNo, ppSeqNo-1): return True for (v, p) in self.commits: if v < viewNo: # Have commits from previous view that are unordered. # TODO: Question: would commits be always ordered, what if # some are never ordered and its fine, go to PBFT. return False if v == viewNo and p < ppSeqNo and (v, p) not in self.ordered: # If unordered commits are found with lower ppSeqNo then this # cannot be ordered. return False # TODO: Revisit PBFT paper, how to make sure that last request of the # last view has been ordered? Need change in `VIEW CHANGE` mechanism. # Somehow view change needs to communicate what the last request was. # Also what if some COMMITs were completely missed in the same view return True def orderStashedCommits(self): # TODO: What if the first few commits were out of order and stashed? # `self.ordered` would be empty if self.ordered: lastOrdered = self.ordered[-1] vToRemove = set() for v in self.stashedCommitsForOrdering: if v < lastOrdered[0] and self.stashedCommitsForOrdering[v]: raise RuntimeError("{} found commits from previous view {}" " that were not ordered but last ordered" " is {}".format(self, v, lastOrdered)) pToRemove = set() for p, commit in self.stashedCommitsForOrdering[v].items(): if (v == lastOrdered[0] and lastOrdered == (v, p - 1)) or \ (v > lastOrdered[0] and self.isLowestCommitInView(commit)): logger.debug("{} ordering stashed commit {}". format(self, commit)) if self.tryOrdering(commit): lastOrdered = (v, p) pToRemove.add(p) for p in pToRemove: del self.stashedCommitsForOrdering[v][p] if not self.stashedCommitsForOrdering[v]: vToRemove.add(v) for v in vToRemove: del self.stashedCommitsForOrdering[v] # if self.stashedCommitsForOrdering: # self._schedule(self.orderStashedCommits, 2) if not self.stashedCommitsForOrdering: self.stopRepeating(self.orderStashedCommits) def isLowestCommitInView(self, commit): # TODO: Assumption: This assumes that at least one commit that was sent # for any request by any node has been received in the view of this # commit ppSeqNos = [] for v, p in self.commits: if v == commit.viewNo: ppSeqNos.append(p) return min(ppSeqNos) == commit.ppSeqNo if ppSeqNos else True def tryOrdering(self, commit: Commit) -> None: """ Attempt to send an ORDERED request for the specified COMMIT to the node. :param commit: the COMMIT message """ key = (commit.viewNo, commit.ppSeqNo) logger.debug("{} trying to order COMMIT{}".format(self, key)) reqKey = self.getReqKeyFrom3PhaseKey(key) # type: Tuple digest = self.getDigestFor3PhaseKey(key) if not digest: logger.error("{} did not find digest for {}, request key {}". format(self, key, reqKey)) return self.doOrder(*key, *reqKey, digest, commit.ppTime) return True def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime): key = (viewNo, ppSeqNo) self.addToOrdered(*key) ordered = Ordered(self.instId, viewNo, identifier, reqId, ppTime) # TODO: Should not order or add to checkpoint while syncing # 3 phase state. self.send(ordered, TPCStat.OrderSent) if key in self.stashingWhileCatchingUp: self.stashingWhileCatchingUp.remove(key) logger.debug("{} ordered request {}".format(self, (viewNo, ppSeqNo))) self.addToCheckpoint(ppSeqNo, digest) def processCheckpoint(self, msg: Checkpoint, sender: str): if self.checkpoints: seqNo = msg.seqNo _, firstChk = self.firstCheckPoint if firstChk.isStable: if firstChk.seqNo == seqNo: self.discard(msg, reason="Checkpoint already stable", logMethod=logger.debug) return if firstChk.seqNo > seqNo: self.discard(msg, reason="Higher stable checkpoint present", logMethod=logger.debug) return for state in self.checkpoints.values(): if state.seqNo == seqNo: if state.digest == msg.digest: state.receivedDigests[sender] = msg.digest break else: logger.error("{} received an incorrect digest {} for " "checkpoint {} from {}".format(self, msg.digest, seqNo, sender)) return if len(state.receivedDigests) == 2*self.f: self.markCheckPointStable(msg.seqNo) else: self.discard(msg, reason="No checkpoints present to tally", logMethod=logger.warn) def _newCheckpointState(self, ppSeqNo, digest) -> CheckpointState: s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ - 1 logger.debug("{} adding new checkpoint state for {}". format(self, (s, e))) state = CheckpointState(ppSeqNo, [digest, ], None, {}, False) self.checkpoints[s, e] = state return state def addToCheckpoint(self, ppSeqNo, digest): for (s, e) in self.checkpoints.keys(): if s <= ppSeqNo <= e: state = self.checkpoints[s, e] # type: CheckpointState state.digests.append(digest) state = updateNamedTuple(state, seqNo=ppSeqNo) self.checkpoints[s, e] = state break else: state = self._newCheckpointState(ppSeqNo, digest) s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ if len(state.digests) == self.config.CHK_FREQ: state = updateNamedTuple(state, digest=serialize(state.digests), digests=[]) self.checkpoints[s, e] = state self.send(Checkpoint(self.instId, self.viewNo, ppSeqNo, state.digest)) def markCheckPointStable(self, seqNo): previousCheckpoints = [] for (s, e), state in self.checkpoints.items(): if e == seqNo: state = updateNamedTuple(state, isStable=True) self.checkpoints[s, e] = state break else: previousCheckpoints.append((s, e)) else: logger.error("{} could not find {} in checkpoints". format(self, seqNo)) return self.h = seqNo for k in previousCheckpoints: logger.debug("{} removing previous checkpoint {}".format(self, k)) self.checkpoints.pop(k) self.gc(seqNo) logger.debug("{} marked stable checkpoint {}".format(self, (s, e))) self.processStashedMsgsForNewWaterMarks() def gc(self, tillSeqNo): logger.debug("{} cleaning up till {}".format(self, tillSeqNo)) tpcKeys = set() reqKeys = set() for (v, p), (reqKey, _) in self.sentPrePrepares.items(): if p <= tillSeqNo: tpcKeys.add((v, p)) reqKeys.add(reqKey) for (v, p), (reqKey, _) in self.prePrepares.items(): if p <= tillSeqNo: tpcKeys.add((v, p)) reqKeys.add(reqKey) logger.debug("{} found {} 3 phase keys to clean". format(self, len(tpcKeys))) logger.debug("{} found {} request keys to clean". format(self, len(reqKeys))) for k in tpcKeys: self.sentPrePrepares.pop(k, None) self.prePrepares.pop(k, None) self.prepares.pop(k, None) self.commits.pop(k, None) if k in self.ordered: self.ordered.remove(k) for k in reqKeys: self.requests.pop(k, None) def processStashedMsgsForNewWaterMarks(self): while self.stashingWhileOutsideWaterMarks: item = self.stashingWhileOutsideWaterMarks.pop() logger.debug("{} processing stashed item {} after new stable " "checkpoint".format(self, item)) if isinstance(item, ReqDigest): self.doPrePrepare(item) elif isinstance(item, tuple) and len(tuple) == 2: self.dispatchThreePhaseMsg(*item) else: logger.error("{} cannot process {} " "from stashingWhileOutsideWaterMarks". format(self, item)) @property def firstCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]: if not self.checkpoints: return None else: return self.checkpoints.peekitem(0) @property def lastCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]: if not self.checkpoints: return None else: return self.checkpoints.peekitem(-1) def isPpSeqNoAcceptable(self, ppSeqNo: int): return self.h < ppSeqNo <= self.H def addToOrdered(self, viewNo: int, ppSeqNo: int): self.ordered.add((viewNo, ppSeqNo)) def enqueuePrePrepare(self, request: PrePrepare, sender: str): logger.debug("Queueing pre-prepares due to unavailability of finalised " "Request. Request {} from {}".format(request, sender)) key = (request.identifier, request.reqId) if key not in self.prePreparesPendingReqDigest: self.prePreparesPendingReqDigest[key] = [] self.prePreparesPendingReqDigest[key].append((request, sender)) def dequeuePrePrepare(self, identifier: int, reqId: int): key = (identifier, reqId) if key in self.prePreparesPendingReqDigest: pps = self.prePreparesPendingReqDigest[key] for (pp, sender) in pps: logger.debug("{} popping stashed PRE-PREPARE{}". format(self, key)) if pp.digest == self.requests.digest(key): self.prePreparesPendingReqDigest.pop(key) self.processPrePrepare(pp, sender) logger.debug( "{} processed {} PRE-PREPAREs waiting for finalised " "request for identifier {} and reqId {}". format(self, pp, identifier, reqId)) break def enqueuePrepare(self, request: Prepare, sender: str): logger.debug("Queueing prepares due to unavailability of PRE-PREPARE. " "Request {} from {}".format(request, sender)) key = (request.viewNo, request.ppSeqNo) if key not in self.preparesWaitingForPrePrepare: self.preparesWaitingForPrePrepare[key] = deque() self.preparesWaitingForPrePrepare[key].append((request, sender)) def dequeuePrepares(self, viewNo: int, ppSeqNo: int): key = (viewNo, ppSeqNo) if key in self.preparesWaitingForPrePrepare: i = 0 # Keys of pending prepares that will be processed below while self.preparesWaitingForPrePrepare[key]: prepare, sender = self.preparesWaitingForPrePrepare[ key].popleft() logger.debug("{} popping stashed PREPARE{}".format(self, key)) self.processPrepare(prepare, sender) i += 1 self.preparesWaitingForPrePrepare.pop(key) logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for" " view no {} and seq no {}". format(self, i, viewNo, ppSeqNo)) def enqueueCommit(self, request: Commit, sender: str): logger.debug("Queueing commit due to unavailability of PREPARE. " "Request {} from {}".format(request, sender)) key = (request.viewNo, request.ppSeqNo) if key not in self.commitsWaitingForPrepare: self.commitsWaitingForPrepare[key] = deque() self.commitsWaitingForPrepare[key].append((request, sender)) def dequeueCommits(self, viewNo: int, ppSeqNo: int): key = (viewNo, ppSeqNo) if key in self.commitsWaitingForPrepare: i = 0 # Keys of pending prepares that will be processed below while self.commitsWaitingForPrepare[key]: commit, sender = self.commitsWaitingForPrepare[ key].popleft() logger.debug("{} popping stashed COMMIT{}".format(self, key)) self.processCommit(commit, sender) i += 1 self.commitsWaitingForPrepare.pop(key) logger.debug("{} processed {} COMMITs waiting for PREPARE for" " view no {} and seq no {}". format(self, i, viewNo, ppSeqNo)) def getDigestFor3PhaseKey(self, key: ThreePhaseKey) -> Optional[str]: reqKey = self.getReqKeyFrom3PhaseKey(key) digest = self.requests.digest(reqKey) if not digest: logger.debug("{} could not find digest in sent or received " "PRE-PREPAREs or PREPAREs for 3 phase key {} and req " "key {}".format(self, key, reqKey)) return None else: return digest def getReqKeyFrom3PhaseKey(self, key: ThreePhaseKey): reqKey = None if key in self.sentPrePrepares: reqKey = self.sentPrePrepares[key][0] elif key in self.prePrepares: reqKey = self.prePrepares[key][0] elif key in self.prepares: reqKey = self.prepares[key][0] else: logger.debug("Could not find request key for 3 phase key {}". format(key)) return reqKey @property def threePhaseState(self): # TODO: This method is incomplete # Gets the current stable and unstable checkpoints and creates digest # of unstable checkpoints if self.checkpoints: pass else: state = [] return ThreePCState(self.instId, state) def process3PhaseState(self, msg: ThreePCState, sender: str): # TODO: This is not complete pass def send(self, msg, stat=None) -> None: """ Send a message to the node on which this replica resides. :param msg: the message to send """ logger.display("{} sending {}".format(self, msg.__class__.__name__), extra={"cli": True}) logger.trace("{} sending {}".format(self, msg)) if stat: self.stats.inc(stat) self.outBox.append(msg)
def test_peekitem(): mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)] temp = SortedDict(mapping) assert temp.peekitem() == ('z', 25) assert temp.peekitem(0) == ('a', 0) assert temp.peekitem(index=4) == ('e', 4)