class StockPrice(object):
def __init__(self):
self.timeToPrice = SortedDict() #time will be sorted
self.priceToTime = SortedDict() #the price will be sorted
def update(self, timestamp, price):
if timestamp in self.timeToPrice:
prevPrice = self.timeToPrice[timestamp]
self.priceToTime[prevPrice].remove(timestamp)
if len(self.priceToTime[prevPrice])==0: self.priceToTime.pop(prevPrice) #[0]
if price not in self.priceToTime: self.priceToTime[price] = set() #initialized
self.priceToTime[price].add(timestamp)
self.timeToPrice[timestamp] = price
def current(self):
return self.timeToPrice.peekitem(-1)[1]
def maximum(self):
return self.priceToTime.peekitem(-1)[0]
def minimum(self):
return self.priceToTime.peekitem(0)[0]
class StockPrice: # 45%, 12%
def __init__(self):
# 两个TreeMap
self.timeToPrice = SortedDict() # 时间->价格，key已排序
self.priceToTimes = SortedDict() # 价格->Set<该价格的时间>，key已排序
def update(self, timestamp: int, price: int) -> None:
if timestamp in self.timeToPrice: # 时间已存在，是更新价格
oldPrice = self.timeToPrice[timestamp]
self.priceToTimes[oldPrice].remove(timestamp) # 去掉老的价格中的时间，因为已失效
if not self.priceToTimes[oldPrice]: # 如果老价格不再对应任何时间，去除key
self.priceToTimes.pop(oldPrice)
self.timeToPrice[timestamp] = price # 更新/新建 时间->价格
if not price in self.priceToTimes: # 更新/新建 价格->时间们
self.priceToTimes[price] = set()
self.priceToTimes[price].add(timestamp)
def current(self) -> int:
return self.timeToPrice.peekitem(-1)[1] # 找最大的时间key，得到其对应价格
def maximum(self) -> int:
return self.priceToTimes.peekitem(-1)[0] # 找最大的价格key，得到key
def minimum(self) -> int:
return self.priceToTimes.peekitem(0)[0] # 找最小的价格key，得到key
def oddEvenJumps(self, arr: List[int]) -> int:
N = len(arr)
odd = [False] * N
even = [False] * N
odd[-1] = True
even[-1] = True
sd = SortedDict()
sd[arr[N - 1]] = N - 1
for i in range(N - 2, -1, -1):
if arr[i] in sd:
odd[i] = even[sd[arr[i]]]
even[i] = odd[sd[arr[i]]]
else:
# greatest smaller
floor_idx = sd.bisect_left(arr[i]) - 1
if floor_idx != -1:
even[i] = odd[sd.peekitem(floor_idx)[1]]
# smallest greater
ceiling_idx = sd.bisect_left(arr[i])
if ceiling_idx != len(sd):
odd[i] = even[sd.peekitem(ceiling_idx)[1]]
sd[arr[i]] = i
return odd.count(True)
def longestSubarray(self, nums: List[int], limit: int) -> int:
n = len(nums)
w = SortedDict()
start = 0
end = 0
def add(value):
if value not in w:
w[value] = 0
w[value] += 1
def delete(value):
if w[value] == 1:
del w[value]
else:
w[value] -= 1
while end < n:
add(nums[end])
largest = w.peekitem(index=-1)
smallest = w.peekitem(index=0)
if largest[0] - smallest[0] > limit:
delete(nums[start])
start += 1
end += 1
return end - start
class StockPrice0:
## not using sorteddict for time_to_prices
def __init__(self):
self.time_to_prices = {}
self.rec = SortedDict()
self.cur_time = -1
def update(self, timestamp: int, price: int) -> None:
self.cur_time = max(self.cur_time, timestamp)
if timestamp in self.time_to_prices:
prev_price = self.time_to_prices[timestamp]
self.rec[prev_price].remove(timestamp)
if len(self.rec[prev_price]) == 0:
self.rec.pop(prev_price)
if not price in self.rec:
self.rec[price] = set()
self.rec[price].add(timestamp)
self.time_to_prices[timestamp] = price
def current(self) -> int:
# return self.time_to_prices.peekitem(-1)[1]
return self.time_to_prices[self.cur_time]
def maximum(self) -> int:
return self.rec.peekitem(-1)[0]
def minimum(self) -> int:
return self.rec.peekitem(0)[0]
def oddEvenJumps(self, nums) -> int:
if len(nums) == 0:
return 0
if len(nums) == 1:
return 1
n = len(nums)
isOkEven = [False] * len(nums)
isOkOdd = [False] * len(nums)
isOkEven[n - 1] = True
isOkOdd[n - 1] = True
tree = SortedDict()
tree[nums[n - 1]] = n - 1
for i in range(n - 2, -1, -1):
val = nums[i]
if val in tree:
isOkEven[i] = isOkOdd[tree[val]]
isOkOdd[i] = isOkEven[tree[val]]
else:
smallestP = tree.bisect_left(val)
largestP = tree.bisect_left(val) - 1
isOkOdd[i] = True if smallestP != len(tree) and isOkEven[
tree.peekitem(smallestP)[1]] else False
isOkEven[i] = True if largestP != -1 and isOkOdd[tree.peekitem(
largestP)[1]] else False
tree[val] = i
res = 0
for e in isOkOdd:
if e:
res += 1
return res
class StockPrice:
def __init__(self):
self.tpdict = SortedDict()
self.ptdict = SortedDict()
def update(self, timestamp: int, price: int) -> None:
if timestamp not in self.tpdict:
self.tpdict[timestamp] = price
if price not in self.ptdict:
self.ptdict[price] = set()
self.ptdict[price].add(timestamp)
else:
prevp = self.tpdict[timestamp]
self.tpdict[timestamp] = price
self.ptdict[prevp].remove(timestamp)
if not self.ptdict[prevp]:
del self.ptdict[prevp]
if price not in self.ptdict:
self.ptdict[price] = set()
self.ptdict[price].add(timestamp)
def current(self) -> int:
return self.tpdict.peekitem(-1)[1]
def maximum(self) -> int:
return self.ptdict.peekitem(-1)[0]
def minimum(self) -> int:
return self.ptdict.peekitem(0)[0]
class MaxStack:
def __init__(self):
self.map = SortedDict()
self.head = Node(-1)
self.tail = Node(-1)
self.head.next, self.tail.prev = self.tail, self.head
def push(self, x: int) -> None:
node = Node(x)
self.insert(node)
def pop(self) -> int:
node = self.tail.prev
self.remove(node)
return node.val
def top(self) -> int:
return self.tail.prev.val
def peekMax(self) -> int:
return self.map.peekitem()[0]
def popMax(self) -> int:
key, nodes = self.map.peekitem()
self.remove(nodes[-1])
return key
def insert(self, node) -> None:
prev, nxt = self.tail.prev, self.tail
node.prev, node.next = prev, nxt
prev.next, nxt.prev = node, node
if node.val in self.map:
self.map[node.val].append(node)
else:
self.map[node.val] = [node]
def remove(self, node) -> None:
prev, nxt = node.prev, node.next
prev.next, nxt.prev = nxt, prev
for i in range(len(self.map[node.val]) - 1, -1, -1):
if self.map[node.val][i] == node:
del self.map[node.val][i]
if not self.map[node.val] or not len(self.map[node.val]):
del self.map[node.val]
return
# Your MaxStack object will be instantiated and called as such:
# obj = MaxStack()
# obj.push(x)
# param_2 = obj.pop()
# param_3 = obj.top()
# param_4 = obj.peekMax()
# param_5 = obj.popMax()
def top_view(self):
sorted_endpoints = []
for segment_ in self.segments:
sorted_endpoints.append(Endpoint(True, segment_))
sorted_endpoints.append(Endpoint(False, segment_))
sorted_endpoints.sort(
key=lambda end_point: end_point.value()) # O n(log n)
resulted_segments = []
# sweeper = SortedDict(key=lambda ep: ep.height)
sweeper = SortedDict()
print(sweeper)
for endpoint_ in sorted_endpoints:
if sweeper.__len__() == 0:
sweeper[endpoint_] = endpoint_.segment
elif not endpoint_.is_left: # right of some segment
left_ep = Endpoint(True, endpoint_.segment)
if left_ep.__eq__(
sweeper.peekitem(0)[0]): # "top most" is ending now
sweeper.pop(sweeper.peekitem(0)[0])
prev_seg = resulted_segments[resulted_segments.__len__() -
1]
resulted_segments.append(
Segment(prev_seg.right, endpoint_.segment.right,
endpoint_.segment.height,
endpoint_.segment.name))
else: # some segment "not at top" needs to end now
sweeper.pop(left_ep)
elif sweeper.peekitem(
0
)[0].segment.height < endpoint_.segment.height: # crossover, a higher segment found
current_top = sweeper.peekitem(0)[0]
if resulted_segments.__len__() > 0:
last_segment = resulted_segments[
resulted_segments.__len__() - 1]
resulted_segments.append(
Segment(last_segment.right, endpoint_.segment.left,
current_top.segment.height,
current_top.segment.name))
else:
resulted_segments.append(
Segment(current_top.segment.left,
endpoint_.segment.left,
current_top.segment.height,
current_top.segment.name))
sweeper[endpoint_] = endpoint_.segment
elif sweeper.peekitem(
0
)[0].segment.height > endpoint_.segment.height: # non contributing end found
sweeper[endpoint_] = endpoint_.segment
return resulted_segments
class SortedSlidingDic:
def __init__(self, stime):
self.flow_dic = {}
self.ts_dic = SortedDict()
self.stime = float(self.unified_ts(stime))
def unified_ts(self, ts):
return round(ts, 10)
def update(self, ts):
ts = self.unified_ts(ts)
while len(self.ts_dic) > 0 and ts - self.ts_dic.peekitem(0)[0] > self.stime:
del self.flow_dic[self.ts_dic.peekitem(0)[1]]
self.ts_dic.popitem(0)
def add(self, flow, ts):
ts = self.unified_ts(ts)
# Remove the previous timestamp for this flow and the new one instead
self.remove(flow)
self.flow_dic[flow] = ts
if ts in self.ts_dic:
del self.flow_dic[self.ts_dic[ts]]
self.ts_dic[ts] = flow
assert len(self.flow_dic) == len(self.ts_dic)
def remove(self, flow):
if flow in self.flow_dic:
ts = self.flow_dic[flow]
try:
del self.flow_dic[flow]
self.ts_dic.pop(ts)
except KeyError:
print 'KeyError ', flow, ts
def __str__(self):
res = ''
for k in self.ts_dic:
res += str("%.10f" % k)+'\t'+str(self.ts_dic[k])+'\n'
print res
class BisectNodePolicy(BaseNodePolicy):
def __init__(self, hash_class=defaultHashClass):
self.ring = SortedDict()
super(BisectNodePolicy, self).__init__(hash_class=hash_class)
def add_node(self, node=None, vnode_count=None):
for i in range_(int(vnode_count)):
self.ring[self._gen_key(node, i)] = node
def remove_node(self, node=None):
keys = list(self.ring.keys())
for key in keys:
if self.ring[key] == node:
self.ring.pop(key)
def get_proper_node(self, key):
key, _ = self.ring.peekitem(self._find_proper_pos(key))
return self.ring[key]
def _find_proper_pos(self, key):
key = self._gen_key(key)
pos = self.ring.bisect(key)
# if object_hash == node_hash, return node index
if key in self.ring:
return pos - 1
# embodies the concept of the ring.
if pos == len(self.ring):
return 0
return pos
def canAttendMeetings(self, intervals: List[List[int]]) -> bool:
points = SortedDict()
for start, end in intervals:
# print(start, end)
# print(points)
i_start = points.bisect_right(start)
i_end = points.bisect_left(end)
# print("i_start", i_start)
# print("i_end", i_end)
if i_end != i_start:
return False
if i_start > 0 and points.peekitem(i_start-1)[1] == 1:
return False
if points.get(start) == -1:
del points[start]
else:
points[start] = 1
if points.get(end) == 1:
del points[end]
else:
points[end] = -1
return True
def getSkyline(self, buildings: List[List[int]]) -> List[List[int]]:
def update_heights_count(typ, sd, height):
if typ == 1:
if height not in sd:
sd[height] = 1
else:
sd[height] += 1
else:
if sd[height] == 1:
del sd[height]
else:
sd[height] -= 1
res = []
d = defaultdict(list)
prev_h = 0
sd = SortedDict()
for l, r, h in buildings:
d[l].append((h, 1))
d[r].append((h, -1))
for x in sorted(d.keys()):
for height, typ in d[x]:
update_heights_count(typ, sd, height)
curr_h = sd.peekitem(-1)[0] if sd else 0
if curr_h != prev_h:
res.append([x, curr_h])
prev_h = curr_h
return res
def find_closest_elements_in_sorted_arrays(sorted_arrays):
diff = float('inf')
tree = SortedDict()
for idx, array in enumerate(sorted_arrays):
i = iter(array)
nxt = next(i)
tree[(nxt, idx)] = i
while True:
min_val, min_idx = tree.peekitem(0)[0]
max_val = tree.peekitem()[0][0]
diff = min(diff, max_val - min_val)
i = tree.popitem(0)[1]
nxt = next(i, None)
if nxt is None:
return diff
tree[(nxt, min_idx)] = i
def find_closest_elements_in_sorted_arrays(sorted_arrays):
# TODO - you fill in here.
min_diff = float('inf')
chosen_set = []
iters = SortedDict()
for idx, s in enumerate(sorted_arrays):
it = iter(s)
first_min = next(it, None)
assert first_min is not None, pdb.set_trace()
if first_min is not None:
iters[first_min] = (idx, it)
while True:
max_val = iters.keys()[-1]
min_val, (min_idx, it) = iters.peekitem(0)
if min_diff > max_val - min_val:
min_diff = max_val - min_val
chosen_set = sorted(list((v, i) for v, (i, _) in iters.items()),
key=lambda x: x[1])
iters.popitem(0)
next_min = next(it, None)
if next_min == None:
print(chosen_set)
return min_diff
iters[next_min] = (min_idx, it)
assert len(iters) == len(sorted_arrays), pdb.set_trace()
class ClientsCreditsInfo:
def __init__(self):
self.offset = 0
self.clients = {}
self.credits = SortedDict()
def insert(self, client_id, c):
self.remove(client_id)
self.clients[client_id] = c - self.offset
self.credits[c - self.offset] = client_id
def remove(self, client_id):
if client_id not in self.clients:
return False
credit = self.clients[client_id]
if not self.credits[credit]:
del self.credits[credit]
del self.clients[client_id]
return True
def lookup(self, client_id):
if client_id not in self.clients:
return -1
credit = self.clients[client_id]
return credit + self.offset
def add_all(self, C):
self.offset += C
def max(self):
if not self.credits:
return ''
return self.credits.peekitem(-1)[1]
class Book(object):
def __init__(self):
self._bid = SortedDict()
self._ask = SortedDict()
self._info = {}
def random_init(self):
for px, count in zip(
np.arange(1, 11),
np.histogram(np.clip(
np.random.poisson(4, 1000),
1,
10,
),
bins=10)[0]):
self._ask[px] = Level("ASK", px, [Order() for _ in range(count)])
for px, count in zip(
np.arange(0, -10, -1),
np.histogram(np.clip(
np.random.poisson(4, 1000),
1,
10,
),
bins=10)[0]):
self._bid[px] = Level("BID", px, [Order() for _ in range(count)])
return self
# def is_crossed(self) -> bool:
# if
def imshow(self):
fig, ax = plt.subplots(figsize=(16, 10))
ax.barh(self._ask.keys(), [level.qty for level in self._ask.values()],
color='red')
ax.barh(self._bid.keys(), [-level.qty for level in self._bid.values()],
color='blue')
ax.grid()
ax.set_yticks(
range(self._bid.peekitem(0)[0],
self._ask.peekitem(-1)[0] + 1))
plt.show()
def __repr__(self):
ask_str = '\n'.join(f'\t{level}' for level in self._ask.values())
bid_str = '\n'.join(f'\t{level}' for level in self._bid.values())
return f'Book {self._info}' + ' {\n' + (ask_str + bid_str).replace(
'\n', '\n\t') + '\n}'
class TimerScheduler:
def __init__(self):
self._epoch = datetime.now()
self._timers_by_expire_time = SortedDict()
def now(self):
absolute_now = datetime.now()
time_since_epoch = absolute_now - self._epoch
return time_since_epoch.total_seconds()
def schedule(self, timer):
expire_time = timer.expire_time()
assert expire_time is not None
if expire_time in self._timers_by_expire_time:
self._timers_by_expire_time[expire_time].append(timer)
else:
self._timers_by_expire_time[expire_time] = [timer]
def unschedule(self, timer):
expire_time = timer.expire_time()
assert expire_time is not None
assert expire_time in self._timers_by_expire_time
timers_with_matching_expire = self._timers_by_expire_time[expire_time]
assert timer in timers_with_matching_expire
timers_with_matching_expire.remove(timer)
if timers_with_matching_expire == []:
self._timers_by_expire_time.pop(expire_time)
def trigger_all_expired_timers(self):
# Trigger all expired timers and return time until next expire
now = self.now()
while True:
if not self._timers_by_expire_time:
return None
next_expire_time = self._timers_by_expire_time.peekitem(0)[0]
if next_expire_time > now:
return next_expire_time - now
expired_timers = self._timers_by_expire_time.popitem(0)[1]
for timer in expired_timers:
timer.trigger_expire()
def stop_all_timers(self):
while self._timers_by_expire_time:
timers = self._timers_by_expire_time.peekitem(0)[1]
for timer in timers:
timer.stop()
def find_closest_elements(sorted_arrays):
min_distance_so_far = float('inf')
iters = SortedDict()
for idx, sorted_array in enumerate(sorted_arrays):
it = iter(sorted_array)
first_min = next(it, None)
if first_min is not None:
iters.setdefault((first_min, idx), default=it)
while True:
min_value, min_idx = iters.peekitem(index=0)[0]
max_value = iters.peekitem()[0][0]
min_distance_so_far = min(max_value - min_value, min_distance_so_far)
it = iters.popitem(index=0)[1]
next_min = next(it, None)
if next_min is None:
return min_distance_so_far
iters.setdefault((next_min, min_idx), it)
class MenuBar(QMenuBar):
def __init__(self,
menu_order: Tuple[MainMenu] = (FileMenu, ViewMenu, ToolsMenu,
AlgorithmsMenu, WindowsMenu,
HelpMenu)):
super().__init__()
self._menu_order = menu_order
self._menus_order_indexes = {
menu_type: i
for (i, menu_type) in enumerate(self._menu_order)
}
self._ordered_added_menus = SortedDict(
) # {order_index: MainMenu class}
def add_menu(self, menu_type: Type[MainMenu]) -> MainMenu:
menu = menu_type()
menu_order_index = self._menu_order_index(menu_type)
self._ordered_added_menus[menu_order_index] = menu
menu_index_in_ordered_added_menus = self._ordered_added_menus.index(
menu_order_index)
# If the menu is the last one
if menu_index_in_ordered_added_menus == len(
self._ordered_added_menus) - 1:
self.addMenu(menu)
else:
next_menu_index_in_ordered_added_menus = menu_index_in_ordered_added_menus + 1
next_menu = self._ordered_added_menus.peekitem(
next_menu_index_in_ordered_added_menus)[1]
self.insertMenu(next_menu.menuAction(), menu)
return menu
def menu(self,
menu_type: Type[MainMenu],
add_nonexistent: bool = True) -> Optional[MainMenu]:
menu = self._ordered_added_menus.get(self._menu_order_index(menu_type))
if menu is None and add_nonexistent:
menu = self.add_menu(menu_type)
return menu
def add_menu_action(self,
menu_type: Type[MainMenu],
action_name,
method,
shortcut=None) -> QAction:
return self.menu(menu_type).addAction(action_name, method, shortcut)
def _menu_order_index(self, menu_type: Type[MainMenu]) -> int:
return self._menus_order_indexes[menu_type]
class QtDictListModel(QAbstractListModel):
def __init__(self):
QAbstractListModel.__init__(self)
self._items = SortedDict()
def role(self, item, role):
return item
def rowCount(self, parent):
if parent.isValid():
return 0
return len(self._items)
def from_index(self, index):
if not index.isValid() or index.row() >= len(self._items):
return None
return self._items.peekitem(index.row())[1]
def data(self, index, role):
item = self.from_index(index)
if item is None:
return None
return self.role(item, role)
def _add(self, key, item):
assert key not in self._items
next_index = self._items.bisect_left(key)
self.beginInsertRows(QModelIndex(), next_index, next_index)
self._items[key] = item
self.endInsertRows()
# TODO - removal is O(n).
def _remove(self, key):
assert key in self._items
item_index = self._items.index(key)
self.beginRemoveRows(QModelIndex(), item_index, item_index)
del self._items[key]
self.endRemoveRows()
def _clear(self):
self.beginRemoveRows(QModelIndex(), 0, len(self._items) - 1)
self._items.clear()
self.endRemoveRows()
# O(n). Rework if it's too slow.
def _update(self, key, roles=None):
item_index = self._items.index(key)
index = self.index(item_index, 0)
if roles is None:
self.dataChanged.emit(index, index)
else:
self.dataChanged.emit(index, index, roles)
class StockPrice:
def __init__(self):
self.timestampToPrice = SortedDict()
self.pricesCount = SortedDict()
def update(self, timestamp: int, price: int) -> None:
if timestamp in self.timestampToPrice:
prevPrice = self.timestampToPrice[timestamp]
self.pricesCount[prevPrice] -= 1
if self.pricesCount[prevPrice] == 0:
del self.pricesCount[prevPrice]
self.timestampToPrice[timestamp] = price
self.pricesCount[price] = self.pricesCount.get(price, 0) + 1
def current(self) -> int:
return self.timestampToPrice.peekitem(-1)[1]
def maximum(self) -> int:
return self.pricesCount.peekitem(-1)[0]
def minimum(self) -> int:
return self.pricesCount.peekitem(0)[0]
def lastStoneWeight(self, stones):
stones_sorted_dict = SortedDict(Counter(stones))
# print(stones_sorted_dict)
while len(stones_sorted_dict) != 1:
last = None
while len(stones_sorted_dict) > 0:
q = stones_sorted_dict.popitem()
if q[1] % 2:
last = q[0]
if last:
break
if len(stones_sorted_dict) == 0:
if last is None:
return 0
else:
return last
second_last = stones_sorted_dict.peekitem()[0]
stones_sorted_dict[second_last] -= 1
if stones_sorted_dict[second_last] == 0:
stones_sorted_dict.popitem()
q = last - second_last
if q in stones_sorted_dict:
stones_sorted_dict[q] += 1
else:
stones_sorted_dict[q] = 1
# print(stones_sorted_dict)
if len(stones_sorted_dict) == 1:
if stones_sorted_dict.peekitem()[1] % 2:
return stones_sorted_dict.peekitem()[0]
else:
return 0
def extract_key_from_dict_of_dicts(dict_of_dicts, key):
ret = SortedDict()
for dk in dict_of_dicts:
ret[dk] = SortedDict()
c_dict = dict_of_dicts[dk]
c_ret = ret[dk]
if key in c_dict:
c_ret[key] = c_dict[key]
if len(ret.values()) == 1:
# only one block, then just return this one as a SortedDict and not as a SortedDict of SortedDict's
return ret.peekitem(0)[1].peekitem(0)[1]
else:
return ret
class BaseSimulator:
def __init__(self):
self.currentTime = 0.0
self.eventList = SortedDict()
def addEvent(self, time, funcName, args):
self.eventList[time] = [funcName, args]
def deleteEvents(self, timeList):
try:
for key in timeList:
self.eventList.pop(key)
except KeyError:
return
def completeRun(self):
try:
while self.eventList.peekitem(0):
self.currentTime, funcCall = self.eventList.peekitem(0)
funcCall[0](funcCall[1])
self.eventList.popitem(0)
except IndexError:
return
def run(self, timeLimit = 0):
try:
while self.eventList.peekitem(0):
currentTime, funcCall = self.eventList.peekitem(0)
if 'timeLimit' in locals():
if currentTime >= timeLimit:
return
funcCall[0](funcCall[1])
self.currentTime = currentTime
self.eventList.popitem(0)
except IndexError:
return
def getSkyline(self, buildings: List[List[int]]) -> List[List[int]]:
pts = [None] * (len(buildings) * 2)
for i, v in enumerate(buildings):
l, r, h = v
pts[i] = (l, h, 's')
pts[len(pts) - i - 1] = (r, h, 'e')
pts.sort(key=lambda x: (x[0], -x[1] if x[2] == 's' else x[1]))
sol = []
pq = SortedDict({0: 1})
for x, y, t in pts:
prev, _ = pq.peekitem(index=-1)
if t == 's':
pq[y] = pq.get(y, 0) + 1
cur, _ = pq.peekitem(index=-1)
if cur > prev:
sol.append([x, y])
else:
pq[y] = pq[y] - 1
if pq[y] == 0:
del pq[y]
cur, _ = pq.peekitem(index=-1)
if cur < prev:
sol.append([x, cur])
return sol
def solve():
n, k = map(int, raw_input().split())
cnts = SortedDict([(n, 1)])
i = 0
while i < k:
sz, cnt = cnts.peekitem()
ls = (sz-1) / 2
rs = sz / 2
if ls:
cnts[ls] = cnts.get(ls, 0) + cnt
if rs:
cnts[rs] = cnts.get(rs, 0) + cnt
del cnts[sz]
i += cnt
return "%d %d" % (rs, ls)
class StockPrice:
def __init__(self):
self.timeToPrices = SortedDict()
self.rec = SortedDict()
def update(self, timestamp: int, price: int) -> None:
if timestamp in self.timeToPrices:
prevPrice = self.timeToPrices[timestamp]
self.rec[prevPrice].remove(timestamp)
if len(self.rec[prevPrice]) == 0:
self.rec.pop(prevPrice)
if not price in self.rec:
self.rec[price] = set()
self.rec[price].add(timestamp)
self.timeToPrices[timestamp] = price
def current(self) -> int:
return self.timeToPrices.peekitem(-1)[1]
def maximum(self) -> int:
return self.rec.peekitem(-1)[0]
def minimum(self) -> int:
return self.rec.peekitem(0)[0]
def longestSubarray(self, nums: List[int], limit: int) -> int:
sd = SortedDict()
l, r, res = 0, 0, 0
while r < len(nums):
new_val = nums[r]
r += 1
if new_val not in sd:
sd[new_val] = 1
else:
sd[new_val] += 1
while abs(sd.peekitem(0)[0] - sd.peekitem(-1)[0]) > limit:
rm_val = nums[l]
l += 1
sd[rm_val] -= 1
if sd[rm_val] == 0:
del sd[rm_val]
res = max(res, r - l)
return res
class Events:
def __init__(self):
self.list_of_events = SortedDict() # for each time, a list of events
def add_event(self, event):
if not self.list_of_events.get(event.time):
self.list_of_events[event.time] = [event]
else:
self.list_of_events[event.time].append(event)
def count_arrivals_at(self, time):
try:
li = self.list_of_events[time]
n = 0
for e in li:
if li.type == 'arrival':
n += 1
return n
except:
return 0
def pop_next_event(self):
li = self.list_of_events.peekitem(0)[1]
#if debug:
# print(self.list_of_events)
if (len(li) > 1):
toret = li.pop(0)
#if debug:
# print(f'returning {toret}')
return toret
else:
lil = self.list_of_events.popitem(0)[1]
toret = lil[0]
#if debug:
# print(f'returning {toret}')
return toret
def count(self):
return len(self.list_of_events)
def __str__(self):
s = ''
for k in self.list_of_events.keys():
s += (str(k) + ": ")
for e in self.list_of_events[k]:
s += str(e) + ', '
return s
def test_peekitem2():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
temp.peekitem(index=100)
class OrderBook(WebsocketClient):
def __init__(self, product_id='BTC-USD', log_to=None):
super(OrderBook, self).__init__(products=product_id)
self._asks = SortedDict()
self._bids = SortedDict()
self._client = PublicClient()
self._sequence = -1
self._log_to = log_to
if self._log_to:
assert hasattr(self._log_to, 'write')
self._current_ticker = None
@property
def product_id(self):
''' Currently OrderBook only supports a single product even though it is stored as a list of products. '''
return self.products[0]
def on_open(self):
self._sequence = -1
print("-- Subscribed to OrderBook! --\n")
def on_close(self):
print("\n-- OrderBook Socket Closed! --")
def reset_book(self):
self._asks = SortedDict()
self._bids = SortedDict()
res = self._client.get_product_order_book(product_id=self.product_id, level=3)
for bid in res['bids']:
self.add({
'id': bid[2],
'side': 'buy',
'price': Decimal(bid[0]),
'size': Decimal(bid[1])
})
for ask in res['asks']:
self.add({
'id': ask[2],
'side': 'sell',
'price': Decimal(ask[0]),
'size': Decimal(ask[1])
})
self._sequence = res['sequence']
def on_message(self, message):
if self._log_to:
pickle.dump(message, self._log_to)
sequence = message.get('sequence', -1)
if self._sequence == -1:
self.reset_book()
return
if sequence <= self._sequence:
# ignore older messages (e.g. before order book initialization from getProductOrderBook)
return
elif sequence > self._sequence + 1:
self.on_sequence_gap(self._sequence, sequence)
return
msg_type = message['type']
if msg_type == 'open':
self.add(message)
elif msg_type == 'done' and 'price' in message:
self.remove(message)
elif msg_type == 'match':
self.match(message)
self._current_ticker = message
elif msg_type == 'change':
self.change(message)
self._sequence = sequence
def on_sequence_gap(self, gap_start, gap_end):
self.reset_book()
print('Error: messages missing ({} - {}). Re-initializing book at sequence.'.format(
gap_start, gap_end, self._sequence))
def add(self, order):
order = {
'id': order.get('order_id') or order['id'],
'side': order['side'],
'price': Decimal(order['price']),
'size': Decimal(order.get('size') or order['remaining_size'])
}
if order['side'] == 'buy':
bids = self.get_bids(order['price'])
if bids is None:
bids = [order]
else:
bids.append(order)
self.set_bids(order['price'], bids)
else:
asks = self.get_asks(order['price'])
if asks is None:
asks = [order]
else:
asks.append(order)
self.set_asks(order['price'], asks)
def remove(self, order):
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is not None:
bids = [o for o in bids if o['id'] != order['order_id']]
if len(bids) > 0:
self.set_bids(price, bids)
else:
self.remove_bids(price)
else:
asks = self.get_asks(price)
if asks is not None:
asks = [o for o in asks if o['id'] != order['order_id']]
if len(asks) > 0:
self.set_asks(price, asks)
else:
self.remove_asks(price)
def match(self, order):
size = Decimal(order['size'])
price = Decimal(order['price'])
if order['side'] == 'buy':
bids = self.get_bids(price)
if not bids:
return
assert bids[0]['id'] == order['maker_order_id']
if bids[0]['size'] == size:
self.set_bids(price, bids[1:])
else:
bids[0]['size'] -= size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if not asks:
return
assert asks[0]['id'] == order['maker_order_id']
if asks[0]['size'] == size:
self.set_asks(price, asks[1:])
else:
asks[0]['size'] -= size
self.set_asks(price, asks)
def change(self, order):
try:
new_size = Decimal(order['new_size'])
except KeyError:
return
try:
price = Decimal(order['price'])
except KeyError:
return
if order['side'] == 'buy':
bids = self.get_bids(price)
if bids is None or not any(o['id'] == order['order_id'] for o in bids):
return
index = [b['id'] for b in bids].index(order['order_id'])
bids[index]['size'] = new_size
self.set_bids(price, bids)
else:
asks = self.get_asks(price)
if asks is None or not any(o['id'] == order['order_id'] for o in asks):
return
index = [a['id'] for a in asks].index(order['order_id'])
asks[index]['size'] = new_size
self.set_asks(price, asks)
tree = self._asks if order['side'] == 'sell' else self._bids
node = tree.get(price)
if node is None or not any(o['id'] == order['order_id'] for o in node):
return
def get_current_ticker(self):
return self._current_ticker
def get_current_book(self):
result = {
'sequence': self._sequence,
'asks': [],
'bids': [],
}
for ask in self._asks:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_ask = self._asks[ask]
except KeyError:
continue
for order in this_ask:
result['asks'].append([order['price'], order['size'], order['id']])
for bid in self._bids:
try:
# There can be a race condition here, where a price point is removed
# between these two ops
this_bid = self._bids[bid]
except KeyError:
continue
for order in this_bid:
result['bids'].append([order['price'], order['size'], order['id']])
return result
def get_ask(self):
return self._asks.peekitem(0)[0]
def get_asks(self, price):
return self._asks.get(price)
def remove_asks(self, price):
del self._asks[price]
def set_asks(self, price, asks):
self._asks[price] = asks
def get_bid(self):
return self._bids.peekitem(-1)[0]
def get_bids(self, price):
return self._bids.get(price)
def remove_bids(self, price):
del self._bids[price]
def set_bids(self, price, bids):
self._bids[price] = bids
def test_peekitem2():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
with pytest.raises(IndexError):
temp.peekitem(index=100)
class Replica(HasActionQueue, MessageProcessor):
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
@property
def h(self) -> int:
return self._h
@h.setter
def h(self, n):
self._h = n
self.H = self._h + self.config.LOG_SIZE
@property
def requests(self):
return self.node.requests
def shouldParticipate(self, viewNo: int, ppSeqNo: int):
# Replica should only participating in the consensus process and the
# replica did not stash any of this request's 3-phase request
return self.node.isParticipating and (viewNo, ppSeqNo) \
not in self.stashingWhileCatchingUp
@staticmethod
def generateName(nodeName: str, instId: int):
"""
Create and return the name for a replica using its nodeName and
instanceId.
Ex: Alpha:1
"""
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".
format(self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".
format(self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicion exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicion exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo == self.viewNo
def isMsgForPrevView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
if self.isMsgForLaterView(msg):
self.discard(msg,
"Cannot get primary status for a request for a later "
"view {}. Request is {}".format(self.viewNo, msg),
logger.error)
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
logger.error("{} cannot get primary for a request for a later "
"view. Request is {}".format(self, msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
logger.debug("{} stashing request digest {} since it does not know "
"its primary status".
format(self, (rd.identifier, rd.reqId)))
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
r = self.inBoxRouter.handleAllSync(self.inBox, limit)
r += self._serviceActions()
return r
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}"
.format(self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param msg: the ThreePhaseMsg to dispatch
:param sender: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
if self.isPpSeqNoAcceptable(msg.ppSeqNo):
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
else:
logger.debug("{} stashing 3 phase message {} since ppSeqNo {} is "
"not between {} and {}".
format(self, msg, msg.ppSeqNo, self.h, self.H))
self.stashingWhileOutsideWaterMarks.append((msg, sender))
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
self.dequeuePrePrepare(rd.identifier, rd.reqId)
else:
self.doPrePrepare(rd)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug("{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".
format(self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
key = (pp.viewNo, pp.ppSeqNo)
logger.debug("{} Receiving PRE-PREPARE{} at {} from {}".
format(self, key, time.perf_counter(), sender))
if self.canProcessPrePrepare(pp, sender):
if not self.node.isParticipating:
self.stashingWhileCatchingUp.add(key)
self.addToPrePrepares(pp)
logger.info("{} processed incoming PRE-PREPARE{}".
format(self, key))
def tryPrepare(self, pp: PrePrepare):
"""
Try to send the Prepare message if the PrePrepare message is ready to
be passed into the Prepare phase.
"""
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
logger.debug("{} received PREPARE{} from {}".
format(self, (prepare.viewNo, prepare.ppSeqNo), sender))
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
logger.debug("{} processed incoming PREPARE {}".
format(self, (prepare.viewNo, prepare.ppSeqNo)))
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning("{} cannot process incoming PREPARE".
format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received COMMIT {} from {}".
format(self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
logger.debug("{} processed incoming COMMIT{}".
format(self, (commit.viewNo, commit.ppSeqNo)))
def tryCommit(self, prepare: Prepare):
"""
Try to commit if the Prepare message is ready to be passed into the
commit phase.
"""
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
"""
Try to order if the Commit message is ready to be ordered.
"""
canOrder, reason = self.canOrder(commit)
if canOrder:
logger.debug("{} returning request to node".format(self))
self.tryOrdering(commit)
else:
logger.trace("{} cannot return request to node: {}".
format(self, reason))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements identifier, reqId, and digest
"""
if not self.node.isParticipating:
logger.error("Non participating node is attempting PRE-PREPARE. "
"This should not happen.")
return
if self.lastPrePrepareSeqNo == self.H:
logger.debug("{} stashing PRE-PREPARE {} since outside greater "
"than high water mark {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo+1),
self.H))
self.stashingWhileOutsideWaterMarks.append(reqDigest)
return
self.lastPrePrepareSeqNo += 1
tm = time.time()*1000
logger.debug("{} Sending PRE-PREPARE {} at {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo),
time.perf_counter()))
prePrepareReq = PrePrepare(self.instId,
self.viewNo,
self.lastPrePrepareSeqNo,
*reqDigest,
tm)
self.sentPrePrepares[self.viewNo, self.lastPrePrepareSeqNo] = (reqDigest.key,
tm)
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE {} at {}".
format(self, (pp.viewNo, pp.ppSeqNo), time.perf_counter()))
prepare = Prepare(self.instId,
pp.viewNo,
pp.ppSeqNo,
pp.digest,
pp.ppTime)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
"""
Create a commit message from the given Prepare message and trigger the
commit phase
:param p: the prepare message
"""
logger.debug("{} Sending COMMIT{} at {}".
format(self, (p.viewNo, p.ppSeqNo), time.perf_counter()))
commit = Commit(self.instId,
p.viewNo,
p.ppSeqNo,
p.digest,
p.ppTime)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool:
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# TODO: Check whether it is rejecting PRE-PREPARE from previous view
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
# A PRE-PREPARE is sent that has already been received
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
key = (pp.identifier, pp.reqId)
if not self.requests.isFinalised(key):
self.enqueuePrePrepare(pp, sender)
return False
# A PRE-PREPARE is sent that does not match request digest
if self.requests.digest(key) != pp.digest:
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
key = (pp.viewNo, pp.ppSeqNo)
self.prePrepares[key] = \
((pp.identifier, pp.reqId), pp.ppTime)
self.dequeuePrepares(*key)
self.dequeueCommits(*key)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def hasPrepared(self, request) -> bool:
return self.prepares.hasPrepareFrom(request, self.name)
def canSendPrepare(self, request) -> bool:
"""
Return whether the request identified by (identifier, requestId) can
proceed to the Prepare step.
:param request: any object with identifier and requestId attributes
"""
return self.shouldParticipate(request.viewNo, request.ppSeqNo) \
and not self.hasPrepared(request) \
and self.requests.isFinalised((request.identifier,
request.reqId))
def isValidPrepare(self, prepare: Prepare, sender: str) -> bool:
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
return False
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT, prepare)
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def hasCommitted(self, request) -> bool:
return self.commits.hasCommitFrom(ThreePhaseKey(
request.viewNo, request.ppSeqNo), self.name)
def canCommit(self, prepare: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param prepare: the PREPARE
"""
return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \
self.prepares.hasQuorum(prepare, self.f) and \
not self.hasCommitted(prepare)
def isValidCommit(self, commit: Commit, sender: str) -> bool:
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
primaryStatus = self.isPrimaryForMsg(commit)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
key = (commit.viewNo, commit.ppSeqNo)
if key not in ppReqs:
self.enqueueCommit(commit, sender)
return False
if (key not in self.prepares and
key not in self.preparesWaitingForPrePrepare):
logger.debug("{} rejecting COMMIT{} due to lack of prepares".
format(self, key))
# raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
return False
elif self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
elif commit.digest != self.getDigestFor3PhaseKey(ThreePhaseKey(*key)):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
elif key in ppReqs and commit.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG,
commit)
else:
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def hasOrdered(self, viewNo, ppSeqNo) -> bool:
return (viewNo, ppSeqNo) in self.ordered
def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
if not self.commits.hasQuorum(commit, self.f):
return False, "no quorum: {} commits where f is {}".\
format(commit, self.f)
if self.hasOrdered(commit.viewNo, commit.ppSeqNo):
return False, "already ordered"
if not self.isNextInOrdering(commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if viewNo not in self.stashedCommitsForOrdering:
self.stashedCommitsForOrdering[viewNo] = {}
self.stashedCommitsForOrdering[viewNo][ppSeqNo] = commit
# self._schedule(self.orderStashedCommits, 2)
self.startRepeating(self.orderStashedCommits, 2)
return False, "stashing {} since out of order".\
format(commit)
return True, None
def isNextInOrdering(self, commit: Commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if self.ordered and self.ordered[-1] == (viewNo, ppSeqNo-1):
return True
for (v, p) in self.commits:
if v < viewNo:
# Have commits from previous view that are unordered.
# TODO: Question: would commits be always ordered, what if
# some are never ordered and its fine, go to PBFT.
return False
if v == viewNo and p < ppSeqNo and (v, p) not in self.ordered:
# If unordered commits are found with lower ppSeqNo then this
# cannot be ordered.
return False
# TODO: Revisit PBFT paper, how to make sure that last request of the
# last view has been ordered? Need change in `VIEW CHANGE` mechanism.
# Somehow view change needs to communicate what the last request was.
# Also what if some COMMITs were completely missed in the same view
return True
def orderStashedCommits(self):
# TODO: What if the first few commits were out of order and stashed?
# `self.ordered` would be empty
if self.ordered:
lastOrdered = self.ordered[-1]
vToRemove = set()
for v in self.stashedCommitsForOrdering:
if v < lastOrdered[0] and self.stashedCommitsForOrdering[v]:
raise RuntimeError("{} found commits from previous view {}"
" that were not ordered but last ordered"
" is {}".format(self, v, lastOrdered))
pToRemove = set()
for p, commit in self.stashedCommitsForOrdering[v].items():
if (v == lastOrdered[0] and lastOrdered == (v, p - 1)) or \
(v > lastOrdered[0] and
self.isLowestCommitInView(commit)):
logger.debug("{} ordering stashed commit {}".
format(self, commit))
if self.tryOrdering(commit):
lastOrdered = (v, p)
pToRemove.add(p)
for p in pToRemove:
del self.stashedCommitsForOrdering[v][p]
if not self.stashedCommitsForOrdering[v]:
vToRemove.add(v)
for v in vToRemove:
del self.stashedCommitsForOrdering[v]
# if self.stashedCommitsForOrdering:
# self._schedule(self.orderStashedCommits, 2)
if not self.stashedCommitsForOrdering:
self.stopRepeating(self.orderStashedCommits)
def isLowestCommitInView(self, commit):
# TODO: Assumption: This assumes that at least one commit that was sent
# for any request by any node has been received in the view of this
# commit
ppSeqNos = []
for v, p in self.commits:
if v == commit.viewNo:
ppSeqNos.append(p)
return min(ppSeqNos) == commit.ppSeqNo if ppSeqNos else True
def tryOrdering(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
logger.debug("{} trying to order COMMIT{}".format(self, key))
reqKey = self.getReqKeyFrom3PhaseKey(key) # type: Tuple
digest = self.getDigestFor3PhaseKey(key)
if not digest:
logger.error("{} did not find digest for {}, request key {}".
format(self, key, reqKey))
return
self.doOrder(*key, *reqKey, digest, commit.ppTime)
return True
def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime):
key = (viewNo, ppSeqNo)
self.addToOrdered(*key)
ordered = Ordered(self.instId,
viewNo,
identifier,
reqId,
ppTime)
# TODO: Should not order or add to checkpoint while syncing
# 3 phase state.
self.send(ordered, TPCStat.OrderSent)
if key in self.stashingWhileCatchingUp:
self.stashingWhileCatchingUp.remove(key)
logger.debug("{} ordered request {}".format(self, (viewNo, ppSeqNo)))
self.addToCheckpoint(ppSeqNo, digest)
def processCheckpoint(self, msg: Checkpoint, sender: str):
if self.checkpoints:
seqNo = msg.seqNo
_, firstChk = self.firstCheckPoint
if firstChk.isStable:
if firstChk.seqNo == seqNo:
self.discard(msg, reason="Checkpoint already stable",
logMethod=logger.debug)
return
if firstChk.seqNo > seqNo:
self.discard(msg, reason="Higher stable checkpoint present",
logMethod=logger.debug)
return
for state in self.checkpoints.values():
if state.seqNo == seqNo:
if state.digest == msg.digest:
state.receivedDigests[sender] = msg.digest
break
else:
logger.error("{} received an incorrect digest {} for "
"checkpoint {} from {}".format(self,
msg.digest,
seqNo,
sender))
return
if len(state.receivedDigests) == 2*self.f:
self.markCheckPointStable(msg.seqNo)
else:
self.discard(msg, reason="No checkpoints present to tally",
logMethod=logger.warn)
def _newCheckpointState(self, ppSeqNo, digest) -> CheckpointState:
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ - 1
logger.debug("{} adding new checkpoint state for {}".
format(self, (s, e)))
state = CheckpointState(ppSeqNo, [digest, ], None, {}, False)
self.checkpoints[s, e] = state
return state
def addToCheckpoint(self, ppSeqNo, digest):
for (s, e) in self.checkpoints.keys():
if s <= ppSeqNo <= e:
state = self.checkpoints[s, e] # type: CheckpointState
state.digests.append(digest)
state = updateNamedTuple(state, seqNo=ppSeqNo)
self.checkpoints[s, e] = state
break
else:
state = self._newCheckpointState(ppSeqNo, digest)
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ
if len(state.digests) == self.config.CHK_FREQ:
state = updateNamedTuple(state, digest=serialize(state.digests),
digests=[])
self.checkpoints[s, e] = state
self.send(Checkpoint(self.instId, self.viewNo, ppSeqNo,
state.digest))
def markCheckPointStable(self, seqNo):
previousCheckpoints = []
for (s, e), state in self.checkpoints.items():
if e == seqNo:
state = updateNamedTuple(state, isStable=True)
self.checkpoints[s, e] = state
break
else:
previousCheckpoints.append((s, e))
else:
logger.error("{} could not find {} in checkpoints".
format(self, seqNo))
return
self.h = seqNo
for k in previousCheckpoints:
logger.debug("{} removing previous checkpoint {}".format(self, k))
self.checkpoints.pop(k)
self.gc(seqNo)
logger.debug("{} marked stable checkpoint {}".format(self, (s, e)))
self.processStashedMsgsForNewWaterMarks()
def gc(self, tillSeqNo):
logger.debug("{} cleaning up till {}".format(self, tillSeqNo))
tpcKeys = set()
reqKeys = set()
for (v, p), (reqKey, _) in self.sentPrePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
for (v, p), (reqKey, _) in self.prePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
logger.debug("{} found {} 3 phase keys to clean".
format(self, len(tpcKeys)))
logger.debug("{} found {} request keys to clean".
format(self, len(reqKeys)))
for k in tpcKeys:
self.sentPrePrepares.pop(k, None)
self.prePrepares.pop(k, None)
self.prepares.pop(k, None)
self.commits.pop(k, None)
if k in self.ordered:
self.ordered.remove(k)
for k in reqKeys:
self.requests.pop(k, None)
def processStashedMsgsForNewWaterMarks(self):
while self.stashingWhileOutsideWaterMarks:
item = self.stashingWhileOutsideWaterMarks.pop()
logger.debug("{} processing stashed item {} after new stable "
"checkpoint".format(self, item))
if isinstance(item, ReqDigest):
self.doPrePrepare(item)
elif isinstance(item, tuple) and len(tuple) == 2:
self.dispatchThreePhaseMsg(*item)
else:
logger.error("{} cannot process {} "
"from stashingWhileOutsideWaterMarks".
format(self, item))
@property
def firstCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(0)
@property
def lastCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(-1)
def isPpSeqNoAcceptable(self, ppSeqNo: int):
return self.h < ppSeqNo <= self.H
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrePrepare(self, request: PrePrepare, sender: str):
logger.debug("Queueing pre-prepares due to unavailability of finalised "
"Request. Request {} from {}".format(request, sender))
key = (request.identifier, request.reqId)
if key not in self.prePreparesPendingReqDigest:
self.prePreparesPendingReqDigest[key] = []
self.prePreparesPendingReqDigest[key].append((request, sender))
def dequeuePrePrepare(self, identifier: int, reqId: int):
key = (identifier, reqId)
if key in self.prePreparesPendingReqDigest:
pps = self.prePreparesPendingReqDigest[key]
for (pp, sender) in pps:
logger.debug("{} popping stashed PRE-PREPARE{}".
format(self, key))
if pp.digest == self.requests.digest(key):
self.prePreparesPendingReqDigest.pop(key)
self.processPrePrepare(pp, sender)
logger.debug(
"{} processed {} PRE-PREPAREs waiting for finalised "
"request for identifier {} and reqId {}".
format(self, pp, identifier, reqId))
break
def enqueuePrepare(self, request: Prepare, sender: str):
logger.debug("Queueing prepares due to unavailability of PRE-PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
logger.debug("{} popping stashed PREPARE{}".format(self, key))
self.processPrepare(prepare, sender)
i += 1
self.preparesWaitingForPrePrepare.pop(key)
logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def enqueueCommit(self, request: Commit, sender: str):
logger.debug("Queueing commit due to unavailability of PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.commitsWaitingForPrepare:
self.commitsWaitingForPrepare[key] = deque()
self.commitsWaitingForPrepare[key].append((request, sender))
def dequeueCommits(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.commitsWaitingForPrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.commitsWaitingForPrepare[key]:
commit, sender = self.commitsWaitingForPrepare[
key].popleft()
logger.debug("{} popping stashed COMMIT{}".format(self, key))
self.processCommit(commit, sender)
i += 1
self.commitsWaitingForPrepare.pop(key)
logger.debug("{} processed {} COMMITs waiting for PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def getDigestFor3PhaseKey(self, key: ThreePhaseKey) -> Optional[str]:
reqKey = self.getReqKeyFrom3PhaseKey(key)
digest = self.requests.digest(reqKey)
if not digest:
logger.debug("{} could not find digest in sent or received "
"PRE-PREPAREs or PREPAREs for 3 phase key {} and req "
"key {}".format(self, key, reqKey))
return None
else:
return digest
def getReqKeyFrom3PhaseKey(self, key: ThreePhaseKey):
reqKey = None
if key in self.sentPrePrepares:
reqKey = self.sentPrePrepares[key][0]
elif key in self.prePrepares:
reqKey = self.prePrepares[key][0]
elif key in self.prepares:
reqKey = self.prepares[key][0]
else:
logger.debug("Could not find request key for 3 phase key {}".
format(key))
return reqKey
@property
def threePhaseState(self):
# TODO: This method is incomplete
# Gets the current stable and unstable checkpoints and creates digest
# of unstable checkpoints
if self.checkpoints:
pass
else:
state = []
return ThreePCState(self.instId, state)
def process3PhaseState(self, msg: ThreePCState, sender: str):
# TODO: This is not complete
pass
def send(self, msg, stat=None) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.display("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
if stat:
self.stats.inc(stat)
self.outBox.append(msg)
def test_peekitem():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert temp.peekitem() == ('z', 25)
assert temp.peekitem(0) == ('a', 0)
assert temp.peekitem(index=4) == ('e', 4)
