class StockPrice(object):
def __init__(self):
self.timeToPrice = SortedDict() #time will be sorted
self.priceToTime = SortedDict() #the price will be sorted
def update(self, timestamp, price):
if timestamp in self.timeToPrice:
prevPrice = self.timeToPrice[timestamp]
self.priceToTime[prevPrice].remove(timestamp)
if len(self.priceToTime[prevPrice])==0: self.priceToTime.pop(prevPrice) #[0]
if price not in self.priceToTime: self.priceToTime[price] = set() #initialized
self.priceToTime[price].add(timestamp)
self.timeToPrice[timestamp] = price
def current(self):
return self.timeToPrice.peekitem(-1)[1]
def maximum(self):
return self.priceToTime.peekitem(-1)[0]
def minimum(self):
return self.priceToTime.peekitem(0)[0]
class SummaryRanges:
def __init__(self):
self.mp = SortedDict()
def addNum(self, val: int) -> None:
n = len(self.mp)
ridx = self.mp.bisect_right(val)
lidx = n if ridx == 0 else ridx - 1
keys = self.mp.keys()
values = self.mp.values()
if lidx != n and ridx != n and values[lidx][1] + 1 == val and values[ridx][0] - 1 == val:
self.mp[keys[lidx]][1] = self.mp[keys[ridx]][1]
self.mp.pop(keys[ridx])
elif lidx != n and val <= values[lidx][1] + 1:
self.mp[keys[lidx]][1] = max(val, self.mp[keys[lidx]][1])
elif ridx != n and val >= values[ridx][0] - 1:
self.mp[keys[ridx]][0] = min(val, self.mp[keys[ridx]][0])
else:
self.mp[val] = [val, val]
def getIntervals(self) -> List[List[int]]:
return list(self.mp.values())
# # Your SummaryRanges object will be instantiated and called as such:
# # obj = SummaryRanges()
# # obj.addNum(val)
# # param_2 = obj.getIntervals()
class GazeItem:
def __init__(self, label, dtype):
self.__label__ = label
self.__data__ = SortedDict({})
self.__dtype__ = dtype
def __getitem__(self, key):
return self.__data__[key]
def __setitem__(self, key, value):
self.__data__[key] = value
def getData(self):
return self.__data__
def getLabel(self):
return self.__label__
def getType(self):
return self.__dtype__
def keys(self):
return list(self.__data__.keys())
def pop(self, key):
self.__data__.pop(key)
def values(self):
return list(self.__data__.values())
def combine(n, q, x_iter, y_iter, z_iter):
d = SortedDict()
for _ in range(n):
xi, yi = next(x_iter), next(y_iter)
li = min(xi, yi) + 1
ri = max(xi, yi) + 1
d[ri] = d.get(ri, 0) + 1
d[li - 1] = d.get(li - 1, 0) - 1
for k in list(d.keys()):
if d[k] == 0:
d.pop(k, None)
acc = 0
for k in reversed(d):
d[k] += acc
acc = d[k]
keys = list(reversed(d.keys()))
ranks = []
acc = 1
for i, k in enumerate(keys):
start = keys[i]
dups = d[start]
ranks.append(Interval(acc, start, dups))
if i + 1 < len(keys):
count = start - keys[i + 1]
acc += count * dups
result = 0
for i in range(1, q + 1):
zi = next(z_iter)
ki = zi + 1
result += i * pick_score(ki, ranks)
return result
class MyCalendarTwo:
# double list
def __init__(self):
self.calendar = []
self.overlaps = []
def book(self, start: int, end: int) -> bool:
for s, e in self.overlaps:
if start < e and end > s:
return False
for s, e in self.calendar:
if start < e and end > s:
self.overlaps.append((max(s, start), min(end, e)))
self.calendar.append((start, end))
return True
# use boundary count
def __init__(self):
self.calendar = SortedDict()
def book(self, start: int, end: int) -> bool:
self.calendar[start] = self.calendar.get(start, 0) + 1
self.calendar[end] = self.calendar.get(end, 0) - 1
active = 0
for v in self.calendar.values():
active += v
if active >= 3:
self.calendar[start] = self.calendar.pop(start, 0) - 1
self.calendar[end] = self.calendar.pop(end, 0) + 1
if self.calendar[start] == 0:
del self.calendar[start]
return False
return True
class StockPrice: # 45%, 12%
def __init__(self):
# 两个TreeMap
self.timeToPrice = SortedDict() # 时间->价格，key已排序
self.priceToTimes = SortedDict() # 价格->Set<该价格的时间>，key已排序
def update(self, timestamp: int, price: int) -> None:
if timestamp in self.timeToPrice: # 时间已存在，是更新价格
oldPrice = self.timeToPrice[timestamp]
self.priceToTimes[oldPrice].remove(timestamp) # 去掉老的价格中的时间，因为已失效
if not self.priceToTimes[oldPrice]: # 如果老价格不再对应任何时间，去除key
self.priceToTimes.pop(oldPrice)
self.timeToPrice[timestamp] = price # 更新/新建 时间->价格
if not price in self.priceToTimes: # 更新/新建 价格->时间们
self.priceToTimes[price] = set()
self.priceToTimes[price].add(timestamp)
def current(self) -> int:
return self.timeToPrice.peekitem(-1)[1] # 找最大的时间key，得到其对应价格
def maximum(self) -> int:
return self.priceToTimes.peekitem(-1)[0] # 找最大的价格key，得到key
def minimum(self) -> int:
return self.priceToTimes.peekitem(0)[0] # 找最小的价格key，得到key
class SummaryRanges:
def __init__(self):
self.mp = SortedDict()
def addNum(self, val: int) -> None:
n = len(self.mp)
ridx = self.mp.bisect_right(val)
lidx = n if ridx == 0 else ridx - 1
keys = self.mp.keys()
values = self.mp.values()
if (
lidx != n
and ridx != n
and values[lidx][1] + 1 == val
and values[ridx][0] - 1 == val
):
self.mp[keys[lidx]][1] = self.mp[keys[ridx]][1]
self.mp.pop(keys[ridx])
elif lidx != n and val <= values[lidx][1] + 1:
self.mp[keys[lidx]][1] = max(val, self.mp[keys[lidx]][1])
elif ridx != n and val >= values[ridx][0] - 1:
self.mp[keys[ridx]][0] = min(val, self.mp[keys[ridx]][0])
else:
self.mp[val] = [val, val]
def getIntervals(self) -> List[List[int]]:
return list(self.mp.values())
def diagonalSort(self, mat: List[List[int]]) -> List[List[int]]:
n = len(mat)
m = len(mat[0])
result = [[0] * m for i in range(n)]
d = n + m + 1
i0 = n - 1
j0 = 0
for i in range(d):
counts = SortedDict()
i1 = i0
j1 = j0
while i1 < n and j1 < m:
v = mat[i1][j1]
counts[v] = (counts.get(v) or 0) + 1
i1 += 1
j1 += 1
i1 = i0
j1 = j0
k = 0
while i1 < n and j1 < m:
k = counts.keys()[0]
result[i1][j1] = k
counts[k] -= 1
if not counts[k]:
counts.pop(k)
i1 += 1
j1 += 1
if i0 == 0:
j0 += 1
else:
i0 -= 1
return result
class BisectNodePolicy(BaseNodePolicy):
def __init__(self, hash_class=defaultHashClass):
self.ring = SortedDict()
super(BisectNodePolicy, self).__init__(hash_class=hash_class)
def add_node(self, node=None, vnode_count=None):
for i in range_(int(vnode_count)):
self.ring[self._gen_key(node, i)] = node
def remove_node(self, node=None):
keys = list(self.ring.keys())
for key in keys:
if self.ring[key] == node:
self.ring.pop(key)
def get_proper_node(self, key):
key, _ = self.ring.peekitem(self._find_proper_pos(key))
return self.ring[key]
def _find_proper_pos(self, key):
key = self._gen_key(key)
pos = self.ring.bisect(key)
# if object_hash == node_hash, return node index
if key in self.ring:
return pos - 1
# embodies the concept of the ring.
if pos == len(self.ring):
return 0
return pos
def medianSlidingWindow(self, nums: List[int], k: int) -> List[float]:
ans = []
d = SortedDict()
for x in range(k):
if nums[x] in d:
d[nums[x]] += 1
else:
d[nums[x]] = 1
#print(d)
ans.append(self.find_median(d, k))
for x in range(1, len(nums) - k + 1):
#print(nums[x-1],"an")
d[nums[x - 1]] -= 1
if d[nums[x - 1]] == 0:
d.pop(nums[x - 1])
if nums[x + k - 1] in d:
d[nums[x + k - 1]] += 1
else:
d[nums[x + k - 1]] = 1
ans.append(self.find_median(d, k))
return ans
class StockPrice0:
## not using sorteddict for time_to_prices
def __init__(self):
self.time_to_prices = {}
self.rec = SortedDict()
self.cur_time = -1
def update(self, timestamp: int, price: int) -> None:
self.cur_time = max(self.cur_time, timestamp)
if timestamp in self.time_to_prices:
prev_price = self.time_to_prices[timestamp]
self.rec[prev_price].remove(timestamp)
if len(self.rec[prev_price]) == 0:
self.rec.pop(prev_price)
if not price in self.rec:
self.rec[price] = set()
self.rec[price].add(timestamp)
self.time_to_prices[timestamp] = price
def current(self) -> int:
# return self.time_to_prices.peekitem(-1)[1]
return self.time_to_prices[self.cur_time]
def maximum(self) -> int:
return self.rec.peekitem(-1)[0]
def minimum(self) -> int:
return self.rec.peekitem(0)[0]
def pop_max(spaces: SortedDict):
max_n = spaces.iloc[-1]
max_n_count = spaces[max_n] - 1
if max_n_count == 0:
spaces.pop(max_n)
else:
spaces[max_n] -= 1
return max_n
def top_view(self):
sorted_endpoints = []
for segment_ in self.segments:
sorted_endpoints.append(Endpoint(True, segment_))
sorted_endpoints.append(Endpoint(False, segment_))
sorted_endpoints.sort(
key=lambda end_point: end_point.value()) # O n(log n)
resulted_segments = []
# sweeper = SortedDict(key=lambda ep: ep.height)
sweeper = SortedDict()
print(sweeper)
for endpoint_ in sorted_endpoints:
if sweeper.__len__() == 0:
sweeper[endpoint_] = endpoint_.segment
elif not endpoint_.is_left: # right of some segment
left_ep = Endpoint(True, endpoint_.segment)
if left_ep.__eq__(
sweeper.peekitem(0)[0]): # "top most" is ending now
sweeper.pop(sweeper.peekitem(0)[0])
prev_seg = resulted_segments[resulted_segments.__len__() -
1]
resulted_segments.append(
Segment(prev_seg.right, endpoint_.segment.right,
endpoint_.segment.height,
endpoint_.segment.name))
else: # some segment "not at top" needs to end now
sweeper.pop(left_ep)
elif sweeper.peekitem(
0
)[0].segment.height < endpoint_.segment.height: # crossover, a higher segment found
current_top = sweeper.peekitem(0)[0]
if resulted_segments.__len__() > 0:
last_segment = resulted_segments[
resulted_segments.__len__() - 1]
resulted_segments.append(
Segment(last_segment.right, endpoint_.segment.left,
current_top.segment.height,
current_top.segment.name))
else:
resulted_segments.append(
Segment(current_top.segment.left,
endpoint_.segment.left,
current_top.segment.height,
current_top.segment.name))
sweeper[endpoint_] = endpoint_.segment
elif sweeper.peekitem(
0
)[0].segment.height > endpoint_.segment.height: # non contributing end found
sweeper[endpoint_] = endpoint_.segment
return resulted_segments
class SortedSlidingDic:
def __init__(self, stime):
self.flow_dic = {}
self.ts_dic = SortedDict()
self.stime = float(self.unified_ts(stime))
def unified_ts(self, ts):
return round(ts, 10)
def update(self, ts):
ts = self.unified_ts(ts)
while len(self.ts_dic) > 0 and ts - self.ts_dic.peekitem(0)[0] > self.stime:
del self.flow_dic[self.ts_dic.peekitem(0)[1]]
self.ts_dic.popitem(0)
def add(self, flow, ts):
ts = self.unified_ts(ts)
# Remove the previous timestamp for this flow and the new one instead
self.remove(flow)
self.flow_dic[flow] = ts
if ts in self.ts_dic:
del self.flow_dic[self.ts_dic[ts]]
self.ts_dic[ts] = flow
assert len(self.flow_dic) == len(self.ts_dic)
def remove(self, flow):
if flow in self.flow_dic:
ts = self.flow_dic[flow]
try:
del self.flow_dic[flow]
self.ts_dic.pop(ts)
except KeyError:
print 'KeyError ', flow, ts
def __str__(self):
res = ''
for k in self.ts_dic:
res += str("%.10f" % k)+'\t'+str(self.ts_dic[k])+'\n'
print res
class Group:
def __init__(self, name, preferences, quota):
'''
@preferences: IE (2, 0, 1). Tuple of group's preferences over applicants with preferences[0] being the most preferred applicant.
@rankings: IE {0:1, 2:0, 1:2}. Dictionary of rank of each applicant.
'''
# self.preferences = preferences
self.applicantToRank = makeRankings(preferences) # ["andrew", "will"]
self.name = name
self.quota = quota
self.waitList = SortedDict() # rank --> applicant
def addToWaitList(self, applicant):
rank = self.applicantToRank[applicant]
self.waitList[rank] = applicant
def acceptQuota(self):
accepted = self.waitList.islice(0, self.quota)
acceptedApplicants = [self.waitList[rank] for rank in accepted]
rejected = self.waitList.islice(self.quota)
rejectedApplicants = [self.waitList.pop(rank) for rank in rejected]
return acceptedApplicants, rejectedApplicants
class TimerScheduler:
def __init__(self):
self._epoch = datetime.now()
self._timers_by_expire_time = SortedDict()
def now(self):
absolute_now = datetime.now()
time_since_epoch = absolute_now - self._epoch
return time_since_epoch.total_seconds()
def schedule(self, timer):
expire_time = timer.expire_time()
assert expire_time is not None
if expire_time in self._timers_by_expire_time:
self._timers_by_expire_time[expire_time].append(timer)
else:
self._timers_by_expire_time[expire_time] = [timer]
def unschedule(self, timer):
expire_time = timer.expire_time()
assert expire_time is not None
assert expire_time in self._timers_by_expire_time
timers_with_matching_expire = self._timers_by_expire_time[expire_time]
assert timer in timers_with_matching_expire
timers_with_matching_expire.remove(timer)
if timers_with_matching_expire == []:
self._timers_by_expire_time.pop(expire_time)
def trigger_all_expired_timers(self):
# Trigger all expired timers and return time until next expire
now = self.now()
while True:
if not self._timers_by_expire_time:
return None
next_expire_time = self._timers_by_expire_time.peekitem(0)[0]
if next_expire_time > now:
return next_expire_time - now
expired_timers = self._timers_by_expire_time.popitem(0)[1]
for timer in expired_timers:
timer.trigger_expire()
def stop_all_timers(self):
while self._timers_by_expire_time:
timers = self._timers_by_expire_time.peekitem(0)[1]
for timer in timers:
timer.stop()
class EventQueue:
def __init__(self):
self.sortedEvents = SortedDict()
self.eventTimes = {}
def pollEvent(self):
return self.sortedEvents.popitem(index=0)
def peekEvent(self):
return self.sortedEvents.peekitem(index=0)
def remove(self, event):
time = self.eventTimes.get(event)
if time != None:
if time in self.sortedEvents:
self.sortedEvents.pop(time)
if event in self.eventTimes:
self.eventTimes.pop(event)
def clear(self):
self.sortedEvents.clear()
self.eventTimes.clear()
def add(self, event, time):
if np.isinf(time):
return None
if self.containsTime(time):
raise ValueError(
"EventQueue does not support two events at the same time" +
" " + str(time))
if isinstance(time, np.ndarray):
time = np.asarray(time, dtype=np.float)[0]
self.sortedEvents[time] = event
self.eventTimes[event] = time
def containsTime(self, time):
keys = np.copy(self.sortedEvents.keys())
keys = np.array(keys)
result = time in keys
return result
def peekTime(self):
return self.sortedEvents.keys()[0]
class SortedDictBook:
def __init__(self, depth):
self.depth = depth
self.orders = SortedDict()
self.empty_price = PriceUpdate()
def update(self, price_update: PriceUpdate) -> bool:
action = {
48: self.new_order,
49: self.update_order,
50: self.delete_order,
51: self.delete_thru,
52: self.delete_from
}.get(price_update.action, None)
if action:
action(price_update)
return True
return False
def update_order(self, price_update: PriceUpdate):
self.orders[price_update.price] = price_update
def delete_order(self, price_update: PriceUpdate):
self.orders.pop(price_update.price)
def delete_thru(self, price_update: PriceUpdate):
self.orders.clear()
def new_order(self, price_update: PriceUpdate):
if len(self.orders) == self.depth:
self.orders.popitem()
self.orders[price_update.price] = price_update
def get_book(self) -> List[PriceUpdate]:
return self.orders.values()
def delete_from(self, price_update: PriceUpdate):
direction = price_update.level
del self.orders.iloc[:direction]
def top(self) -> PriceUpdate:
return self.orders.peekitem(0)[1] if self.orders else self.empty_price
def isNStraightHand(self, hand: List[int], groupSize: int) -> bool:
if len(hand) % groupSize != 0:
return False
sd = SortedDict()
for h in hand:
if h in sd:
sd[h] += 1
else:
sd[h] = 1
while sd:
v = sd.peekitem(0)[0]
for i in range(v, v + groupSize):
if i not in sd:
return False
if sd[i] == 1:
sd.pop(i)
else:
sd[i] -= 1
return True
def isPossibleDivide(self, nums: List[int], k: int) -> bool:
if len(nums) % k != 0:
return False
sd = SortedDict()
for h in nums:
if h in sd:
sd[h] += 1
else:
sd[h] = 1
while sd:
v = sd.peekitem(0)[0]
for i in range(v, v + k):
if i not in sd:
return False
if sd[i] == 1:
sd.pop(i)
else:
sd[i] -= 1
return True
def simulate(arrival_rate,exercise_duration,exercise_number,alpha,graph,start_node,duration,mean_fitness,std_fitness):
arrival_duration = 1/arrival_rate
geom_p = 1/(exercise_number+1)
queue = 0
available_benches = set([node for node in graph.nodes if node[0]=='B'])
occupied_benches = dict([])
available_weights = SortedDict(zip(map(lambda x:x*5,range(len(graph.nodes))),filter(lambda x:x[0]=='S',graph.nodes)))
events=[]
time=np.random.exponential(arrival_duration)
event='arrival'
requests=set([])
schedule_items=0
cur_node = start_node
shortest_paths = nx.all_pairs_dijkstra(graph)
while time < duration:
event_type=event[0]
if event_type == 'arrival':
if len(available_benches) == 0:
queue+=1
else:
fitness=np.random.normal(mean_fitness,std_fitness)
bench=available_benches.pop()
occupied_benches[bench]=fitness
max_weight=np.random.uniform(high=2.0)*fitness
weight=next(available_weights.irange(max_weight,reverse=true)))
requests.add(Request(time,'delivery',weight,available_weights.pop(weight),bench))
next_arrival=np.random.exponential(arrival_duration)
elif event_type == 'departure':
if queue > 0 :
queue-=1
event=('arrival')
break
elif event_type == 'delivery_request':
bench=event[1]
fitness=occupied_benches[bench]
max_weight=np.random.uniform(high=2.0)*fitness
weight=next(available_weights.irange(max_weight,reverse=true)))
requests.add(Request(time,'delivery',weight,available_weights.pop(weight),bench))
if schedule_items==0:
r.solve_schedule(
elif event_type == 'delivery_completed':
(time,event)=events.pop()
class TimerManager():
"""TimerManager
"""
def __init__(self):
self.__max_timers = 16 * 1000000
self.__timers = SortedDict()
def setup_timer(self, seconds_from_now, timer_data):
"""setup_timer\n
param timer_data = ("app-spesific-timer-type", "app-spesific-id")\n
return status, (timer-expiry-ts <float>, timer-unique-id <int>)\n
"""
if len(self.__timers) > self.__max_timers:
return TR_TOOMANY, ()
unique_time = UniqueTime(seconds_from_now=seconds_from_now)
self.__timers[unique_time] = timer_data
return TR_OK, (unique_time.get_timestamp(),
unique_time.get_unique_id())
def cancel_timer(self, timer):
"""cancel_timer\n
param timer = (ts <float>, unique-id <int>)\n
"""
cancelled_timer = UniqueTime(ts=timer[0], uid=timer[1])
try:
del self.__timers[cancelled_timer]
except KeyError:
print("cancel_timer failed, ts={}, uid={}, cancelled_timer={}"\
.format(timer[0], timer[1], cancelled_timer))
return TR_NOTFOUND
return TR_OK
def poll_timer(self):
"""poll_timer
"""
now = UniqueTime()
# print("poll_timer, now: {}".format(now))
if not self.__timers:
return TR_NOTFOUND, ()
first_timer = next(iter(self.__timers))
if first_timer < now:
try:
timer_data = self.__timers.pop(first_timer)
except KeyError:
return TR_NOTFOUND, ()
return TR_OK, timer_data
return TR_NOTFOUND, ()
def get_timers(self):
"""get_timers
"""
return self.__timers
class PointerQueue(object):
"""
Implementation of linked list style queue
"""
def __init__(self, initial, reserve=0):
# Sorted dict has O(log(n)) runtime for pop, set, and delete operations
# May implement faster cython DS in future
self.queue = SortedDict(zip(initial, range(len(initial))))
# Key for looking up queues from specific time steps
self.pointer = initial
self.pointer.extend([None] * reserve)
def __len__(self):
return len(self.queue)
def __iter__(self):
return iter(self.queue)
def __list__(self):
return self.pointer
def __setitem__(self, key, value):
self.queue[key] = value
self.pointer[value] = key
def __next__(self):
try:
return self.queue.popitem()
except AssertionError:
raise StopIteration
def __nonzero__(self):
return self.queue
def pop(self, key, **kwargs):
return self.queue.pop(key, **kwargs)
def popindex(self, index, **kwargs):
return self.queue.pop(self.pointer[index], **kwargs)
class RangeModule2:
def __init__(self):
self.track = SortedDict()
def __str__(self):
return str(self.track)
def find(self, left, right):
sl = SortedList(self.track.keys())
l = sl.bisect_left(left)
r = sl.bisect_left(right)
if l > 0:
l -= 1
if self.track[sl[l]] < left:
l += 1
if l == r:
return (left, right)
else:
i = min(sl[l], left)
if r == len(sl): r -= 1
j = max(self.track[sl[r]], right)
for it in list(sl.irange(sl[l], sl[r])):
self.track.pop(it)
return (i, j)
def addRange(self, left, right):
i, j = self.find(left, right)
self.track[i] = j
def removeRange(self, left, right):
i, j = self.find(left, right)
if left > i: self.track[i] = left
if right < j: self.track[right] = j
def queryRange(self, left, right):
sl = SortedList(self.track.keys())
l = sl.bisect_right(left)
print('l = ', l)
return self.track[sl[l - 1]] >= right
def containsNearbyAlmostDuplicateOrderMap(self, nums, k, t):
"""
:type nums: List[int]
:type k: int
:type t: int
:rtype: bool
"""
m = SortedDict()
j = 0
for i in range(len(nums)):
# only need to consider index range [i-k, i]
if i - j > k:
m.pop(nums[j])
j += 1
# abs(nums[i]-nums[j]) <= t --> nums[i] - t < nums[j]
# if nums[j] < nums[i]-t, abs(nums[i]-nums[j]) <= t won't hold
# search for an index where nums[a] first >= nums[i]-t
a = m.bisect_left(nums[i] - t)
keys = m.keys()
if a < len(m) and abs(keys[a] - nums[i]) <= t:
return True
m[nums[i]] = i
return False
def get_actual_burndown(self):
"""
Get the actual burndown curve dates and task times.
This method creates and returns a SortedDict with remaining task times
for each date. The key is the ordinal date and the value is the sum of
the remaining time for tasks due on that date. The remaining times are
calculated using the average expected time to complete the task.
:return: _time_remaining; dictionary of remaining program time by date.
:rtype: dict
"""
_time_remaining = {}
_dates = SortedDict(self.status_tree.nodes).keys()
_dates.pop(0)
for _key in _dates:
_entity = self.status_tree.get_node(_key).data
_time_remaining[date_to_ordinal(
_entity.date_status)] = _entity.time_remaining
return _time_remaining
class BaseSimulator:
def __init__(self):
self.currentTime = 0.0
self.eventList = SortedDict()
def addEvent(self, time, funcName, args):
self.eventList[time] = [funcName, args]
def deleteEvents(self, timeList):
try:
for key in timeList:
self.eventList.pop(key)
except KeyError:
return
def completeRun(self):
try:
while self.eventList.peekitem(0):
self.currentTime, funcCall = self.eventList.peekitem(0)
funcCall[0](funcCall[1])
self.eventList.popitem(0)
except IndexError:
return
def run(self, timeLimit = 0):
try:
while self.eventList.peekitem(0):
currentTime, funcCall = self.eventList.peekitem(0)
if 'timeLimit' in locals():
if currentTime >= timeLimit:
return
funcCall[0](funcCall[1])
self.currentTime = currentTime
self.eventList.popitem(0)
except IndexError:
return
def fallingSquares(self, positions):
"""
:type positions: List[List[int]]
:rtype: List[int]
"""
res = []
sd = SortedDict()
curMax = 0
for pos in positions:
start, end, h = pos[0], pos[0] + pos[1], 0
t = []
keys = sd.keys()
index = sd.bisect_left((start, start))
if index > 0:
index -= 1
if keys[index][1] <= start:
index += 1
delete = []
while index < len(keys) and keys[index][0] < end:
if (start > keys[index][0]):
t.append((keys[index][0], start, sd[keys[index]]))
if (end < keys[index][1]):
t.append((end, keys[index][1], sd[keys[index]]))
h = max(h, sd[keys[index]])
delete.append(keys[index])
index += 1
for d in delete:
sd.pop(d)
sd[(start, end)] = h + pos[1]
for a in t:
sd[(a[0], a[1])] = a[2]
curMax = max(curMax, h + pos[1])
res.append(curMax)
return res
class MaxStack:
def __init__(self):
"""
initialize your data structure here.
"""
self.stack_top = DLL()
self.sorted_stack = SortedDict() ## key: val; val: list of DLL nodes
def push(self, x: int) -> None:
previous = self.stack_top.tail.prev
previous.next = DoublyLinkedNode(val=-x)
previous.next.prev = previous
previous.next.next = self.stack_top.tail
self.stack_top.tail.prev = previous.next
if -x in self.sorted_stack:
self.sorted_stack[-x].append(self.stack_top.tail.prev)
else:
self.sorted_stack[-x] = [self.stack_top.tail.prev]
def pop(self) -> int:
res = self.stack_top.tail.prev.val
popped_node = self.sorted_stack[res].pop()
if not len(self.sorted_stack[res]):
self.sorted_stack.pop(res)
previous, next_one = popped_node.prev, popped_node.next
previous.next = next_one
next_one.prev = previous
return -res
def top(self) -> int:
# print('top', 'sorted_stack', [{-i:len(self.sorted_stack[i])} for i in self.sorted_stack.keys()])
# print(self.stack_top.val)
return -self.stack_top.tail.prev.val
class StockPrice:
def __init__(self):
self.timeToPrices = SortedDict()
self.rec = SortedDict()
def update(self, timestamp: int, price: int) -> None:
if timestamp in self.timeToPrices:
prevPrice = self.timeToPrices[timestamp]
self.rec[prevPrice].remove(timestamp)
if len(self.rec[prevPrice]) == 0:
self.rec.pop(prevPrice)
if not price in self.rec:
self.rec[price] = set()
self.rec[price].add(timestamp)
self.timeToPrices[timestamp] = price
def current(self) -> int:
return self.timeToPrices.peekitem(-1)[1]
def maximum(self) -> int:
return self.rec.peekitem(-1)[0]
def minimum(self) -> int:
return self.rec.peekitem(0)[0]
class Replica(HasActionQueue, MessageProcessor):
def __init__(self, node: 'plenum.server.node.Node', instId: int,
isMaster: bool = False):
"""
Create a new replica.
:param node: Node on which this replica is located
:param instId: the id of the protocol instance the replica belongs to
:param isMaster: is this a replica of the master protocol instance
"""
super().__init__()
self.stats = Stats(TPCStat)
self.config = getConfig()
routerArgs = [(ReqDigest, self._preProcessReqDigest)]
for r in [PrePrepare, Prepare, Commit]:
routerArgs.append((r, self.processThreePhaseMsg))
routerArgs.append((Checkpoint, self.processCheckpoint))
routerArgs.append((ThreePCState, self.process3PhaseState))
self.inBoxRouter = Router(*routerArgs)
self.threePhaseRouter = Router(
(PrePrepare, self.processPrePrepare),
(Prepare, self.processPrepare),
(Commit, self.processCommit)
)
self.node = node
self.instId = instId
self.name = self.generateName(node.name, self.instId)
self.outBox = deque()
"""
This queue is used by the replica to send messages to its node. Replica
puts messages that are consumed by its node
"""
self.inBox = deque()
"""
This queue is used by the replica to receive messages from its node.
Node puts messages that are consumed by the replica
"""
self.inBoxStash = deque()
"""
If messages need to go back on the queue, they go here temporarily and
are put back on the queue on a state change
"""
self.isMaster = isMaster
# Indicates name of the primary replica of this protocol instance.
# None in case the replica does not know who the primary of the
# instance is
self._primaryName = None # type: Optional[str]
# Requests waiting to be processed once the replica is able to decide
# whether it is primary or not
self.postElectionMsgs = deque()
# PRE-PREPAREs that are waiting to be processed but do not have the
# corresponding request digest. Happens when replica has not been
# forwarded the request by the node but is getting 3 phase messages.
# The value is a list since a malicious entry might send PRE-PREPARE
# with a different digest and since we dont have the request finalised,
# we store all PRE-PPREPARES
self.prePreparesPendingReqDigest = {} # type: Dict[Tuple[str, int], List]
# PREPAREs that are stored by non primary replica for which it has not
# got any PRE-PREPARE. Dictionary that stores a tuple of view no and
# prepare sequence number as key and a deque of PREPAREs as value.
# This deque is attempted to be flushed on receiving every
# PRE-PREPARE request.
self.preparesWaitingForPrePrepare = {}
# type: Dict[Tuple[int, int], deque]
# COMMITs that are stored for which there are no PRE-PREPARE or PREPARE
# received
self.commitsWaitingForPrepare = {}
# type: Dict[Tuple[int, int], deque]
# Dictionary of sent PRE-PREPARE that are stored by primary replica
# which it has broadcasted to all other non primary replicas
# Key of dictionary is a 2 element tuple with elements viewNo,
# pre-prepare seqNo and value is a tuple of Request Digest and time
self.sentPrePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received PRE-PREPAREs. Key of dictionary is a 2
# element tuple with elements viewNo, pre-prepare seqNo and value is
# a tuple of Request Digest and time
self.prePrepares = {}
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], float]]
# Dictionary of received Prepare requests. Key of dictionary is a 2
# element tuple with elements viewNo, seqNo and value is a 2 element
# tuple containing request digest and set of sender node names(sender
# replica names in case of multiple protocol instances)
# (viewNo, seqNo) -> ((identifier, reqId), {senders})
self.prepares = Prepares()
# type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]
self.commits = Commits() # type: Dict[Tuple[int, int],
# Tuple[Tuple[str, int], Set[str]]]
# Set of tuples to keep track of ordered requests. Each tuple is
# (viewNo, ppSeqNo)
self.ordered = OrderedSet() # type: OrderedSet[Tuple[int, int]]
# Dictionary to keep track of the which replica was primary during each
# view. Key is the view no and value is the name of the primary
# replica during that view
self.primaryNames = {} # type: Dict[int, str]
# Holds msgs that are for later views
self.threePhaseMsgsForLaterView = deque()
# type: deque[(ThreePhaseMsg, str)]
# Holds tuple of view no and prepare seq no of 3-phase messages it
# received while it was not participating
self.stashingWhileCatchingUp = set() # type: Set[Tuple]
# Commits which are not being ordered since commits with lower view
# numbers and sequence numbers have not been ordered yet. Key is the
# viewNo and value a map of pre-prepare sequence number to commit
self.stashedCommitsForOrdering = {} # type: Dict[int,
# Dict[int, Commit]]
self.checkpoints = SortedDict(lambda k: k[0])
self.stashingWhileOutsideWaterMarks = deque()
# Low water mark
self._h = 0 # type: int
# High water mark
self.H = self._h + self.config.LOG_SIZE # type: int
self.lastPrePrepareSeqNo = self.h # type: int
@property
def h(self) -> int:
return self._h
@h.setter
def h(self, n):
self._h = n
self.H = self._h + self.config.LOG_SIZE
@property
def requests(self):
return self.node.requests
def shouldParticipate(self, viewNo: int, ppSeqNo: int):
# Replica should only participating in the consensus process and the
# replica did not stash any of this request's 3-phase request
return self.node.isParticipating and (viewNo, ppSeqNo) \
not in self.stashingWhileCatchingUp
@staticmethod
def generateName(nodeName: str, instId: int):
"""
Create and return the name for a replica using its nodeName and
instanceId.
Ex: Alpha:1
"""
return "{}:{}".format(nodeName, instId)
@staticmethod
def getNodeName(replicaName: str):
return replicaName.split(":")[0]
@property
def isPrimary(self):
"""
Is this node primary?
:return: True if this node is primary, False otherwise
"""
return self._primaryName == self.name if self._primaryName is not None \
else None
@property
def primaryName(self):
"""
Name of the primary replica of this replica's instance
:return: Returns name if primary is known, None otherwise
"""
return self._primaryName
@primaryName.setter
def primaryName(self, value: Optional[str]) -> None:
"""
Set the value of isPrimary.
:param value: the value to set isPrimary to
"""
if not value == self._primaryName:
self._primaryName = value
self.primaryNames[self.viewNo] = value
logger.debug("{} setting primaryName for view no {} to: {}".
format(self, self.viewNo, value))
logger.debug("{}'s primaryNames for views are: {}".
format(self, self.primaryNames))
self._stateChanged()
def _stateChanged(self):
"""
A series of actions to be performed when the state of this replica
changes.
- UnstashInBox (see _unstashInBox)
"""
self._unstashInBox()
if self.isPrimary is not None:
# TODO handle suspicion exceptions here
self.process3PhaseReqsQueue()
# TODO handle suspicion exceptions here
try:
self.processPostElectionMsgs()
except SuspiciousNode as ex:
self.outBox.append(ex)
self.discard(ex.msg, ex.reason, logger.warning)
def _stashInBox(self, msg):
"""
Stash the specified message into the inBoxStash of this replica.
:param msg: the message to stash
"""
self.inBoxStash.append(msg)
def _unstashInBox(self):
"""
Append the inBoxStash to the right of the inBox.
"""
self.inBox.extend(self.inBoxStash)
self.inBoxStash.clear()
def __repr__(self):
return self.name
@property
def f(self) -> int:
"""
Return the number of Byzantine Failures that can be tolerated by this
system. Equal to (N - 1)/3, where N is the number of nodes in the
system.
"""
return self.node.f
@property
def viewNo(self):
"""
Return the current view number of this replica.
"""
return self.node.viewNo
def isPrimaryInView(self, viewNo: int) -> Optional[bool]:
"""
Return whether a primary has been selected for this view number.
"""
return self.primaryNames[viewNo] == self.name
def isMsgForLaterView(self, msg):
"""
Return whether this request's view number is greater than the current
view number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo > self.viewNo
def isMsgForCurrentView(self, msg):
"""
Return whether this request's view number is equal to the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo == self.viewNo
def isMsgForPrevView(self, msg):
"""
Return whether this request's view number is less than the current view
number of this replica.
"""
viewNo = getattr(msg, "viewNo", None)
return viewNo < self.viewNo
def isPrimaryForMsg(self, msg) -> Optional[bool]:
"""
Return whether this replica is primary if the request's view number is
equal this replica's view number and primary has been selected for
the current view.
Return None otherwise.
:param msg: message
"""
if self.isMsgForLaterView(msg):
self.discard(msg,
"Cannot get primary status for a request for a later "
"view {}. Request is {}".format(self.viewNo, msg),
logger.error)
else:
return self.isPrimary if self.isMsgForCurrentView(msg) \
else self.isPrimaryInView(msg.viewNo)
def isMsgFromPrimary(self, msg, sender: str) -> bool:
"""
Return whether this message was from primary replica
:param msg:
:param sender:
:return:
"""
if self.isMsgForLaterView(msg):
logger.error("{} cannot get primary for a request for a later "
"view. Request is {}".format(self, msg))
else:
return self.primaryName == sender if self.isMsgForCurrentView(
msg) else self.primaryNames[msg.viewNo] == sender
def _preProcessReqDigest(self, rd: ReqDigest) -> None:
"""
Process request digest if this replica is not a primary, otherwise stash
the message into the inBox.
:param rd: the client Request Digest
"""
if self.isPrimary is not None:
self.processReqDigest(rd)
else:
logger.debug("{} stashing request digest {} since it does not know "
"its primary status".
format(self, (rd.identifier, rd.reqId)))
self._stashInBox(rd)
def serviceQueues(self, limit=None):
"""
Process `limit` number of messages in the inBox.
:param limit: the maximum number of messages to process
:return: the number of messages successfully processed
"""
# TODO should handle SuspiciousNode here
r = self.inBoxRouter.handleAllSync(self.inBox, limit)
r += self._serviceActions()
return r
# Messages that can be processed right now needs to be added back to the
# queue. They might be able to be processed later
def processPostElectionMsgs(self):
"""
Process messages waiting for the election of a primary replica to
complete.
"""
while self.postElectionMsgs:
msg = self.postElectionMsgs.popleft()
logger.debug("{} processing pended msg {}".format(self, msg))
self.dispatchThreePhaseMsg(*msg)
def process3PhaseReqsQueue(self):
"""
Process the 3 phase requests from the queue whose view number is equal
to the current view number of this replica.
"""
unprocessed = deque()
while self.threePhaseMsgsForLaterView:
request, sender = self.threePhaseMsgsForLaterView.popleft()
logger.debug("{} processing pended 3 phase request: {}"
.format(self, request))
# If the request is for a later view dont try to process it but add
# it back to the queue.
if self.isMsgForLaterView(request):
unprocessed.append((request, sender))
else:
self.processThreePhaseMsg(request, sender)
self.threePhaseMsgsForLaterView = unprocessed
@property
def quorum(self) -> int:
r"""
Return the quorum of this RBFT system. Equal to :math:`2f + 1`.
Return None if `f` is not yet determined.
"""
return self.node.quorum
def dispatchThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str) -> Any:
"""
Create a three phase request to be handled by the threePhaseRouter.
:param msg: the ThreePhaseMsg to dispatch
:param sender: the name of the node that sent this request
"""
senderRep = self.generateName(sender, self.instId)
if self.isPpSeqNoAcceptable(msg.ppSeqNo):
try:
self.threePhaseRouter.handleSync((msg, senderRep))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
else:
logger.debug("{} stashing 3 phase message {} since ppSeqNo {} is "
"not between {} and {}".
format(self, msg, msg.ppSeqNo, self.h, self.H))
self.stashingWhileOutsideWaterMarks.append((msg, sender))
def processReqDigest(self, rd: ReqDigest):
"""
Process a request digest. Works only if this replica has decided its
primary status.
:param rd: the client request digest to process
"""
self.stats.inc(TPCStat.ReqDigestRcvd)
if self.isPrimary is False:
self.dequeuePrePrepare(rd.identifier, rd.reqId)
else:
self.doPrePrepare(rd)
def processThreePhaseMsg(self, msg: ThreePhaseMsg, sender: str):
"""
Process a 3-phase (pre-prepare, prepare and commit) request.
Dispatch the request only if primary has already been decided, otherwise
stash it.
:param msg: the Three Phase message, one of PRE-PREPARE, PREPARE,
COMMIT
:param sender: name of the node that sent this message
"""
# Can only proceed further if it knows whether its primary or not
if self.isMsgForLaterView(msg):
self.threePhaseMsgsForLaterView.append((msg, sender))
logger.debug("{} pended received 3 phase request for a later view: "
"{}".format(self, msg))
else:
if self.isPrimary is None:
self.postElectionMsgs.append((msg, sender))
logger.debug("Replica {} pended request {} from {}".
format(self, msg, sender))
else:
self.dispatchThreePhaseMsg(msg, sender)
def processPrePrepare(self, pp: PrePrepare, sender: str):
"""
Validate and process the PRE-PREPARE specified.
If validation is successful, create a PREPARE and broadcast it.
:param pp: a prePrepareRequest
:param sender: name of the node that sent this message
"""
key = (pp.viewNo, pp.ppSeqNo)
logger.debug("{} Receiving PRE-PREPARE{} at {} from {}".
format(self, key, time.perf_counter(), sender))
if self.canProcessPrePrepare(pp, sender):
if not self.node.isParticipating:
self.stashingWhileCatchingUp.add(key)
self.addToPrePrepares(pp)
logger.info("{} processed incoming PRE-PREPARE{}".
format(self, key))
def tryPrepare(self, pp: PrePrepare):
"""
Try to send the Prepare message if the PrePrepare message is ready to
be passed into the Prepare phase.
"""
if self.canSendPrepare(pp):
self.doPrepare(pp)
else:
logger.debug("{} cannot send PREPARE".format(self))
def processPrepare(self, prepare: Prepare, sender: str) -> None:
"""
Validate and process the PREPARE specified.
If validation is successful, create a COMMIT and broadcast it.
:param prepare: a PREPARE msg
:param sender: name of the node that sent the PREPARE
"""
# TODO move this try/except up higher
logger.debug("{} received PREPARE{} from {}".
format(self, (prepare.viewNo, prepare.ppSeqNo), sender))
try:
if self.isValidPrepare(prepare, sender):
self.addToPrepares(prepare, sender)
self.stats.inc(TPCStat.PrepareRcvd)
logger.debug("{} processed incoming PREPARE {}".
format(self, (prepare.viewNo, prepare.ppSeqNo)))
else:
# TODO let's have isValidPrepare throw an exception that gets
# handled and possibly logged higher
logger.warning("{} cannot process incoming PREPARE".
format(self))
except SuspiciousNode as ex:
self.node.reportSuspiciousNodeEx(ex)
def processCommit(self, commit: Commit, sender: str) -> None:
"""
Validate and process the COMMIT specified.
If validation is successful, return the message to the node.
:param commit: an incoming COMMIT message
:param sender: name of the node that sent the COMMIT
"""
logger.debug("{} received COMMIT {} from {}".
format(self, commit, sender))
if self.isValidCommit(commit, sender):
self.stats.inc(TPCStat.CommitRcvd)
self.addToCommits(commit, sender)
logger.debug("{} processed incoming COMMIT{}".
format(self, (commit.viewNo, commit.ppSeqNo)))
def tryCommit(self, prepare: Prepare):
"""
Try to commit if the Prepare message is ready to be passed into the
commit phase.
"""
if self.canCommit(prepare):
self.doCommit(prepare)
else:
logger.debug("{} not yet able to send COMMIT".format(self))
def tryOrder(self, commit: Commit):
"""
Try to order if the Commit message is ready to be ordered.
"""
canOrder, reason = self.canOrder(commit)
if canOrder:
logger.debug("{} returning request to node".format(self))
self.tryOrdering(commit)
else:
logger.trace("{} cannot return request to node: {}".
format(self, reason))
def doPrePrepare(self, reqDigest: ReqDigest) -> None:
"""
Broadcast a PRE-PREPARE to all the replicas.
:param reqDigest: a tuple with elements identifier, reqId, and digest
"""
if not self.node.isParticipating:
logger.error("Non participating node is attempting PRE-PREPARE. "
"This should not happen.")
return
if self.lastPrePrepareSeqNo == self.H:
logger.debug("{} stashing PRE-PREPARE {} since outside greater "
"than high water mark {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo+1),
self.H))
self.stashingWhileOutsideWaterMarks.append(reqDigest)
return
self.lastPrePrepareSeqNo += 1
tm = time.time()*1000
logger.debug("{} Sending PRE-PREPARE {} at {}".
format(self, (self.viewNo, self.lastPrePrepareSeqNo),
time.perf_counter()))
prePrepareReq = PrePrepare(self.instId,
self.viewNo,
self.lastPrePrepareSeqNo,
*reqDigest,
tm)
self.sentPrePrepares[self.viewNo, self.lastPrePrepareSeqNo] = (reqDigest.key,
tm)
self.send(prePrepareReq, TPCStat.PrePrepareSent)
def doPrepare(self, pp: PrePrepare):
logger.debug("{} Sending PREPARE {} at {}".
format(self, (pp.viewNo, pp.ppSeqNo), time.perf_counter()))
prepare = Prepare(self.instId,
pp.viewNo,
pp.ppSeqNo,
pp.digest,
pp.ppTime)
self.send(prepare, TPCStat.PrepareSent)
self.addToPrepares(prepare, self.name)
def doCommit(self, p: Prepare):
"""
Create a commit message from the given Prepare message and trigger the
commit phase
:param p: the prepare message
"""
logger.debug("{} Sending COMMIT{} at {}".
format(self, (p.viewNo, p.ppSeqNo), time.perf_counter()))
commit = Commit(self.instId,
p.viewNo,
p.ppSeqNo,
p.digest,
p.ppTime)
self.send(commit, TPCStat.CommitSent)
self.addToCommits(commit, self.name)
def canProcessPrePrepare(self, pp: PrePrepare, sender: str) -> bool:
"""
Decide whether this replica is eligible to process a PRE-PREPARE,
based on the following criteria:
- this replica is non-primary replica
- the request isn't in its list of received PRE-PREPAREs
- the request is waiting to for PRE-PREPARE and the digest value matches
:param pp: a PRE-PREPARE msg to process
:param sender: the name of the node that sent the PRE-PREPARE msg
:return: True if processing is allowed, False otherwise
"""
# TODO: Check whether it is rejecting PRE-PREPARE from previous view
# PRE-PREPARE should not be sent from non primary
if not self.isMsgFromPrimary(pp, sender):
raise SuspiciousNode(sender, Suspicions.PPR_FRM_NON_PRIMARY, pp)
# A PRE-PREPARE is being sent to primary
if self.isPrimaryForMsg(pp) is True:
raise SuspiciousNode(sender, Suspicions.PPR_TO_PRIMARY, pp)
# A PRE-PREPARE is sent that has already been received
if (pp.viewNo, pp.ppSeqNo) in self.prePrepares:
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PPR_SENT, pp)
key = (pp.identifier, pp.reqId)
if not self.requests.isFinalised(key):
self.enqueuePrePrepare(pp, sender)
return False
# A PRE-PREPARE is sent that does not match request digest
if self.requests.digest(key) != pp.digest:
raise SuspiciousNode(sender, Suspicions.PPR_DIGEST_WRONG, pp)
return True
def addToPrePrepares(self, pp: PrePrepare) -> None:
"""
Add the specified PRE-PREPARE to this replica's list of received
PRE-PREPAREs.
:param pp: the PRE-PREPARE to add to the list
"""
key = (pp.viewNo, pp.ppSeqNo)
self.prePrepares[key] = \
((pp.identifier, pp.reqId), pp.ppTime)
self.dequeuePrepares(*key)
self.dequeueCommits(*key)
self.stats.inc(TPCStat.PrePrepareRcvd)
self.tryPrepare(pp)
def hasPrepared(self, request) -> bool:
return self.prepares.hasPrepareFrom(request, self.name)
def canSendPrepare(self, request) -> bool:
"""
Return whether the request identified by (identifier, requestId) can
proceed to the Prepare step.
:param request: any object with identifier and requestId attributes
"""
return self.shouldParticipate(request.viewNo, request.ppSeqNo) \
and not self.hasPrepared(request) \
and self.requests.isFinalised((request.identifier,
request.reqId))
def isValidPrepare(self, prepare: Prepare, sender: str) -> bool:
"""
Return whether the PREPARE specified is valid.
:param prepare: the PREPARE to validate
:param sender: the name of the node that sent the PREPARE
:return: True if PREPARE is valid, False otherwise
"""
key = (prepare.viewNo, prepare.ppSeqNo)
primaryStatus = self.isPrimaryForMsg(prepare)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
# If a non primary replica and receiving a PREPARE request before a
# PRE-PREPARE request, then proceed
# PREPARE should not be sent from primary
if self.isMsgFromPrimary(prepare, sender):
raise SuspiciousNode(sender, Suspicions.PR_FRM_PRIMARY, prepare)
# If non primary replica
if primaryStatus is False:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE not received for the PREPARE, might be slow network
if key not in ppReqs:
self.enqueuePrepare(prepare, sender)
return False
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
# If primary replica
else:
if self.prepares.hasPrepareFrom(prepare, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_PR_SENT, prepare)
# If PRE-PREPARE was not sent for this PREPARE, certainly
# malicious behavior
elif key not in ppReqs:
raise SuspiciousNode(sender, Suspicions.UNKNOWN_PR_SENT, prepare)
elif prepare.digest != self.requests.digest(ppReqs[key][0]):
raise SuspiciousNode(sender, Suspicions.PR_DIGEST_WRONG, prepare)
elif prepare.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.PR_TIME_WRONG,
prepare)
else:
return True
def addToPrepares(self, prepare: Prepare, sender: str):
self.prepares.addVote(prepare, sender)
self.tryCommit(prepare)
def hasCommitted(self, request) -> bool:
return self.commits.hasCommitFrom(ThreePhaseKey(
request.viewNo, request.ppSeqNo), self.name)
def canCommit(self, prepare: Prepare) -> bool:
"""
Return whether the specified PREPARE can proceed to the Commit
step.
Decision criteria:
- If this replica has got just 2f PREPARE requests then commit request.
- If less than 2f PREPARE requests then probably there's no consensus on
the request; don't commit
- If more than 2f then already sent COMMIT; don't commit
:param prepare: the PREPARE
"""
return self.shouldParticipate(prepare.viewNo, prepare.ppSeqNo) and \
self.prepares.hasQuorum(prepare, self.f) and \
not self.hasCommitted(prepare)
def isValidCommit(self, commit: Commit, sender: str) -> bool:
"""
Return whether the COMMIT specified is valid.
:param commit: the COMMIT to validate
:return: True if `request` is valid, False otherwise
"""
primaryStatus = self.isPrimaryForMsg(commit)
ppReqs = self.sentPrePrepares if primaryStatus else self.prePrepares
key = (commit.viewNo, commit.ppSeqNo)
if key not in ppReqs:
self.enqueueCommit(commit, sender)
return False
if (key not in self.prepares and
key not in self.preparesWaitingForPrePrepare):
logger.debug("{} rejecting COMMIT{} due to lack of prepares".
format(self, key))
# raise SuspiciousNode(sender, Suspicions.UNKNOWN_CM_SENT, commit)
return False
elif self.commits.hasCommitFrom(commit, sender):
raise SuspiciousNode(sender, Suspicions.DUPLICATE_CM_SENT, commit)
elif commit.digest != self.getDigestFor3PhaseKey(ThreePhaseKey(*key)):
raise SuspiciousNode(sender, Suspicions.CM_DIGEST_WRONG, commit)
elif key in ppReqs and commit.ppTime != ppReqs[key][1]:
raise SuspiciousNode(sender, Suspicions.CM_TIME_WRONG,
commit)
else:
return True
def addToCommits(self, commit: Commit, sender: str):
"""
Add the specified COMMIT to this replica's list of received
commit requests.
:param commit: the COMMIT to add to the list
:param sender: the name of the node that sent the COMMIT
"""
self.commits.addVote(commit, sender)
self.tryOrder(commit)
def hasOrdered(self, viewNo, ppSeqNo) -> bool:
return (viewNo, ppSeqNo) in self.ordered
def canOrder(self, commit: Commit) -> Tuple[bool, Optional[str]]:
"""
Return whether the specified commitRequest can be returned to the node.
Decision criteria:
- If have got just 2f+1 Commit requests then return request to node
- If less than 2f+1 of commit requests then probably don't have
consensus on the request; don't return request to node
- If more than 2f+1 then already returned to node; don't return request
to node
:param commit: the COMMIT
"""
if not self.commits.hasQuorum(commit, self.f):
return False, "no quorum: {} commits where f is {}".\
format(commit, self.f)
if self.hasOrdered(commit.viewNo, commit.ppSeqNo):
return False, "already ordered"
if not self.isNextInOrdering(commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if viewNo not in self.stashedCommitsForOrdering:
self.stashedCommitsForOrdering[viewNo] = {}
self.stashedCommitsForOrdering[viewNo][ppSeqNo] = commit
# self._schedule(self.orderStashedCommits, 2)
self.startRepeating(self.orderStashedCommits, 2)
return False, "stashing {} since out of order".\
format(commit)
return True, None
def isNextInOrdering(self, commit: Commit):
viewNo, ppSeqNo = commit.viewNo, commit.ppSeqNo
if self.ordered and self.ordered[-1] == (viewNo, ppSeqNo-1):
return True
for (v, p) in self.commits:
if v < viewNo:
# Have commits from previous view that are unordered.
# TODO: Question: would commits be always ordered, what if
# some are never ordered and its fine, go to PBFT.
return False
if v == viewNo and p < ppSeqNo and (v, p) not in self.ordered:
# If unordered commits are found with lower ppSeqNo then this
# cannot be ordered.
return False
# TODO: Revisit PBFT paper, how to make sure that last request of the
# last view has been ordered? Need change in `VIEW CHANGE` mechanism.
# Somehow view change needs to communicate what the last request was.
# Also what if some COMMITs were completely missed in the same view
return True
def orderStashedCommits(self):
# TODO: What if the first few commits were out of order and stashed?
# `self.ordered` would be empty
if self.ordered:
lastOrdered = self.ordered[-1]
vToRemove = set()
for v in self.stashedCommitsForOrdering:
if v < lastOrdered[0] and self.stashedCommitsForOrdering[v]:
raise RuntimeError("{} found commits from previous view {}"
" that were not ordered but last ordered"
" is {}".format(self, v, lastOrdered))
pToRemove = set()
for p, commit in self.stashedCommitsForOrdering[v].items():
if (v == lastOrdered[0] and lastOrdered == (v, p - 1)) or \
(v > lastOrdered[0] and
self.isLowestCommitInView(commit)):
logger.debug("{} ordering stashed commit {}".
format(self, commit))
if self.tryOrdering(commit):
lastOrdered = (v, p)
pToRemove.add(p)
for p in pToRemove:
del self.stashedCommitsForOrdering[v][p]
if not self.stashedCommitsForOrdering[v]:
vToRemove.add(v)
for v in vToRemove:
del self.stashedCommitsForOrdering[v]
# if self.stashedCommitsForOrdering:
# self._schedule(self.orderStashedCommits, 2)
if not self.stashedCommitsForOrdering:
self.stopRepeating(self.orderStashedCommits)
def isLowestCommitInView(self, commit):
# TODO: Assumption: This assumes that at least one commit that was sent
# for any request by any node has been received in the view of this
# commit
ppSeqNos = []
for v, p in self.commits:
if v == commit.viewNo:
ppSeqNos.append(p)
return min(ppSeqNos) == commit.ppSeqNo if ppSeqNos else True
def tryOrdering(self, commit: Commit) -> None:
"""
Attempt to send an ORDERED request for the specified COMMIT to the
node.
:param commit: the COMMIT message
"""
key = (commit.viewNo, commit.ppSeqNo)
logger.debug("{} trying to order COMMIT{}".format(self, key))
reqKey = self.getReqKeyFrom3PhaseKey(key) # type: Tuple
digest = self.getDigestFor3PhaseKey(key)
if not digest:
logger.error("{} did not find digest for {}, request key {}".
format(self, key, reqKey))
return
self.doOrder(*key, *reqKey, digest, commit.ppTime)
return True
def doOrder(self, viewNo, ppSeqNo, identifier, reqId, digest, ppTime):
key = (viewNo, ppSeqNo)
self.addToOrdered(*key)
ordered = Ordered(self.instId,
viewNo,
identifier,
reqId,
ppTime)
# TODO: Should not order or add to checkpoint while syncing
# 3 phase state.
self.send(ordered, TPCStat.OrderSent)
if key in self.stashingWhileCatchingUp:
self.stashingWhileCatchingUp.remove(key)
logger.debug("{} ordered request {}".format(self, (viewNo, ppSeqNo)))
self.addToCheckpoint(ppSeqNo, digest)
def processCheckpoint(self, msg: Checkpoint, sender: str):
if self.checkpoints:
seqNo = msg.seqNo
_, firstChk = self.firstCheckPoint
if firstChk.isStable:
if firstChk.seqNo == seqNo:
self.discard(msg, reason="Checkpoint already stable",
logMethod=logger.debug)
return
if firstChk.seqNo > seqNo:
self.discard(msg, reason="Higher stable checkpoint present",
logMethod=logger.debug)
return
for state in self.checkpoints.values():
if state.seqNo == seqNo:
if state.digest == msg.digest:
state.receivedDigests[sender] = msg.digest
break
else:
logger.error("{} received an incorrect digest {} for "
"checkpoint {} from {}".format(self,
msg.digest,
seqNo,
sender))
return
if len(state.receivedDigests) == 2*self.f:
self.markCheckPointStable(msg.seqNo)
else:
self.discard(msg, reason="No checkpoints present to tally",
logMethod=logger.warn)
def _newCheckpointState(self, ppSeqNo, digest) -> CheckpointState:
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ - 1
logger.debug("{} adding new checkpoint state for {}".
format(self, (s, e)))
state = CheckpointState(ppSeqNo, [digest, ], None, {}, False)
self.checkpoints[s, e] = state
return state
def addToCheckpoint(self, ppSeqNo, digest):
for (s, e) in self.checkpoints.keys():
if s <= ppSeqNo <= e:
state = self.checkpoints[s, e] # type: CheckpointState
state.digests.append(digest)
state = updateNamedTuple(state, seqNo=ppSeqNo)
self.checkpoints[s, e] = state
break
else:
state = self._newCheckpointState(ppSeqNo, digest)
s, e = ppSeqNo, ppSeqNo + self.config.CHK_FREQ
if len(state.digests) == self.config.CHK_FREQ:
state = updateNamedTuple(state, digest=serialize(state.digests),
digests=[])
self.checkpoints[s, e] = state
self.send(Checkpoint(self.instId, self.viewNo, ppSeqNo,
state.digest))
def markCheckPointStable(self, seqNo):
previousCheckpoints = []
for (s, e), state in self.checkpoints.items():
if e == seqNo:
state = updateNamedTuple(state, isStable=True)
self.checkpoints[s, e] = state
break
else:
previousCheckpoints.append((s, e))
else:
logger.error("{} could not find {} in checkpoints".
format(self, seqNo))
return
self.h = seqNo
for k in previousCheckpoints:
logger.debug("{} removing previous checkpoint {}".format(self, k))
self.checkpoints.pop(k)
self.gc(seqNo)
logger.debug("{} marked stable checkpoint {}".format(self, (s, e)))
self.processStashedMsgsForNewWaterMarks()
def gc(self, tillSeqNo):
logger.debug("{} cleaning up till {}".format(self, tillSeqNo))
tpcKeys = set()
reqKeys = set()
for (v, p), (reqKey, _) in self.sentPrePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
for (v, p), (reqKey, _) in self.prePrepares.items():
if p <= tillSeqNo:
tpcKeys.add((v, p))
reqKeys.add(reqKey)
logger.debug("{} found {} 3 phase keys to clean".
format(self, len(tpcKeys)))
logger.debug("{} found {} request keys to clean".
format(self, len(reqKeys)))
for k in tpcKeys:
self.sentPrePrepares.pop(k, None)
self.prePrepares.pop(k, None)
self.prepares.pop(k, None)
self.commits.pop(k, None)
if k in self.ordered:
self.ordered.remove(k)
for k in reqKeys:
self.requests.pop(k, None)
def processStashedMsgsForNewWaterMarks(self):
while self.stashingWhileOutsideWaterMarks:
item = self.stashingWhileOutsideWaterMarks.pop()
logger.debug("{} processing stashed item {} after new stable "
"checkpoint".format(self, item))
if isinstance(item, ReqDigest):
self.doPrePrepare(item)
elif isinstance(item, tuple) and len(tuple) == 2:
self.dispatchThreePhaseMsg(*item)
else:
logger.error("{} cannot process {} "
"from stashingWhileOutsideWaterMarks".
format(self, item))
@property
def firstCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(0)
@property
def lastCheckPoint(self) -> Tuple[Tuple[int, int], CheckpointState]:
if not self.checkpoints:
return None
else:
return self.checkpoints.peekitem(-1)
def isPpSeqNoAcceptable(self, ppSeqNo: int):
return self.h < ppSeqNo <= self.H
def addToOrdered(self, viewNo: int, ppSeqNo: int):
self.ordered.add((viewNo, ppSeqNo))
def enqueuePrePrepare(self, request: PrePrepare, sender: str):
logger.debug("Queueing pre-prepares due to unavailability of finalised "
"Request. Request {} from {}".format(request, sender))
key = (request.identifier, request.reqId)
if key not in self.prePreparesPendingReqDigest:
self.prePreparesPendingReqDigest[key] = []
self.prePreparesPendingReqDigest[key].append((request, sender))
def dequeuePrePrepare(self, identifier: int, reqId: int):
key = (identifier, reqId)
if key in self.prePreparesPendingReqDigest:
pps = self.prePreparesPendingReqDigest[key]
for (pp, sender) in pps:
logger.debug("{} popping stashed PRE-PREPARE{}".
format(self, key))
if pp.digest == self.requests.digest(key):
self.prePreparesPendingReqDigest.pop(key)
self.processPrePrepare(pp, sender)
logger.debug(
"{} processed {} PRE-PREPAREs waiting for finalised "
"request for identifier {} and reqId {}".
format(self, pp, identifier, reqId))
break
def enqueuePrepare(self, request: Prepare, sender: str):
logger.debug("Queueing prepares due to unavailability of PRE-PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.preparesWaitingForPrePrepare:
self.preparesWaitingForPrePrepare[key] = deque()
self.preparesWaitingForPrePrepare[key].append((request, sender))
def dequeuePrepares(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.preparesWaitingForPrePrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.preparesWaitingForPrePrepare[key]:
prepare, sender = self.preparesWaitingForPrePrepare[
key].popleft()
logger.debug("{} popping stashed PREPARE{}".format(self, key))
self.processPrepare(prepare, sender)
i += 1
self.preparesWaitingForPrePrepare.pop(key)
logger.debug("{} processed {} PREPAREs waiting for PRE-PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def enqueueCommit(self, request: Commit, sender: str):
logger.debug("Queueing commit due to unavailability of PREPARE. "
"Request {} from {}".format(request, sender))
key = (request.viewNo, request.ppSeqNo)
if key not in self.commitsWaitingForPrepare:
self.commitsWaitingForPrepare[key] = deque()
self.commitsWaitingForPrepare[key].append((request, sender))
def dequeueCommits(self, viewNo: int, ppSeqNo: int):
key = (viewNo, ppSeqNo)
if key in self.commitsWaitingForPrepare:
i = 0
# Keys of pending prepares that will be processed below
while self.commitsWaitingForPrepare[key]:
commit, sender = self.commitsWaitingForPrepare[
key].popleft()
logger.debug("{} popping stashed COMMIT{}".format(self, key))
self.processCommit(commit, sender)
i += 1
self.commitsWaitingForPrepare.pop(key)
logger.debug("{} processed {} COMMITs waiting for PREPARE for"
" view no {} and seq no {}".
format(self, i, viewNo, ppSeqNo))
def getDigestFor3PhaseKey(self, key: ThreePhaseKey) -> Optional[str]:
reqKey = self.getReqKeyFrom3PhaseKey(key)
digest = self.requests.digest(reqKey)
if not digest:
logger.debug("{} could not find digest in sent or received "
"PRE-PREPAREs or PREPAREs for 3 phase key {} and req "
"key {}".format(self, key, reqKey))
return None
else:
return digest
def getReqKeyFrom3PhaseKey(self, key: ThreePhaseKey):
reqKey = None
if key in self.sentPrePrepares:
reqKey = self.sentPrePrepares[key][0]
elif key in self.prePrepares:
reqKey = self.prePrepares[key][0]
elif key in self.prepares:
reqKey = self.prepares[key][0]
else:
logger.debug("Could not find request key for 3 phase key {}".
format(key))
return reqKey
@property
def threePhaseState(self):
# TODO: This method is incomplete
# Gets the current stable and unstable checkpoints and creates digest
# of unstable checkpoints
if self.checkpoints:
pass
else:
state = []
return ThreePCState(self.instId, state)
def process3PhaseState(self, msg: ThreePCState, sender: str):
# TODO: This is not complete
pass
def send(self, msg, stat=None) -> None:
"""
Send a message to the node on which this replica resides.
:param msg: the message to send
"""
logger.display("{} sending {}".format(self, msg.__class__.__name__),
extra={"cli": True})
logger.trace("{} sending {}".format(self, msg))
if stat:
self.stats.inc(stat)
self.outBox.append(msg)
def test_pop2():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
with pytest.raises(KeyError):
temp.pop('A')
def test_pop2():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
temp.pop('A')
def test_pop():
mapping = [(val, pos) for pos, val in enumerate(string.ascii_lowercase)]
temp = SortedDict(mapping)
assert temp.pop('a') == 0
assert temp.pop('a', -1) == -1
class DotMap(MutableMapping):
def __init__(self, *args, **kwargs):
self._map = SortedDict()
if args:
d = args[0]
if type(d) is dict:
for k, v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v)
self._map[k] = v
if kwargs:
for k, v in self.__call_items(kwargs):
self._map[k] = v
@staticmethod
def __call_items(obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map:
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k == '_map':
super(DotMap, self).__setattr__(k, v)
else:
self[k] = v
def __getattr__(self, k):
if k == '_map':
return self._map
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k, v in self.__call_items(self._map):
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def to_dict(self):
d = {}
for k, v in self.items():
if type(v) is DotMap:
v = v.to_dict()
d[k] = v
return d
def pprint(self):
pprint(self.to_dict())
# proper dict subclassing
def values(self):
return self._map.values()
@staticmethod
def parse_other(other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parse_other(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parse_other(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parse_other(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parse_other(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parse_other(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parse_other(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def copy(self):
return self
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def setdefault(self, key, default=None):
return self._map.setdefault(key, default)
def viewitems(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewitems()
else:
return self._map.items()
def viewkeys(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewkeys()
else:
return self._map.keys()
def viewvalues(self):
if version_info.major == 2 and version_info.minor >= 7:
return self._map.viewvalues()
else:
return self._map.values()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = SortedDict.fromkeys(seq, value)
return d
class StreamChangeCache(object):
"""Keeps track of the stream positions of the latest change in a set of entities.
Typically the entity will be a room or user id.
Given a list of entities and a stream position, it will give a subset of
entities that may have changed since that position. If position key is too
old then the cache will simply return all given entities.
"""
def __init__(self, name, current_stream_pos, max_size=10000, prefilled_cache=None):
self._max_size = int(max_size * caches.CACHE_SIZE_FACTOR)
self._entity_to_key = {}
self._cache = SortedDict()
self._earliest_known_stream_pos = current_stream_pos
self.name = name
self.metrics = caches.register_cache("cache", self.name, self._cache)
if prefilled_cache:
for entity, stream_pos in prefilled_cache.items():
self.entity_has_changed(entity, stream_pos)
def has_entity_changed(self, entity, stream_pos):
"""Returns True if the entity may have been updated since stream_pos
"""
assert type(stream_pos) in integer_types
if stream_pos < self._earliest_known_stream_pos:
self.metrics.inc_misses()
return True
latest_entity_change_pos = self._entity_to_key.get(entity, None)
if latest_entity_change_pos is None:
self.metrics.inc_hits()
return False
if stream_pos < latest_entity_change_pos:
self.metrics.inc_misses()
return True
self.metrics.inc_hits()
return False
def get_entities_changed(self, entities, stream_pos):
"""
Returns subset of entities that have had new things since the given
position. Entities unknown to the cache will be returned. If the
position is too old it will just return the given list.
"""
assert type(stream_pos) is int
if stream_pos >= self._earliest_known_stream_pos:
changed_entities = {
self._cache[k] for k in self._cache.islice(
start=self._cache.bisect_right(stream_pos),
)
}
result = changed_entities.intersection(entities)
self.metrics.inc_hits()
else:
result = set(entities)
self.metrics.inc_misses()
return result
def has_any_entity_changed(self, stream_pos):
"""Returns if any entity has changed
"""
assert type(stream_pos) is int
if not self._cache:
# If we have no cache, nothing can have changed.
return False
if stream_pos >= self._earliest_known_stream_pos:
self.metrics.inc_hits()
return self._cache.bisect_right(stream_pos) < len(self._cache)
else:
self.metrics.inc_misses()
return True
def get_all_entities_changed(self, stream_pos):
"""Returns all entites that have had new things since the given
position. If the position is too old it will return None.
"""
assert type(stream_pos) is int
if stream_pos >= self._earliest_known_stream_pos:
return [self._cache[k] for k in self._cache.islice(
start=self._cache.bisect_right(stream_pos))]
else:
return None
def entity_has_changed(self, entity, stream_pos):
"""Informs the cache that the entity has been changed at the given
position.
"""
assert type(stream_pos) is int
if stream_pos > self._earliest_known_stream_pos:
old_pos = self._entity_to_key.get(entity, None)
if old_pos is not None:
stream_pos = max(stream_pos, old_pos)
self._cache.pop(old_pos, None)
self._cache[stream_pos] = entity
self._entity_to_key[entity] = stream_pos
while len(self._cache) > self._max_size:
k, r = self._cache.popitem(0)
self._earliest_known_stream_pos = max(
k, self._earliest_known_stream_pos,
)
self._entity_to_key.pop(r, None)
def get_max_pos_of_last_change(self, entity):
"""Returns an upper bound of the stream id of the last change to an
entity.
"""
return self._entity_to_key.get(entity, self._earliest_known_stream_pos)
