def test_islice():
return
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.islice())
values = sorted(range(53), key=negate)
slt.update(values)
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop, reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(slt.islice(start=start)) == values[start:]
assert list(slt.islice(start=start, reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(slt.islice(stop=stop)) == values[:stop]
assert list(slt.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_islice():
return
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.islice())
values = sorted(range(53), key=negate)
slt.update(values)
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(slt.islice(start, stop,
reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(slt.islice(start=start)) == values[start:]
assert list(slt.islice(start=start,
reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(slt.islice(stop=stop)) == values[:stop]
assert list(slt.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_islice():
sl = SortedKeyList(key=modulo)
sl._reset(7)
assert [] == list(sl.islice())
values = sorted(range(100), key=modulo)
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop,
reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start,
reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_bisect():
slt = SortedKeyList(key=negate)
assert slt.bisect(10) == 0
slt = SortedKeyList(range(100), key=negate)
slt._reset(17)
slt.update(range(100))
slt._check()
assert slt.bisect(10) == 180
assert slt.bisect(0) == 200
def test_eq():
this = SortedKeyList(range(10), key=negate)
this._reset(4)
that = SortedKeyList(range(20), key=negate)
that._reset(4)
assert not (this == that)
that.clear()
that.update(range(10))
assert this == that
def test_bisect_right():
slt = SortedKeyList(key=modulo)
assert slt.bisect_right(10) == 0
slt = SortedKeyList(range(100), key=modulo)
slt._reset(17)
slt.update(range(100))
slt._check()
assert slt.bisect_right(10) == 20
assert slt.bisect_right(0) == 20
def test_update_order_consistency():
setup = [10, 20, 30]
slt1 = SortedKeyList(setup, key=modulo)
slt2 = SortedKeyList(setup, key=modulo)
addition = [40, 50, 60]
for value in addition:
slt1.add(value)
slt2.update(addition)
assert slt1 == slt2
def test_bisect_right():
slt = SortedKeyList(key=modulo)
assert slt.bisect_right(10) == 0
slt = SortedKeyList(range(100), key=modulo)
slt._reset(17)
slt.update(range(100))
slt._check()
assert slt.bisect_right(10) == 20
assert slt.bisect_right(0) == 20
def test_eq():
this = SortedKeyList(range(10), key=negate)
this._reset(4)
that = SortedKeyList(range(20), key=negate)
that._reset(4)
assert not (this == that)
that.clear()
that.update(range(10))
assert this == that
def test_bisect():
slt = SortedKeyList(key=negate)
assert slt.bisect(10) == 0
slt = SortedKeyList(range(100), key=negate)
slt._reset(17)
slt.update(range(100))
slt._check()
assert slt.bisect(10) == 180
assert slt.bisect(0) == 200
def test_update():
slt = SortedKeyList(key=modulo)
slt.update(range(1000))
assert all(tup[0] == tup[1] for tup in zip(slt, sorted(range(1000), key=modulo)))
assert len(slt) == 1000
slt._check()
slt.update(range(10000))
assert len(slt) == 11000
slt._check()
def test_update():
slt = SortedKeyList(key=modulo)
slt.update(range(1000))
assert all(tup[0] == tup[1]
for tup in zip(slt, sorted(range(1000), key=modulo)))
assert len(slt) == 1000
slt._check()
slt.update(range(10000))
assert len(slt) == 11000
slt._check()
def test_contains():
slt = SortedKeyList(key=negate)
assert 0 not in slt
slt.update(range(10000))
for val in range(10000):
assert val in slt
assert 10000 not in slt
assert -1 not in slt
slt._check()
def test_contains():
slt = SortedKeyList(key=negate)
assert 0 not in slt
slt.update(range(10000))
for val in range(10000):
assert val in slt
assert 10000 not in slt
assert -1 not in slt
slt._check()
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=modulo)
slt._reset(17)
slt.add(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def get_hashes_and_next_idx(self, from_idx: RangeIdx, count: int) -> Tuple[List[bytes], Optional[RangeIdx]]:
timestamp, offset = from_idx
idx = self._index.bisect_key_left((timestamp, b''))
txs = SortedKeyList(key=lambda x: (x.timestamp, x.hash))
txs.update(self._index[idx:idx+offset+count])
ret_txs = txs[offset:offset+count]
hashes = [tx.hash for tx in ret_txs]
if len(ret_txs) < count:
return hashes, None
else:
next_offset = offset + count
next_timestamp = ret_txs[-1].timestamp
if next_timestamp != timestamp:
next_idx = txs.bisect_key_left((next_timestamp, b''))
next_offset -= next_idx
return hashes, RangeIdx(next_timestamp, next_offset)
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=modulo)
slt._reset(17)
slt.add(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_contains():
slt = SortedKeyList(key=modulo)
slt._reset(7)
assert 0 not in slt
slt.update(range(100))
for val in range(100):
assert val in slt
assert 100 not in slt
slt._check()
slt = SortedKeyList(range(100), key=modulo)
slt._reset(4)
assert all(val not in slt for val in range(100, 200))
def rearrangeString(self, s: str, k: int) -> str:
if k == 0:
return s
cnt = SortedKeyList(Counter(s).items(), key=lambda t: (-t[1], t[0]))
last = defaultdict(lambda: -k)
ans = ''
while len(cnt) > 0:
upd = []
for _ in range(min(k, len(cnt))):
sym, num = cnt.pop(0)
if len(ans) - last[sym] < k:
return ''
last[sym] = len(ans)
ans += sym
if num > 1:
upd.append((sym, num - 1))
cnt.update(upd)
return ans
def test_contains():
slt = SortedKeyList(key=modulo)
slt._reset(7)
assert 0 not in slt
slt.update(range(100))
for val in range(100):
assert val in slt
assert 100 not in slt
slt._check()
slt = SortedKeyList(range(100), key=modulo)
slt._reset(4)
assert all(val not in slt for val in range(100, 200))
def test_irange():
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.irange())
values = list(range(53))
slt.update(values)
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end,
start)) == values[start:(end + 1)][::-1]
assert list(slt.irange(end, start,
reverse=True)) == values[start:(end + 1)]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(
end, start,
(True, False))) == values[(start + 1):(end + 1)][::-1]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end, start,
(False, True))) == values[start:end][::-1]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(
end, start, (False, False))) == values[(start + 1):end][::-1]
for start in range(53):
assert list(slt.irange(start)) == values[:(start + 1)][::-1]
for end in range(53):
assert list(slt.irange(None, end,
(True, False))) == values[(end + 1):][::-1]
assert list(slt.irange(inclusive=(False, False))) == values[::-1]
assert list(slt.irange(-1)) == []
assert list(slt.irange(None, -1, (True, False))) == values[::-1]
def find_candidate_gaps(gaps: List[CalendarGap],
gap_request: GapRequest) -> List[CalendarGap]:
"""
gap_request ={'minimum_start...}
Returns a subset of the original gaps, but not copies.
one list, it is sorted by start date.
Find all gaps with a start date is is after the required minimum start date, call this sublist 1. O(log2(n))
now sort sublist1 by gap end date O(N log N)
search the second list on the end date criteria: sublist2
now sort sublist2 by gap duration: sublist 3
search sublist3 to eliminate gaps which aren't big enough, and sort by start date into sublist4.
Now take the first 1.
"""
def key_fun_startdate(cg: CalendarGap):
return cg.gap_start_datetime
def key_fun_enddate(cg: CalendarGap):
return cg.gap_end_datetime
gaps_sorted_by_start = SortedKeyList(
key=key_fun_startdate) #type: SortedKeyList[CalendarGap]
gaps_sorted_by_start.update(gaps)
index_start_date = gaps_sorted_by_start.bisect_key_left(
gap_request.minimum_start_date)
remaining_gaps_sorted_by_end = SortedKeyList(
key=key_fun_enddate) #type: SortedKeyList[CalendarGap]
remaining_gaps_sorted_by_end.update(gaps[index_start_date:])
index_end_date = remaining_gaps_sorted_by_end.bisect_key_left(
gap_request.maximum_end_date)
gaps_of_sufficient_duration = [
g for g in remaining_gaps_sorted_by_end[:index_end_date]
if g.gap_duration.seconds / 60 >= gap_request.gap_duration_minutes
]
return gaps_of_sufficient_duration
def test_islice():
sl = SortedKeyList(key=modulo)
sl._reset(7)
assert [] == list(sl.islice())
values = sorted(range(100), key=modulo)
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop, reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start, reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_irange():
slt = SortedKeyList(key=negate)
slt._reset(7)
assert [] == list(slt.irange())
values = list(range(53))
slt.update(values)
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end, start)) == values[start:(end + 1)][::-1]
assert list(slt.irange(end, start, reverse=True)) == values[start:(end + 1)]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end, start, (True, False))) == values[(start + 1):(end + 1)][::-1]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end, start, (False, True))) == values[start:end][::-1]
for start in range(53):
for end in range(start, 53):
assert list(slt.irange(end, start, (False, False))) == values[(start + 1):end][::-1]
for start in range(53):
assert list(slt.irange(start)) == values[:(start + 1)][::-1]
for end in range(53):
assert list(slt.irange(None, end, (True, False))) == values[(end + 1):][::-1]
assert list(slt.irange(inclusive=(False, False))) == values[::-1]
assert list(slt.irange(-1)) == []
assert list(slt.irange(None, -1, (True, False))) == values[::-1]
def test_update():
slt = SortedKeyList(key=negate)
slt.update(range(1000))
assert len(slt) == 1000
slt._check()
slt.update(range(100))
assert len(slt) == 1100
slt._check()
slt.update(range(10000))
assert len(slt) == 11100
slt._check()
values = sorted((val for val in chain(range(100), range(1000), range(10000))), key=negate)
assert all(tup[0] == tup[1] for tup in zip(slt, values))
def test_update():
slt = SortedKeyList(key=negate)
slt.update(range(1000))
assert len(slt) == 1000
slt._check()
slt.update(range(100))
assert len(slt) == 1100
slt._check()
slt.update(range(10000))
assert len(slt) == 11100
slt._check()
values = sorted(
(val for val in chain(range(100), range(1000), range(10000))),
key=negate)
assert all(tup[0] == tup[1] for tup in zip(slt, values))
class FreshPondSim:
def __init__(self,
distance,
start_time,
end_time,
entrances,
entrance_weights,
rand_velocities_and_distances_func,
entrance_rate,
entrance_rate_integral=None,
entrance_rate_integral_inverse=None,
interpolate_rate=True,
interpolate_rate_integral=True,
interpolate_res=None,
snap_exit=True):
assert_positive_real(distance, 'distance')
assert_real(start_time, 'start_time')
assert_real(end_time, 'end_time')
if not (start_time < end_time):
raise ValueError(f"start_time should be less than end_time")
assert len(entrances) == len(entrance_weights)
self.start_time = start_time
self.end_time = end_time
self.dist_around = distance
self.entrances = entrances
self.entrance_weights = entrance_weights
self.rand_velocities_and_distances = rand_velocities_and_distances_func
self._snap_exit = snap_exit
if interpolate_rate or interpolate_rate_integral:
if interpolate_res is None:
raise ValueError("Specify interpolate_res for interpolation")
if interpolate_rate:
self.entrance_rate = DynamicBoundedInterpolator(
entrance_rate, start_time, end_time, interpolate_res)
else:
self.entrance_rate = entrance_rate
if interpolate_rate_integral: # Want to interplate the integral function
if entrance_rate_integral is None: # No integral function given
# Do numerical integration and interpolate to speed it up
def integral_func(t):
y, abserr = integrate.quad(entrance_rate, start_time, t)
return y
self.entrance_rate_integral = DynamicBoundedInterpolator(
integral_func, start_time, end_time, interpolate_res)
else: # Integral function was provided
# Use the provided rate integral function but interpolate it
self.entrance_rate_integral = DynamicBoundedInterpolator(
entrance_rate_integral, start_time, end_time, interpolate_res)
else: # Don't want to interpolate the integral function
# If entrance_rate_integral is not None (i.e. is provided) then
# that function will be used as the rate integral.
# If entrance_rate_integral is None, numerical integration will
# be used.
self.entrance_rate_integral = entrance_rate_integral
self.entrance_rate_integral_inverse = entrance_rate_integral_inverse
self.pedestrians = SortedKeyList(key=attrgetter('start_time'))
self._counts = SortedDict()
self._counts[self.start_time] = 0
self._counts_are_correct = True
self.refresh_pedestrians()
def _distance(self, a, b):
"""signed distance of a relative to b"""
return circular_diff(a % self.dist_around, b % self.dist_around,
self.dist_around)
def _distance_from(self, b):
"""returns a function that returns the signed sitance from b"""
return lambda a: self._distance(a, b)
def _abs_distance_from(self, b):
"""returns a function that returns the distance from b"""
return lambda a: abs(self._distance(a, b))
def _closest_exit(self, dist):
"""Returns the closest number to dist that is equivalent mod dist_around
to an element of entrances"""
closest_exit = min(self.entrances, key=self._abs_distance_from(dist))
diff = self._distance(closest_exit, dist)
corrected_dist = dist + diff
return corrected_dist
def refresh_pedestrians(self):
"""Refreshes the pedestrians in the simulation to random ones"""
self.clear_pedestrians()
start_times = list(
random_times(self.start_time, self.end_time,
self.entrance_rate,
self.entrance_rate_integral,
self.entrance_rate_integral_inverse))
n_pedestrians = len(start_times)
entrances = random.choices(population=self.entrances,
weights=self.entrance_weights,
k=n_pedestrians)
velocities, distances = self.rand_velocities_and_distances(
n_pedestrians).T
def pedestrians_generator():
for start_time, entrance, velocity, dist in zip(
start_times, entrances, velocities, distances):
assert dist > 0
if self._snap_exit:
original_exit = entrance + dist * sign(velocity)
corrected_exit = self._closest_exit(original_exit)
corrected_dist = abs(corrected_exit - entrance)
if math.isclose(corrected_dist, 0, abs_tol=1e-10):
corrected_dist = self.dist_around
else:
corrected_dist = dist
yield FreshPondPedestrian(self.dist_around, entrance,
corrected_dist, start_time, velocity)
self.add_pedestrians(pedestrians_generator())
def clear_pedestrians(self):
"""Removes all pedestrains in the simulation"""
self.pedestrians.clear()
self._reset_counts()
self._counts_are_correct = True
def add_pedestrians(self, pedestrians):
"""Adds all the given pedestrians to the simulation"""
def checked_pedestrians():
for p in pedestrians:
self._assert_pedestrian_in_range(p)
yield p
initial_num_pedestrians = self.num_pedestrians()
self.pedestrians.update(checked_pedestrians())
final_num_pedestrians = self.num_pedestrians()
if final_num_pedestrians > initial_num_pedestrians:
self._counts_are_correct = False
else:
assert final_num_pedestrians == initial_num_pedestrians
def _assert_pedestrian_in_range(self, p):
"""Makes sure the pedestrian's start time is in the simulation's
time interval"""
if not (self.start_time <= p.start_time < self.end_time):
raise ValueError(
"Pedestrian start time is not in range [start_time, end_time)")
def add_pedestrian(self, p):
"""Adds a new pedestrian to the simulation"""
self._assert_pedestrian_in_range(p)
self.pedestrians.add(p)
# Update counts only when counts are correct
if self._counts_are_correct:
# add a new breakpoint at the pedestrian's start time if it not there
self._counts[p.start_time] = self.n_people(p.start_time)
# add a new breakpoint at the pedestrian's end time if it not there
self._counts[p.end_time] = self.n_people(p.end_time)
# increment all the counts in the pedestrian's interval of time
# inclusive on the left, exclusive on the right
# If it were inclusive on the right, then the count would be one more
# than it should be in the period after end_time and before the next
# breakpoint after end_time
for t in self._counts.irange(p.start_time,
p.end_time,
inclusive=(True, False)):
self._counts[t] += 1
def _reset_counts(self):
"""Clears _counts and sets count at start_time to 0"""
self._counts.clear()
self._counts[self.start_time] = 0
def _recompute_counts(self):
"""Store how many people there are whenever someone enters or exits so
the number of people at a given time can be found quickly later"""
# print("Recomputing counts")
self._reset_counts()
if self.num_pedestrians() == 0:
return
# pedestrians are already sorted by start time
start_times = [p.start_time for p in self.pedestrians]
end_times = sorted([p.end_time for p in self.pedestrians])
n = len(start_times)
curr_count = 0 # current number of people
start_times_index = 0
end_times_index = 0
starts_done = False # whether all the start times have been added
ends_done = False # whether all the end times have been added
while not (starts_done and ends_done):
# determine whether a start time or an end time should be added next
# store this in the variable take_start which is true if a start
# time should be added next
if starts_done:
# already added all the start times; add an end time
take_start = False
elif ends_done:
# already added all the end times; add a start time
take_start = True
else:
# didn't add all the end times nor all the start times
# add the time that is earliest
next_start_time = start_times[start_times_index]
next_end_time = end_times[end_times_index]
take_start = next_start_time < next_end_time
if take_start:
# add next start
curr_count += 1
start_time = start_times[start_times_index]
self._counts[start_time] = curr_count
start_times_index += 1
if start_times_index == n:
starts_done = True
else:
# add next end
curr_count -= 1
end_time = end_times[end_times_index]
self._counts[end_time] = curr_count
end_times_index += 1
if end_times_index == n:
ends_done = True
def n_unique_people_saw(self, p):
"""Returns the number of unique people that a pedestrian sees"""
n = 0
for q in self.pedestrians:
if p.intersects(q):
n += 1
return n
def n_people_saw(self, p):
"""Returns the number of times a pedestrian sees someone"""
n = 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
n += p.n_intersections(q)
return n
def intersection_directions(self, p):
"""Returns the number of people seen going in the same direction and the
number of people seen going in the opposite direction by p as a tuple"""
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
d = q.intersection_direction(p)
if d == 1:
n_same += 1
elif d == -1:
n_diff += 1
return n_same, n_diff
def intersection_directions_total(self, p):
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
i = p.total_intersection_direction(q)
if i < 0:
n_diff += -i
elif i > 0:
n_same += i
return n_same, n_diff
def n_people(self, t):
"""Returns the number of people at a given time"""
if not self._counts_are_correct:
self._recompute_counts()
self._counts_are_correct = True
if t in self._counts:
return self._counts[t]
elif t < self.start_time:
return 0
else:
index = self._counts.bisect_left(t)
return self._counts.values()[index - 1]
def num_pedestrians(self):
"""Returns the total number of pedestrians in the simulation"""
return len(self.pedestrians)
def get_pedestrians_in_interval(self, start, stop):
"""Returns a list of all the pedestrians who entered in the interval
[start, stop]"""
return list(self.pedestrians.irange_key(start, stop))
def num_entrances_in_interval(self, start, stop):
"""Returns the number of pedestrians who entered in the given interval
of time [start, stop]"""
return len(self.get_pedestrians_in_interval(start, stop))
def get_enter_and_exit_times_in_interval(self, start, stop):
"""Returns the entrance and exit times in a given time interval
as a tuple of lists (entrance_times, exit_times)."""
start_times = []
end_times = []
for p in self.pedestrians:
if start <= p.start_time <= stop:
start_times.append(p.start_time)
if start <= p.end_time <= stop:
end_times.append(p.end_time)
return start_times, end_times
def get_pedestrians_at_time(self, t):
"""Returns a list of all the pedestrians who were there at time t"""
# get all pedestrians who entered at or before time t
entered_before_t = self.pedestrians.irange_key(
min_key=None, max_key=t, inclusive=(True, True))
# Of those, return return the ones who exited after time t
return [p for p in entered_before_t if p.end_time > t]
class TransactionsIndex:
""" Index of transactions sorted by their timestamps.
"""
transactions: 'SortedKeyList[TransactionIndexElement]'
def __init__(self) -> None:
self.transactions = SortedKeyList(key=lambda x: (x.timestamp, x.hash))
def __getitem__(self, index: slice) -> List[TransactionIndexElement]:
""" Get items from SortedKeyList given a slice
:param index: list index slice, for eg [1:6]
"""
return self.transactions[index]
def update(self, values: List[TransactionIndexElement]) -> None:
""" Update sorted list by adding all values from iterable
:param values: new values to add to SortedKeyList
"""
self.transactions.update(values)
def add_tx(self, tx: BaseTransaction) -> None:
""" Add a transaction to the index
:param tx: Transaction to be added
"""
assert tx.hash is not None
# It is safe to use the in operator because it is O(log(n)).
# http://www.grantjenks.com/docs/sortedcontainers/sortedlist.html#sortedcontainers.SortedList.__contains__
element = TransactionIndexElement(tx.timestamp, tx.hash)
if element in self.transactions:
return
self.transactions.add(element)
def del_tx(self, tx: BaseTransaction) -> None:
""" Delete a transaction from the index
:param tx: Transaction to be deleted
"""
idx = self.transactions.bisect_key_left((tx.timestamp, tx.hash))
if idx < len(self.transactions) and self.transactions[idx].hash == tx.hash:
self.transactions.pop(idx)
def find_tx_index(self, tx: BaseTransaction) -> Optional[int]:
"""Return the index of a transaction in the index
:param tx: Transaction to be found
"""
idx = self.transactions.bisect_key_left((tx.timestamp, tx.hash))
if idx < len(self.transactions) and self.transactions[idx].hash == tx.hash:
return idx
return None
def get_newest(self, count: int) -> Tuple[List[bytes], bool]:
""" Get transactions or blocks from the newest to the oldest
:param count: Number of transactions or blocks to be returned
:return: List of tx hashes and a boolean indicating if has more txs
"""
return get_newest_sorted_key_list(self.transactions, count)
def get_older(self, timestamp: int, hash_bytes: bytes, count: int) -> Tuple[List[bytes], bool]:
""" Get transactions or blocks from the timestamp/hash_bytes reference to the oldest
:param timestamp: Timestamp reference to start the search
:param hash_bytes: Hash reference to start the search
:param count: Number of transactions or blocks to be returned
:return: List of tx hashes and a boolean indicating if has more txs
"""
return get_older_sorted_key_list(self.transactions, timestamp, hash_bytes, count)
def get_newer(self, timestamp: int, hash_bytes: bytes, count: int) -> Tuple[List[bytes], bool]:
""" Get transactions or blocks from the timestamp/hash_bytes reference to the newest
:param timestamp: Timestamp reference to start the search
:param hash_bytes: Hash reference to start the search
:param count: Number of transactions or blocks to be returned
:return: List of tx hashes and a boolean indicating if has more txs
"""
return get_newer_sorted_key_list(self.transactions, timestamp, hash_bytes, count)
def find_first_at_timestamp(self, timestamp: int) -> int:
""" Get index of first element at the given timestamp, or where it would be inserted if
the timestamp is not in the list.
Eg: SortedKeyList = [(3,hash1), (3, hash2), (7, hash3), (8, hash4)]
find_first_at_timestamp(7) = 2, which is the index of (7, hash3)
find_first_at_timestamp(4) = 2, which is the index of (7, hash3)
:param timestamp: timestamp we're interested in
:return: the index of the element, or None if timestamp is greater than all in the list
"""
idx = self.transactions.bisect_key_left((timestamp, b''))
return idx
class SortedIntvls:
""" """
def __init__(self):
# we sort by increasing start offset then increasing annotation id for this
self._by_start = SortedKeyList(key=lambda x: (x[0], x[2]))
# for this we sort by end offset only
self._by_end = SortedKeyList(key=lambda x: x[1])
def add(self, start, end, data):
"""
Adds an interval.
"""
self._by_start.add((start, end, data))
self._by_end.add((start, end, data))
def update(self, tupleiterable):
"""
Updates from an iterable of intervals.
"""
self._by_start.update(tupleiterable)
self._by_end.update(tupleiterable)
def remove(self, start, end, data):
"""
Removes an interval, exception if the interval does not exist.
"""
self._by_start.remove((start, end, data))
self._by_end.remove((start, end, data))
def discard(self, start, end, data):
"""
Removes and interval, do nothing if the interval does not exist.
"""
self._by_start.discard((start, end, data))
self._by_end.discard((start, end, data))
def __len__(self):
"""
Returns the number of intervals.
"""
return len(self._by_start)
def starting_at(self, offset):
"""
Returns an iterable of (start, end, data) tuples where start==offset
"""
return self._by_start.irange_key(min_key=(offset, 0),
max_key=(offset, sys.maxsize))
def ending_at(self, offset):
"""
Returns an iterable of (start, end, data) tuples where end==offset
"""
return self._by_end.irange_key(min_key=offset, max_key=offset)
def at(self, start, end):
"""
Returns an iterable of tuples where start==start and end==end
"""
for intvl in self._by_start.irange_key(min_key=(start, 0),
max_key=(start, sys.maxsize)):
if intvl[1] == end:
yield intvl
def within(self, start, end):
"""
Returns intervals which are fully contained within start...end
"""
# get all the intervals that start within the range, then keep those which also end within the range
for intvl in self._by_start.irange_key(min_key=(start, 0),
max_key=(end, sys.maxsize)):
if intvl[1] <= end:
yield intvl
def starting_from(self, offset):
"""
Returns intervals that start at or after offset.
"""
return self._by_start.irange_key(min_key=(offset, 0))
def starting_before(self, offset):
"""
Returns intervals that start before offset.
"""
return self._by_start.irange_key(max_key=(offset - 1, sys.maxsize))
def ending_to(self, offset):
"""
Returns intervals that end before or at the given end offset.
"""
return self._by_end.irange_key(max_key=offset)
def ending_after(self, offset):
"""
Returns intervals the end after the given offset.
"""
return self._by_end.irange_key(min_key=offset + 1)
def covering(self, start, end):
"""
Returns intervals that contain the given range.
"""
# All intervals that start at or before the start and end at or after the end offset
# we do this by first getting the intervals the start before or atthe start
# then filtering by end
for intvl in self._by_start.irange_key(max_key=(start, sys.maxsize)):
if intvl[1] >= end:
yield intvl
def overlapping(self, start, end):
"""
Returns intervals that overlap with the given range.
"""
# All intervals where the start or end offset lies within the given range.
# This excludes the ones where the end offset is before the start or
# where the start offset is after the end of the range.
# Here we do this by looking at all intervals where the start offset is before the
# end of the range. This still includes those which also end before the start of the range
# so we check in addition that the end is larger than the start of the range.
for intvl in self._by_start.irange_key(max_key=(end - 1, sys.maxsize)):
if intvl[1] > start + 1:
yield intvl
def firsts(self):
"""
Yields all intervals which start at the smallest known offset.
"""
laststart = None
# logger.info("DEBUG: set laststart to None")
for intvl in self._by_start.irange_key():
# logger.info("DEBUG: checking interval {}".format(intvl))
if laststart is None:
laststart = intvl[0]
# logger.info("DEBUG: setting laststart to {} and yielding {}".format(intvl[0], intvl))
yield intvl
elif intvl[0] == laststart:
# logger.info("DEBUG: yielding {}".format(intvl))
yield intvl
else:
# logger.info("DEBUG: returning since we got {}".format(intvl))
return
def lasts(self):
"""
Yields all intervals which start at the last known start offset.
"""
laststart = None
for intvl in reversed(self._by_start):
if laststart is None:
laststart = intvl[0]
yield intvl
elif intvl[0] == laststart:
yield intvl
else:
return
def min_start(self):
"""
Returns the smallest known start offset.
"""
return self._by_start[0][0]
def max_end(self):
"""
Returns the biggest known end offset.
"""
return self._by_end[-1][1]
def irange(self,
minoff=None,
maxoff=None,
reverse=False,
inclusive=(True, True)):
"""
Yields an iterator of intervals with a start offset between minoff and maxoff, inclusive.
Args:
minoff: minimum offset, default None indicates any
maxoff: maximum offset, default None indicates any
reverse: if `True` yield in reverse order
inclusive: if the minoff and maxoff values should be inclusive, default is (True,True)
Returns:
"""
return self._by_start.irange_key(min_key=minoff,
max_key=maxoff,
reverse=reverse,
inclusive=inclusive)
def __repr__(self):
return "SortedIntvls({},{})".format(self._by_start, self._by_end)
