def test_eq():
this = SortedKeyList(range(10), key=negate)
this._reset(4)
that = SortedKeyList(range(20), key=negate)
that._reset(4)
assert not (this == that)
that.clear()
that.update(range(10))
assert this == that
def test_eq():
this = SortedKeyList(range(10), key=negate)
this._reset(4)
that = SortedKeyList(range(20), key=negate)
that._reset(4)
assert not (this == that)
that.clear()
that.update(range(10))
assert this == that
class HalfSnap:
def __init__(self, bids: bool):
if bids:
self.data = SortedKeyList(key=lambda val: -val[0])
else:
self.data = SortedKeyList(key=lambda val: val[0])
self.is_bids = bids
self.time = None
def fill(self, source):
self.data.clear()
for item in source:
self.add(item)
def add(self, item):
price = item[0]
size = item[1]
self.data.add([price, size])
def update(self, price: float, size: float):
key = -price if self.is_bids else price
i = self.data.bisect_key_left(key)
if 0 <= i < len(self.data):
value = self.data[i]
else:
if size <= VERY_SMALL_NUMBER:
return False
self.data.add([price, size])
return True
if size <= VERY_SMALL_NUMBER:
if value[0] == price:
self.data.discard(value)
return True
else:
return False
if value[0] == price:
self.data[i][1] = size
else:
self.data.add([price, size])
return True
def delete(self, price: float):
return self.updatef(price, 0.0)
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=modulo)
slt._reset(17)
slt.add(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=modulo)
slt._reset(17)
slt.add(5)
slt._build_index()
slt._check()
slt.clear()
lst = list(random.random() for rpt in range(100))
slt.update(lst)
lst.sort(key=modulo)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_init():
slt = SortedKeyList(key=negate)
slt._check()
slt = SortedKeyList(key=negate)
slt._reset(10000)
assert slt._load == 10000
slt._check()
slt = SortedKeyList(range(10000), key=negate)
assert all(tup[0] == tup[1] for tup in zip(slt, reversed(range(10000))))
slt.clear()
assert slt._len == 0
assert slt._maxes == []
assert slt._lists == []
slt._check()
def test_init():
slt = SortedKeyList(key=negate)
slt._check()
slt = SortedKeyList(key=negate)
slt._reset(10000)
assert slt._load == 10000
slt._check()
slt = SortedKeyList(range(10000), key=negate)
assert all(tup[0] == tup[1] for tup in zip(slt, reversed(range(10000))))
slt.clear()
assert slt._len == 0
assert slt._maxes == []
assert slt._lists == []
slt._check()
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=negate)
slt._reset(17)
slt.add(5)
assert slt[0] == 5
slt.clear()
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort(reverse=True)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_getitem():
random.seed(0)
slt = SortedKeyList(key=negate)
slt._reset(17)
slt.add(5)
assert slt[0] == 5
slt.clear()
lst = list()
for rpt in range(100):
val = random.random()
slt.add(val)
lst.append(val)
lst.sort(reverse=True)
assert all(slt[idx] == lst[idx] for idx in range(100))
assert all(slt[idx - 99] == lst[idx - 99] for idx in range(100))
def test_init():
slt = SortedKeyList(key=modulo)
assert slt.key == modulo
slt._check()
slt = SortedKeyList(key=modulo)
slt._reset(10000)
assert slt._load == 10000
slt._check()
slt = SortedKeyList(range(10000), key=modulo)
assert all(tup[0] == tup[1] for tup in zip(slt, sorted(range(10000), key=modulo)))
slt.clear()
assert slt._len == 0
assert slt._maxes == []
assert slt._lists == []
assert isinstance(slt, SortedList)
assert isinstance(slt, SortedKeyList)
slt._check()
def test_init():
slt = SortedKeyList(key=modulo)
assert slt.key == modulo
slt._check()
slt = SortedKeyList(key=modulo)
slt._reset(10000)
assert slt._load == 10000
slt._check()
slt = SortedKeyList(range(10000), key=modulo)
assert all(tup[0] == tup[1]
for tup in zip(slt, sorted(range(10000), key=modulo)))
slt.clear()
assert slt._len == 0
assert slt._maxes == []
assert slt._lists == []
assert isinstance(slt, SortedList)
assert isinstance(slt, SortedKeyList)
slt._check()
class PriorityQueue:
def __init__(self, capacity=None, key=None):
self._data = SortedKeyList(key=self._rank)
self._capacity = inf if capacity is None else capacity
self._key = key
def _rank(self, item):
if self._key:
return self._key(*item)
return item.rank
def add(self, value, rank):
self._data.add(Element(value, rank))
self._shrink()
def clear(self):
return self._data.clear()
def __repr__(self):
return f"PriorityQueue([{', '.join(f'{v}: {r}' for (v, r) in self._data)}])"
def _shrink(self):
while len(self._data) > self._capacity:
self._data.pop()
def update(self, src):
raise NotImplementedError
def __contains__(self, value):
raise NotImplementedError
def __iter__(self):
for value, rank in self._data:
yield value
def __getitem__(self, index):
if isinstance(index, int):
return self._data[index].value
return list(self)[index]
def size(self):
return len(self._data)
class MyMongoCollection:
def __init__(self, database, name: str):
from mymongoDB import MyMongoDB
if not isinstance(database, MyMongoDB):
raise MongoException(
"Only MongoDB objects can be passed as the database argument")
self.name: str = name
self.parent_database: MyMongoDB = database
# сортирован по ключам словарей
self.__docs: SortedKeyList[MongoId, MyMongoDoc] = SortedKeyList(
key=lambda doc: doc['objectId'])
self.__indices: Dict[FieldName, SortedKeyList] = SortedDict()
self.__reserved_ids: MutableSet = SortedSet(set())
def __repr__(self) -> str:
meta = f"MyMongoCollection({repr(self.parent_database)}, {self.name})"
return meta
def __len__(self) -> int:
return len(self.__docs)
def __getstate__(self):
return self.__dict__
def __setstate__(self, state):
self.__dict__ = state
def sort(self, key):
self.__docs = SortedKeyList(self.__docs, key=key)
for field in self.__indices.keys():
self.create_index(field)
def create_index(self, field: str) -> None:
if not isinstance(field, str):
raise MongoException("'Field' argument must be a string")
# только документы с данным полем
relevant_docs: List[MyMongoDoc] = [
doc for doc in self.__docs if field in doc.keys()
]
self.__indices[field] = SortedKeyList(relevant_docs,
key=lambda doc: doc[field])
def insert_one(self, doc: Union[Dict,
MutableMapping]) -> MongoLastInserted:
if not (isinstance(doc, dict)
or issubclass(doc.__class__, MutableMapping)):
raise MongoException(
f"Document \n{doc}\n must be an instance of dict"
"or a type that inherits from collections.MutableMapping")
new_doc = _MyMongoDocFactory.get_doc(data=doc)
if new_doc.objectId in self.__reserved_ids:
raise MongoException(
f"Duplicate key error: document already in collection {new_doc}"
)
else:
self.__reserved_ids.add(new_doc.objectId)
self.__docs.add(new_doc)
return MongoLastInserted(new_doc)
def insert_many(self, *docs) -> List[MyMongoDoc]:
last: List[MyMongoDoc] = list()
if isinstance(docs[0], dict) or issubclass(docs[0].__class__,
MutableMapping):
pass
# любой другой iterable на свой страх и риск
elif hasattr(docs[0], "__iter__") and len(docs) == 1:
docs = docs[0]
else:
raise MongoException(
f"Function accepts iterables of dicts or a type that inherits from "
"collections.MutableMapping, or simply non-keyword arguments.")
for doc in docs:
last.append(self.insert_one(doc).document)
return last
def delete_one(self, object_id: MongoId) -> None:
try:
if isinstance(object_id, MongoId):
self.__reserved_ids.remove(object_id)
# найдём объект для удаления в теле коллекции документов по его MongoId, используя
# куклу с таким же индексом
dummy = {"objectId": object_id}
object_idx: int = self.__docs.bisect_left(dummy)
doc: MyMongoDoc = self.__docs[object_idx]
# очистим индексы от удаляемого объекта
for field in self.__indices.keys():
if field in doc.keys():
obj_idx = self.__indices[field].bisect_left(doc)
self.__indices[field].remove(obj_idx)
self.__docs.pop(object_idx)
else:
raise TypeError(
"Only instances of MongoId can serve as document identifiers."
)
except KeyError as e:
raise KeyError(
f"Collection {self.name} has no object with id {object_id}.")
def delete_many(self, object_ids: Iterable[MongoId]):
for object_id in object_ids:
self.delete_one(object_id)
def clear(self):
self.__docs.clear()
self.__indices.clear()
self.__reserved_ids.clear()
def find_one(self, query: Dict[str, Any]):
result = None
relevant_docs, _query = self.__get_relevant_info(query)
if len(_query) == 0:
return relevant_docs
else:
relevant_docs = iter(relevant_docs)
while result is None:
candidate = next(relevant_docs)
try:
boolean = [
candidate[key] == value for key, value in _query.items()
]
if all(boolean):
result = candidate
except KeyError:
pass
return result
def find(self, query: Dict[str, Any]) -> Iterable[MyMongoDoc]:
result = list()
relevant_docs, _query = self.__get_relevant_info(query)
if len(_query) == 0:
return list(relevant_docs)
else:
relevant_docs = iter(relevant_docs)
for candidate in relevant_docs:
try:
boolean = [
candidate[key] == value for key, value in _query.items()
]
if all(boolean):
result.append(candidate)
except KeyError:
pass
return list(result)
def find_and_update(self, filter_, update_: Dict[FieldName, Any]) -> None:
docs = self.find(filter_)
for doc in docs:
for k, v in update_.items():
doc[k] = v
return docs
def find_one_and_update(self, filter_, update_: Dict[FieldName,
Any]) -> None:
doc = self.find_one(filter_)
for k, v in update_.items():
doc[k] = v
return doc
def query(
self, where: MutableMapping[FieldName, Callable[..., Bool]]
) -> Iterable[MyMongoDoc]:
result = list()
relevant_docs, _query = self.__get_relevant_info(where)
if len(_query) == 0:
return list(relevant_docs)
else:
# уменьшаем количество документов для поиска
relevant_docs = iter(relevant_docs)
for candidate in relevant_docs:
try:
boolean = [
function(candidate[key])
for key, function in _query.items()
]
if all(boolean):
result.append(candidate)
except KeyError:
pass
return list(result)
def __get_relevant_info(self, query) -> Tuple[Iterable, MutableMapping]:
relevant_docs = set()
query_ = deepcopy(query)
indexed_query_fields = self.__indices.keys() & query_.keys()
# Авось придёт запрос по индексирвованным полям
if len(indexed_query_fields) != 0:
# если по полям составлен индекс, то учитывая, что все элементы query логически
# связаны оператором AND, то для начала можно просто вынуть пересечение документов, удовлетворяющих
# требованиям к индексированным полям.
for indexed_query_field in indexed_query_fields:
# marker - dummy объект, словарик с искомым полем и значением. Мы ищем с логарифмической сложностью,
# куда его можно приткнуть в наш индекс (находим точку до документов с идентичным значением в
# искомом поле), а затем извлекаем 0+ равнозначных (в рамках искомого поля) объектов
marker = {indexed_query_field: query[indexed_query_field]}
index: SortedKeyList = self.__indices[indexed_query_field]
# Return an index to insert value in the sorted list. If the value is already present,
# the insertion point will be before (to the left of) any
# existing values.
idx = index.bisect_left(marker)
while idx < len(index) and query[indexed_query_field] == index[
idx][indexed_query_field]:
relevant_docs.add(index[idx])
idx += 1
# мы можем больше не смотреть на поля, по которым имеется индекс
query_ = {
k: v
for k, v in query.items() if k not in indexed_query_fields
}
else:
# Что поделать, раз уж запрос такой?
relevant_docs = self.__docs
return relevant_docs, query_
class FreshPondSim:
def __init__(self,
distance,
start_time,
end_time,
entrances,
entrance_weights,
rand_velocities_and_distances_func,
entrance_rate,
entrance_rate_integral=None,
entrance_rate_integral_inverse=None,
interpolate_rate=True,
interpolate_rate_integral=True,
interpolate_res=None,
snap_exit=True):
assert_positive_real(distance, 'distance')
assert_real(start_time, 'start_time')
assert_real(end_time, 'end_time')
if not (start_time < end_time):
raise ValueError(f"start_time should be less than end_time")
assert len(entrances) == len(entrance_weights)
self.start_time = start_time
self.end_time = end_time
self.dist_around = distance
self.entrances = entrances
self.entrance_weights = entrance_weights
self.rand_velocities_and_distances = rand_velocities_and_distances_func
self._snap_exit = snap_exit
if interpolate_rate or interpolate_rate_integral:
if interpolate_res is None:
raise ValueError("Specify interpolate_res for interpolation")
if interpolate_rate:
self.entrance_rate = DynamicBoundedInterpolator(
entrance_rate, start_time, end_time, interpolate_res)
else:
self.entrance_rate = entrance_rate
if interpolate_rate_integral: # Want to interplate the integral function
if entrance_rate_integral is None: # No integral function given
# Do numerical integration and interpolate to speed it up
def integral_func(t):
y, abserr = integrate.quad(entrance_rate, start_time, t)
return y
self.entrance_rate_integral = DynamicBoundedInterpolator(
integral_func, start_time, end_time, interpolate_res)
else: # Integral function was provided
# Use the provided rate integral function but interpolate it
self.entrance_rate_integral = DynamicBoundedInterpolator(
entrance_rate_integral, start_time, end_time, interpolate_res)
else: # Don't want to interpolate the integral function
# If entrance_rate_integral is not None (i.e. is provided) then
# that function will be used as the rate integral.
# If entrance_rate_integral is None, numerical integration will
# be used.
self.entrance_rate_integral = entrance_rate_integral
self.entrance_rate_integral_inverse = entrance_rate_integral_inverse
self.pedestrians = SortedKeyList(key=attrgetter('start_time'))
self._counts = SortedDict()
self._counts[self.start_time] = 0
self._counts_are_correct = True
self.refresh_pedestrians()
def _distance(self, a, b):
"""signed distance of a relative to b"""
return circular_diff(a % self.dist_around, b % self.dist_around,
self.dist_around)
def _distance_from(self, b):
"""returns a function that returns the signed sitance from b"""
return lambda a: self._distance(a, b)
def _abs_distance_from(self, b):
"""returns a function that returns the distance from b"""
return lambda a: abs(self._distance(a, b))
def _closest_exit(self, dist):
"""Returns the closest number to dist that is equivalent mod dist_around
to an element of entrances"""
closest_exit = min(self.entrances, key=self._abs_distance_from(dist))
diff = self._distance(closest_exit, dist)
corrected_dist = dist + diff
return corrected_dist
def refresh_pedestrians(self):
"""Refreshes the pedestrians in the simulation to random ones"""
self.clear_pedestrians()
start_times = list(
random_times(self.start_time, self.end_time,
self.entrance_rate,
self.entrance_rate_integral,
self.entrance_rate_integral_inverse))
n_pedestrians = len(start_times)
entrances = random.choices(population=self.entrances,
weights=self.entrance_weights,
k=n_pedestrians)
velocities, distances = self.rand_velocities_and_distances(
n_pedestrians).T
def pedestrians_generator():
for start_time, entrance, velocity, dist in zip(
start_times, entrances, velocities, distances):
assert dist > 0
if self._snap_exit:
original_exit = entrance + dist * sign(velocity)
corrected_exit = self._closest_exit(original_exit)
corrected_dist = abs(corrected_exit - entrance)
if math.isclose(corrected_dist, 0, abs_tol=1e-10):
corrected_dist = self.dist_around
else:
corrected_dist = dist
yield FreshPondPedestrian(self.dist_around, entrance,
corrected_dist, start_time, velocity)
self.add_pedestrians(pedestrians_generator())
def clear_pedestrians(self):
"""Removes all pedestrains in the simulation"""
self.pedestrians.clear()
self._reset_counts()
self._counts_are_correct = True
def add_pedestrians(self, pedestrians):
"""Adds all the given pedestrians to the simulation"""
def checked_pedestrians():
for p in pedestrians:
self._assert_pedestrian_in_range(p)
yield p
initial_num_pedestrians = self.num_pedestrians()
self.pedestrians.update(checked_pedestrians())
final_num_pedestrians = self.num_pedestrians()
if final_num_pedestrians > initial_num_pedestrians:
self._counts_are_correct = False
else:
assert final_num_pedestrians == initial_num_pedestrians
def _assert_pedestrian_in_range(self, p):
"""Makes sure the pedestrian's start time is in the simulation's
time interval"""
if not (self.start_time <= p.start_time < self.end_time):
raise ValueError(
"Pedestrian start time is not in range [start_time, end_time)")
def add_pedestrian(self, p):
"""Adds a new pedestrian to the simulation"""
self._assert_pedestrian_in_range(p)
self.pedestrians.add(p)
# Update counts only when counts are correct
if self._counts_are_correct:
# add a new breakpoint at the pedestrian's start time if it not there
self._counts[p.start_time] = self.n_people(p.start_time)
# add a new breakpoint at the pedestrian's end time if it not there
self._counts[p.end_time] = self.n_people(p.end_time)
# increment all the counts in the pedestrian's interval of time
# inclusive on the left, exclusive on the right
# If it were inclusive on the right, then the count would be one more
# than it should be in the period after end_time and before the next
# breakpoint after end_time
for t in self._counts.irange(p.start_time,
p.end_time,
inclusive=(True, False)):
self._counts[t] += 1
def _reset_counts(self):
"""Clears _counts and sets count at start_time to 0"""
self._counts.clear()
self._counts[self.start_time] = 0
def _recompute_counts(self):
"""Store how many people there are whenever someone enters or exits so
the number of people at a given time can be found quickly later"""
# print("Recomputing counts")
self._reset_counts()
if self.num_pedestrians() == 0:
return
# pedestrians are already sorted by start time
start_times = [p.start_time for p in self.pedestrians]
end_times = sorted([p.end_time for p in self.pedestrians])
n = len(start_times)
curr_count = 0 # current number of people
start_times_index = 0
end_times_index = 0
starts_done = False # whether all the start times have been added
ends_done = False # whether all the end times have been added
while not (starts_done and ends_done):
# determine whether a start time or an end time should be added next
# store this in the variable take_start which is true if a start
# time should be added next
if starts_done:
# already added all the start times; add an end time
take_start = False
elif ends_done:
# already added all the end times; add a start time
take_start = True
else:
# didn't add all the end times nor all the start times
# add the time that is earliest
next_start_time = start_times[start_times_index]
next_end_time = end_times[end_times_index]
take_start = next_start_time < next_end_time
if take_start:
# add next start
curr_count += 1
start_time = start_times[start_times_index]
self._counts[start_time] = curr_count
start_times_index += 1
if start_times_index == n:
starts_done = True
else:
# add next end
curr_count -= 1
end_time = end_times[end_times_index]
self._counts[end_time] = curr_count
end_times_index += 1
if end_times_index == n:
ends_done = True
def n_unique_people_saw(self, p):
"""Returns the number of unique people that a pedestrian sees"""
n = 0
for q in self.pedestrians:
if p.intersects(q):
n += 1
return n
def n_people_saw(self, p):
"""Returns the number of times a pedestrian sees someone"""
n = 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
n += p.n_intersections(q)
return n
def intersection_directions(self, p):
"""Returns the number of people seen going in the same direction and the
number of people seen going in the opposite direction by p as a tuple"""
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
d = q.intersection_direction(p)
if d == 1:
n_same += 1
elif d == -1:
n_diff += 1
return n_same, n_diff
def intersection_directions_total(self, p):
n_same, n_diff = 0, 0
for q in self.pedestrians:
if p.end_time > q.start_time and p.start_time < q.end_time:
i = p.total_intersection_direction(q)
if i < 0:
n_diff += -i
elif i > 0:
n_same += i
return n_same, n_diff
def n_people(self, t):
"""Returns the number of people at a given time"""
if not self._counts_are_correct:
self._recompute_counts()
self._counts_are_correct = True
if t in self._counts:
return self._counts[t]
elif t < self.start_time:
return 0
else:
index = self._counts.bisect_left(t)
return self._counts.values()[index - 1]
def num_pedestrians(self):
"""Returns the total number of pedestrians in the simulation"""
return len(self.pedestrians)
def get_pedestrians_in_interval(self, start, stop):
"""Returns a list of all the pedestrians who entered in the interval
[start, stop]"""
return list(self.pedestrians.irange_key(start, stop))
def num_entrances_in_interval(self, start, stop):
"""Returns the number of pedestrians who entered in the given interval
of time [start, stop]"""
return len(self.get_pedestrians_in_interval(start, stop))
def get_enter_and_exit_times_in_interval(self, start, stop):
"""Returns the entrance and exit times in a given time interval
as a tuple of lists (entrance_times, exit_times)."""
start_times = []
end_times = []
for p in self.pedestrians:
if start <= p.start_time <= stop:
start_times.append(p.start_time)
if start <= p.end_time <= stop:
end_times.append(p.end_time)
return start_times, end_times
def get_pedestrians_at_time(self, t):
"""Returns a list of all the pedestrians who were there at time t"""
# get all pedestrians who entered at or before time t
entered_before_t = self.pedestrians.irange_key(
min_key=None, max_key=t, inclusive=(True, True))
# Of those, return return the ones who exited after time t
return [p for p in entered_before_t if p.end_time > t]
