def _split_subtree(tree: SortedList,
max_length: int) -> Tuple[SortedList, SortedList]:
"""
Split a tree by its median element into two smaller trees
:param tree: The tree to split
:param max_length: The size of the largest possible element in the trie in bits
:return: The tree with the smaller elements,
and the tree with the larger elements
"""
median = tree.bisect_right(
YFastTrie._calculate_representative(tree[len(tree) // 2],
max_length))
return SortedList(tree.islice(stop=median)), SortedList(
tree.islice(start=median))
def test_merge_subtrees(values): values = SortedList(values) split = randint(1, len(values) - 1) left_tree = SortedList(values.islice(stop=split)) right_tree = SortedList(values.islice(start=split)) new_left, new_right = YFastTrie._merge_subtrees(left_tree, right_tree, 2 * max_trie_entry_size) if len(values) <= 2 * max_trie_entry_size: assert new_right is None assert isinstance(new_left, SortedList) assert len(new_left) == len(values) else: assert isinstance(new_left, SortedList) assert isinstance(new_right, SortedList) assert len(new_left) + len(new_right) == len(values) assert YFastTrie._calculate_representative(max(new_left), max_trie_entry_size) < min(new_right)
class Leaderboard:
def __init__(self):
# Time Complexity: O(1) (init)
# Space Complexity: O(N) (overall)
self.scores = {}
self.sorted_scores = SortedList()
def addScore(self, playerId: int, score: int) -> None:
# Time Complexity: O(log N)
# Space Complexity: O(1)
if playerId in self.scores:
self.sorted_scores.remove(self.scores[playerId])
score += self.scores[playerId]
self.scores[playerId] = score
self.sorted_scores.add(score)
def top(self, K: int) -> int:
# Time Complexity: O(K) or O(N) (depending on implementation)
# Space Complexity: O(1)
start = max(0, len(self.scores) - K)
return sum(self.sorted_scores.islice(start))
def reset(self, playerId: int) -> None:
# Time Complexity: O(log N)
# Space Complexity: O(1)
self.addScore(playerId, -self.scores.get(playerId, 0))
class RangeList:
"""Add and query non-overlapping intervals. Intervals are semi-closed, e.g.
the interval [1, 3) contains the points {1, 2}.
"""
def __init__(self, init=None):
self.data = SortedList(init, key=lambda x: x[0])
def add(self, start, end):
left = self.data.bisect_right((start, 0))
if left > 0:
if self.data[left - 1][1] >= start:
start = self.data[left - 1][0]
left -= 1
right = self.data.bisect_right((end, 0))
if right > 0:
if self.data[right - 1][1] >= end:
end = self.data[right - 1][1]
for _ in range(right - left):
self.data.pop(left)
self.data.add((start, end))
def list(self):
return list(self.data)
def iter(self):
return self.data.islice(start=0)
def test_islice():
sl = SortedList(load=7)
assert [] == list(sl.islice())
values = list(range(53))
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop,
reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start,
reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
def test_merge_large_subtrees(values): values = SortedList(values) split = len(values) // 2 left_tree = SortedList(values.islice(stop=split)) right_tree = SortedList(values.islice(start=split)) new_left, new_right = YFastTrie._merge_subtrees(left_tree, right_tree, 2 * max_trie_entry_size) assert isinstance(new_left, SortedList) assert isinstance(new_right, SortedList) assert len(new_left) + len(new_right) == len(values) assert YFastTrie._calculate_representative(max(new_left), max_trie_entry_size) < min(new_right) split += 1 left_tree = SortedList(values.islice(stop=split)) right_tree = SortedList(values.islice(start=split)) new_left, new_right = YFastTrie._merge_subtrees(left_tree, right_tree, 2 * max_trie_entry_size) assert isinstance(new_left, SortedList) assert isinstance(new_right, SortedList) assert len(new_left) + len(new_right) == len(values) assert YFastTrie._calculate_representative(max(new_left), max_trie_entry_size) < min(new_right)
class ZSet:
def __init__(self):
self.mem2score = {}
self.scores = SortedList()
def __contains__(self, val):
return val in self.mem2score
def __setitem__(self, val, score):
self.add(val, score)
def __getitem__(self, key):
return self.mem2score[key]
def __len__(self):
return len(self.mem2score)
def __iter__(self):
def f():
for score, val in self.scores:
yield val
return f()
def __str__(self):
ans = []
return
def get(self, key, default=None):
return self.mem2score.get(key, default)
def add(self, val, score):
s_prev = self.mem2score.get(val, None)
if s_prev:
if s_prev == score:
return False
self.scores.remove((s_prev, val))
self.mem2score[val] = score
self.scores.add((score, val))
return True
def discard(self, key):
try:
score = self.mem2score.pop(key)
except KeyError:
return
self.scores.remove((score, key))
def items(self):
return self.mem2score.items()
def rank(self, member):
return self.scores.index((self.mem2score[member], member))
def islice_score(self, start, stop, reverse=False):
return self.scores.islice(start, stop, reverse)
class MovieRentingSystem(object):
def __init__(self, n, entries):
"""
:type n: int
:type entries: List[List[int]]
"""
self.movies = defaultdict(SortedList)
self.shops = defaultdict(dict)
self.renting = SortedList([])
for shop, movie, price in entries:
self.movies[movie].add((price, shop))
self.shops[shop][movie] = price
def search(self, movie):
"""
:type movie: int
:rtype: List[int]
"""
return [i[1] for i in list(self.movies[movie].islice(stop=5))]
def rent(self, shop, movie):
"""
:type shop: int
:type movie: int
:rtype: None
"""
price = self.shops[shop][movie]
self.movies[movie].discard((price, shop))
self.renting.add((price, shop, movie))
def drop(self, shop, movie):
"""
:type shop: int
:type movie: int
:rtype: None
"""
price = self.shops[shop][movie]
self.movies[movie].add((price, shop))
self.renting.discard((price, shop, movie))
def report(self):
"""
:rtype: List[List[int]]
"""
return [[x, y] for _, x, y in self.renting.islice(stop=5)]
def generateTestData(queryPositivePercentage, sampleSize, maxUniverse):
uncompressedList = []
queryList = SortedList([])
negativeList = []
samplePercentage = sampleSize / maxUniverse * 100
random.seed(datetime.now())
for x in range(1, maxUniverse):
if (random.randint(1, 100) <= samplePercentage):
uncompressedList.append(x)
if (random.randint(1, 100) <= queryPositivePercentage):
queryList.append(x)
elif (len(negativeList) > 0):
# you add some random element from the negative list
randomIndex = random.randrange(len(negativeList))
queryList.add(negativeList.pop(randomIndex))
else:
negativeList.append(x)
return (uncompressedList, list(queryList.islice(0, len(queryList))))
def get_top_recs(self, user_id, sim_users, n):
'''
Given user_id with its top k similar users, return top n
chefs with highest predicted ratings. Does not filter out
chefs that the user has already rated.
@param int user_id user to get n chef recommendations for
@param int sim_users the list of similar users for user_id
@param int n number of chef recommendations to return
@returns list top_chefs top n predicted rated chefs for user_id
'''
summation = None
for s_user in sim_users:
if summation == None:
summation = self.sparse_ratings_matrix[s_user]
else:
summation += self.sparse_ratings_matrix[s_user]
length = len(sim_users)
summation = summation/length
rated_chefs = summation[0].nonzero()[1] # list of chef ids with non-zero ratings
sorted_chefs = SortedList([])
for rated_chef_id in rated_chefs:
sorted_chefs.add(KeyValPair(rated_chef_id, summation[0, rated_chef_id]))
# give predicted rating, too, maybe
top_chefs = list(map(lambda x: x.key, list(sorted_chefs.islice(0, n, reverse=True))))
return top_chefs
# start = time.time()
# x = ChefRecommendationEngine()
# end = time.time()
# print(end - start)
# print(x.recs)
# start = time.time()
# x.receive_new_rating(32, 22, 2)
# end = time.time()
# print(end - start)
# print(x.recs)
def test_islice():
sl = SortedList(load=7)
assert [] == list(sl.islice())
values = list(range(53))
sl.update(values)
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop)) == values[start:stop]
for start in range(53):
for stop in range(53):
assert list(sl.islice(start, stop, reverse=True)) == values[start:stop][::-1]
for start in range(53):
assert list(sl.islice(start=start)) == values[start:]
assert list(sl.islice(start=start, reverse=True)) == values[start:][::-1]
for stop in range(53):
assert list(sl.islice(stop=stop)) == values[:stop]
assert list(sl.islice(stop=stop, reverse=True)) == values[:stop][::-1]
class ManagerNode(object):
def __init__(self, node, parent, position):
"""
:param node: the behavior that is represented by this ManagerNode
:type node: py_trees.behaviour.Behaviour
:param parent: the parent of the behavior that is represented by this ManagerNode
:type parent: py_trees.behaviour.Behaviour
:param position: the position of the node in the list of children of the parent
:type position: int
"""
self.node = node
self.parent = parent
self.position = position
self.disabled_children = SortedList()
self.enabled_children = SortedList()
def __lt__(self, other):
return self.position < other.position
def __gt__(self, other):
return self.position > other.position
def __eq__(self, other):
return self.node == other.node and self.parent == other.parent
def disable_child(self, manager_node):
"""
marks the given manager node as disabled in the internal tree representation and removes it to the behavior tree
:param manager_node:
:type manager_node: TreeManager.ManagerNode
:return:
"""
self.enabled_children.remove(manager_node)
self.disabled_children.add(manager_node)
if isinstance(self.node, PluginBehavior):
self.node.remove_plugin(manager_node.node.name)
else:
self.node.remove_child(manager_node.node)
def enable_child(self, manager_node):
"""
marks the given manager node as enabled in the internal tree representation and adds it to the behavior tree
:param manager_node:
:type manager_node: TreeManager.ManagerNode
:return:
"""
self.disabled_children.remove(manager_node)
self.enabled_children.add(manager_node)
if isinstance(self.node, PluginBehavior):
self.node.add_plugin(manager_node.node)
else:
idx = self.enabled_children.index(manager_node)
self.node.insert_child(manager_node.node, idx)
def add_child(self, manager_node):
"""
adds the given manager node to the internal tree map and the corresponding behavior to the behavior tree
:param manager_node:
:type manager_node: TreeManager.ManagerNode
:return:
"""
if isinstance(self.node, PluginBehavior):
self.enabled_children.add(manager_node)
self.node.add_plugin(manager_node.node)
else:
if manager_node.position < 0:
manager_node.position = 0
if self.enabled_children:
manager_node.position = max(
manager_node.position,
self.enabled_children[-1].position + 1)
if self.disabled_children:
manager_node.position = max(
manager_node.position,
self.disabled_children[-1].position + 1)
idx = manager_node.position
else:
idx = self.disabled_children.bisect_left(manager_node)
for c in self.disabled_children.islice(start=idx):
c.position += 1
idx = self.enabled_children.bisect_left(manager_node)
for c in self.enabled_children.islice(start=idx):
c.position += 1
self.node.insert_child(manager_node.node, idx)
self.enabled_children.add(manager_node)
def remove_child(self, manager_node):
"""
removes the given manager_node from the internal tree map and the corresponding behavior from the behavior tree
:param manager_node:
:type manager_node: TreeManager.ManagerNode
:return:
"""
if isinstance(self.node, PluginBehavior):
if manager_node in self.enabled_children:
self.enabled_children.remove(manager_node)
self.node.remove_plugin(manager_node.node.name)
elif manager_node in self.disabled_children:
self.disabled_children.remove(manager_node)
else:
raise RuntimeError(
'could not remove node from parent. this probably means that the tree is inconsistent'
)
else:
if manager_node in self.enabled_children:
self.enabled_children.remove(manager_node)
self.node.remove_child(manager_node.node)
elif manager_node in self.disabled_children:
self.disabled_children.remove(manager_node)
else:
raise RuntimeError(
'could not remove node. this probably means that the tree is inconsistent'
)
idx = self.disabled_children.bisect_right(manager_node)
for c in self.disabled_children.islice(start=idx):
c.position -= 1
idx = self.enabled_children.bisect_right(manager_node)
for c in self.enabled_children.islice(start=idx):
c.position -= 1
def process_swaps_for_eon(operator_eon_number):
checkpoint_created = RootCommitment.objects.filter(
eon_number=operator_eon_number).exists()
notification_queue = []
with transaction.atomic():
default_time = timezone.now() - datetime.timedelta(days=365000)
# Match swaps
last_unprocessed_swap_time = timezone.make_aware(
datetime.datetime.fromtimestamp(
cache.get_or_set('last_unprocessed_swap_time',
default_time.timestamp())))
unprocessed_swaps = Transfer.objects \
.filter(
time__gte=last_unprocessed_swap_time,
processed=False,
complete=False,
voided=False,
cancelled=False,
swap=True,
eon_number=operator_eon_number,
sender_active_state__operator_signature__isnull=False,
recipient_active_state__operator_signature__isnull=False) \
.select_for_update() \
.order_by('time')
order_books_cache = {}
for swap in unprocessed_swaps:
last_unprocessed_swap_time = max(
last_unprocessed_swap_time,
swap.time + datetime.timedelta(milliseconds=1))
matched_successfully = False
with transaction.atomic(), swap.lock(
auto_renewal=True), swap.wallet.lock(
auto_renewal=True), swap.recipient.lock(
auto_renewal=True):
swap_wallet_view_context = WalletTransferContext(
wallet=swap.wallet, transfer=swap)
swap_recipient_view_context = WalletTransferContext(
wallet=swap.recipient, transfer=swap)
if swap_expired(swap, operator_eon_number, checkpoint_created):
logger.info('Retiring swap')
swap.retire_swap()
continue
if should_void_swap(swap, swap_wallet_view_context,
swap_recipient_view_context,
operator_eon_number, checkpoint_created):
logger.info('Voiding swap.')
swap.close(voided=True)
continue
elif swap.is_fulfilled_swap():
logger.info('Skipping finalized swap.')
continue
opposite_order_book_name = '{}-{}'.format(
swap.recipient.token.short_name,
swap.wallet.token.short_name)
# If this is a sell order then the opposite orderbook is for buys, which should be sorted
# in decremental order by price such that the first element in the list is the highest priced
opposite_comparison_function = price_comparison_function(
inverse=swap.sell_order, reverse=swap.sell_order)
if opposite_order_book_name not in order_books_cache:
print("FETCHED")
opposite_swaps = Transfer.objects\
.filter(
id__lte=swap.id,
wallet__token=swap.recipient.token,
recipient__token=swap.wallet.token,
processed=False,
complete=False,
voided=False,
cancelled=False,
swap=True,
eon_number=operator_eon_number,
sender_active_state__operator_signature__isnull=False,
recipient_active_state__operator_signature__isnull=False)\
.select_for_update()
order_books_cache[opposite_order_book_name] = SortedList(
opposite_swaps,
key=cmp_to_key(opposite_comparison_function))
else:
print("CACHED")
opposite_order_book = SortedList(
order_books_cache[opposite_order_book_name],
key=cmp_to_key(opposite_comparison_function))
opposite_orders_consumed = 0
if len(opposite_order_book) == 0:
print("EMPTY")
opposite_swaps = Transfer.objects \
.filter(
id__lte=swap.id,
wallet__token=swap.recipient.token,
recipient__token=swap.wallet.token,
processed=False,
complete=False,
voided=False,
cancelled=False,
swap=True,
eon_number=operator_eon_number,
sender_active_state__operator_signature__isnull=False,
recipient_active_state__operator_signature__isnull=False) \
.select_for_update()
assert (len(opposite_order_book) == opposite_swaps.count())
for opposite in opposite_order_book:
# BUY Price: amount / amount_swapped
# SELL Price: amount_swapped / amount
if swap.sell_order:
logger.info('SELL FOR {} VS BUY AT {}'.format(
swap.amount_swapped / swap.amount,
opposite.amount / opposite.amount_swapped))
else:
logger.info('BUY AT {} VS SELL FOR {}'.format(
swap.amount / swap.amount_swapped,
opposite.amount_swapped / opposite.amount))
# The invariant is that the buy order price is greater than or equal to the sell order price
invariant = swap.amount * \
opposite.amount >= opposite.amount_swapped * swap.amount_swapped
if not invariant:
break
with opposite.lock(
auto_renewal=True), opposite.wallet.lock(
auto_renewal=True), opposite.recipient.lock(
auto_renewal=True):
opposite_wallet_view_context = WalletTransferContext(
wallet=opposite.wallet, transfer=opposite)
opposite_recipient_view_context = WalletTransferContext(
wallet=opposite.recipient, transfer=opposite)
if swap_expired(opposite, operator_eon_number,
checkpoint_created):
opposite.retire_swap()
opposite_orders_consumed += 1
continue
if should_void_swap(opposite,
opposite_wallet_view_context,
opposite_recipient_view_context,
operator_eon_number,
checkpoint_created):
opposite.close(voided=True)
opposite_orders_consumed += 1
continue
elif opposite.is_fulfilled_swap():
opposite_orders_consumed += 1
continue
matched_successfully = match_limit_to_limit(
swap, opposite)
if opposite.is_fulfilled_swap():
opposite_orders_consumed += 1
try:
opposite.sign_swap_fulfillment(
settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
except LookupError as e:
logger.error(e)
if swap.is_fulfilled_swap():
try:
swap.sign_swap_fulfillment(
settings.HUB_OWNER_ACCOUNT_ADDRESS,
settings.HUB_OWNER_ACCOUNT_KEY)
except LookupError as e:
logger.error(e)
if swap.is_fulfilled_swap():
break
order_books_cache[
opposite_order_book_name] = opposite_order_book.islice(
opposite_orders_consumed)
swap_order_book_name = '{}-{}'.format(
swap.wallet.token.short_name,
swap.recipient.token.short_name)
if not swap.is_fulfilled_swap(
) and swap_order_book_name in order_books_cache:
swap_comparison_function = price_comparison_function(
inverse=not swap.sell_order,
reverse=not swap.sell_order)
swap_order_book = SortedList(
order_books_cache[swap_order_book_name],
key=cmp_to_key(swap_comparison_function))
swap_order_book.add(swap)
order_books_cache[swap_order_book_name] = swap_order_book
if matched_successfully:
notification_queue.append((swap.id, opposite.id))
cache.set('last_unprocessed_swap_time',
last_unprocessed_swap_time.timestamp())
for swap_id, opposite_id in notification_queue:
operator_celery.send_task('auditor.tasks.on_swap_matching',
args=[swap_id, opposite_id])
def band_search(alpha, A, y, A_fix, s_start, N1_max, bandwidth, adaptive, a, b,
algorithm):
"""Performs the forward band search fo FBMP2 from a given s-vector
Args:
alpha (float): hyperparameter alpha = sigma / sigma_x
A (numpy.ndarray): the feature matrix [A_{m,n}]
y (numpy.ndarray): target vector [y_m]
A_fix (numpy.ndarray): fixed feature matrix [A_fix_{m,n_fix}]
s_start (sequence of :obj:`int`): starting s-vector
N1_max (int): highest total number of active feature to be investigated
bandwidth (int): If `adaptive`==False, this is the number of extensions
in each layer of the search. If `adaptive`==True, it is the minimal
number of times any one feature's s'_n must be found 0 and 1 among
the set of nodes (s') targeted by the extensions.
adaptive (bool): If True, the set of extensions is adaptively chosen,
such that it includes both 0 and 1 for all s'_n values at least
`bandwidth` number of times.
a (float): "a" parameter of inverse-gamma prior of sigma_x^2 (>= 0)
b (float): "b" parameter of inverse-gamma prior of sigma_x^2 (>= 0)
algorithm (str): "M-space" or "N1-space", which selects
whether NodeM and ExtensionM, or NodeN1 and ExtensionN1 classes
are used
Returns:
dict: dict containing
"alpha" (float): input alpha value
"s" (:obj:`list` of :obj:`bytes`): discovered s vectors
"logL" (:obj:`list` of :obj:`float`): discovered ln(L(s)) values
"""
M, N = A.shape
s_start = np.array(s_start, dtype=np.uint8)
if algorithm not in {'M-space', 'N1-space'}:
raise ValueError('argument `algorithm` must be either '
'"M-space" or "N1-space"')
if adaptive:
# In the adaptive mode, we do not know how many of the possible forward
# extensions will we need from each discovery. We need to keep all.
discovery_bandwidth = N
else:
# When bandwidth is truly fixed, it's enough to keep the best
# `bandwidth` from each discovery
discovery_bandwidth = bandwidth
# Initialize starting node
if algorithm == 'M-space':
starting_node = initialize_nodeM(alpha, A, y, A_fix, s_start)
elif algorithm == 'N1-space':
starting_node = initialize_nodeN1(alpha, A, y, A_fix, s_start)
AAdiag = np.einsum('mn,mn->n', A, A)
Ay = A.T.dot(y)
alphasq = alpha**2
else:
raise ValueError('argument `algorithm` must be either '
'"M-space" or "N1-space"')
# initialize result containers
nodes_to_extend = [starting_node]
results = {
bytes(s_start): compute_logL(starting_node.G, starting_node.H, M, a, b)
}
# perform layer-by-layer discovery + extension iterations
for N1 in range(sum(s_start), min(N + 1, N1_max), 1):
best_forward_extensions = SortedList() # always sorted
# We discover the neighbors
# and keep track of all discovered s, ln(L(s)),
# and all necessary forward extensions.
#
# Crucially, `s_record_set` is also updated in each iteration.
# This enables avoiding re-computing already discovered nodes
for node in nodes_to_extend:
if algorithm == 'M-space':
new_logLs, new_forward_extensions = \
node.discover_neighbors(A, y,
discovery_bandwidth,
a, b)
elif algorithm == 'N1-space':
new_logLs, new_forward_extensions = \
node.discover_neighbors(A, AAdiag, Ay, alphasq,
bandwidth, a, b)
else:
raise ValueError('argument `algorithm` must be either '
'"M-space" or "N1-space"')
s = node.s
s_neighbors = list(
map(bytes, (np.repeat(s[None, :], len(s), axis=0) != np.eye(
len(s), dtype=np.uint8)).astype(np.uint8)))
results.update(dict(zip(s_neighbors, new_logLs)))
best_forward_extensions.update(new_forward_extensions)
if not adaptive:
# trim list to contain only the best `bandwidth` extensions
best_forward_extensions = SortedList(
best_forward_extensions.islice(-bandwidth, None))
if adaptive:
# The adaptive logic works the following way:
# - For each feature n, we keep two counters, counting
# how many s-vectors we've included in the set of nodes to
# extend that have s_n = 0 (and 1, respectively)
# - We iterate through the possible extensions in descending
# order with respect to their log_L values
# - We select an extension only if it moves at least one of the
# counters forward that is below the goal count,
# i.e. the `bandwidth`
c0 = np.zeros(N, dtype=int) # counter for (s_n == 0)
c1 = np.zeros(N, dtype=int) # counter for (s_n == 1)
selected_forward_extensions = []
for ext in best_forward_extensions[::-1]:
# check if this extension moves any of the low counter up
s_new = ext.s.copy()
s_new[ext.n] = 1 - s_new[ext.n]
if (1 - s_new)[c0 < bandwidth].any() or \
s_new[c1 < bandwidth].any():
c0 = c0 + 1 - s_new # record the inactive features
c1 = c1 + s_new # record the active features
selected_forward_extensions.append(ext)
best_forward_extensions = selected_forward_extensions
# Compute the target nodes of each selected extension.
# This enables the next iteration to start.
if algorithm == 'M-space':
nodes_to_extend = [
ext.get_target_node(A) for ext in best_forward_extensions
]
elif algorithm == 'N1-space':
nodes_to_extend = [
ext.get_target_node(A, y) for ext in best_forward_extensions
]
else:
raise ValueError('argument `algorithm` must be either '
'"M-space" or "N1-space"')
s_list, logL_list = zip(*results.items())
return {'alpha': alpha, 's': s_list, 'logL': logL_list}
