def test_bubble_down_root(self):
h = IndexedHeap()
h.data = [{
'key': 4,
'value': 40,
'index': 0
}, {
'key': 2,
'value': 20,
'index': 1
}, {
'key': 3,
'value': 30,
'index': 2
}]
h.bubble_down(0)
expected = [{
'key': 2,
'value': 20,
'index': 0
}, {
'key': 4,
'value': 40,
'index': 1
}, {
'key': 3,
'value': 30,
'index': 2
}]
self.assertItemsEqual(h.data, expected, 'sorts the data correctly')
def test_bubble_up_last(self):
h = IndexedHeap()
h.data = [
{
'key': 4,
'value': 40,
'index': 0
},
{
'key': 3,
'value': 30,
'index': 1
},
{
'key': 2,
'value': 20,
'index': 2
},
]
h.bubble_up(2)
expected = [{
'key': 2,
'value': 20,
'index': 0
}, {
'key': 3,
'value': 30,
'index': 1
}, {
'key': 4,
'value': 40,
'index': 2
}]
self.assertItemsEqual(h.data, expected,
'bubbles up the data correctly')
def test_remove(self):
h = IndexedHeap()
h.data = [{
'key': 2,
'value': 20,
'index': 0
}, {
'key': 3,
'value': 30,
'index': 1
}, {
'key': 4,
'value': 40,
'index': 2
}]
item = h.remove(1)
expected = [{
'key': 2,
'value': 20,
'index': 0
}, {
'key': 4,
'value': 40,
'index': 1
}]
self.assertItemsEqual(h.data, expected, 'correctly removes the data')
self.assertEqual(item, {
'key': 3,
'value': 30,
'index': 1
}, 'removed item is correct')
def test_get_min_for_indices(self):
h = IndexedHeap()
h.data = [{'key': i, 'value': i, 'index': i} for i in range(10)]
actual = h.get_min_for_indices([9, None, None])
self.assertEqual(actual, 9, 'no children so root is min')
actual = h.get_min_for_indices([7, 8, 9])
self.assertEqual(actual, 7, 'root is the smallest')
actual = h.get_min_for_indices([2, 8, 1])
self.assertEqual(actual, 1, 'third is the smallest')
def test_bubble_down_last(self):
h = IndexedHeap()
h.data = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 4, 'value': 40, 'index': 1},
{'key': 3, 'value': 30, 'index': 2},
]
h.bubble_down(2)
expected = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 4, 'value': 40, 'index': 1},
{'key': 3, 'value': 30, 'index': 2},
]
self.assertItemsEqual(h.data, expected, 'sorts the data correctly')
def test_bubble_up_root(self):
h = IndexedHeap()
h.data = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 3, 'value': 30, 'index': 1},
{'key': 4, 'value': 40, 'index': 2}
]
h.bubble_up(0)
expected = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 3, 'value': 30, 'index': 1},
{'key': 4, 'value': 40, 'index': 2}
]
self.assertItemsEqual(h.data, expected, 'bubbles up the data correctly')
def test_remove(self):
h = IndexedHeap()
h.data = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 3, 'value': 30, 'index': 1},
{'key': 4, 'value': 40, 'index': 2}
]
item = h.remove(1)
expected = [
{'key': 2, 'value': 20, 'index': 0},
{'key': 4, 'value': 40, 'index': 1}
]
self.assertItemsEqual(h.data, expected,
'correctly removes the data')
self.assertEqual(item, {'key': 3, 'value': 30, 'index': 1},
'removed item is correct')
def test_heapify(self):
h = IndexedHeap.heapify([(3, 'z'), (2, 'y'), (1, 'x')])
expected = [
{'key': 1, 'value': 'x', 'index': 0},
{'key': 2, 'value': 'y', 'index': 1},
{'key': 3, 'value': 'z', 'index': 2}
]
self.assertItemsEqual(expected, h.data, 'should create a valid heap')
def test_get_parent_index(self):
h = IndexedHeap()
h.data = range(10)
self.assertEqual(h.get_parent_index(0), None, 'no parent for root')
self.assertEqual(h.get_parent_index(1), 0, '0 parent for 1')
self.assertEqual(h.get_parent_index(2), 0, '0 parent for 2')
self.assertEqual(h.get_parent_index(3), 1, '1 parent for 3')
self.assertEqual(h.get_parent_index(4), 1, '1 parent for 4')
self.assertEqual(h.get_parent_index(5), 2, '2 parent for 5')
self.assertEqual(h.get_parent_index(6), 2, '2 parent for 6')
self.assertEqual(h.get_parent_index(7), 3, '3 parent for 7')
self.assertEqual(h.get_parent_index(8), 3, '3 parent for 8')
self.assertEqual(h.get_parent_index(9), 4, '4 parent for 9')
def test_insert(self):
h = IndexedHeap()
inserted1 = h.insert({'key': 'z', 'value': 300})
self.assertEqual(inserted1, {'key': 'z', 'value': 300, 'index': 0},
'correctly inserted and returned new item')
inserted2 = h.insert({'key': 'y', 'value': 200})
self.assertEqual(inserted2, {'key': 'y', 'value': 200, 'index': 0},
'correctly inserted and returned new item')
self.assertEqual(inserted1, {'key': 'z', 'value': 300, 'index': 1},
'correctly updated existing items')
inserted3 = h.insert({'key': 'x', 'value': 100})
self.assertEqual(inserted3, {'key': 'x', 'value': 100, 'index': 0},
'correctly inserted and returned new item')
self.assertEqual(inserted2, {'key': 'y', 'value': 200, 'index': 2},
'correctly inserted and returned item')
self.assertEqual(inserted1, {'key': 'z', 'value': 300, 'index': 1},
'correctly updated existing items')
def test_insert(self):
h = IndexedHeap()
inserted1 = h.insert({'key': 'z', 'value': 300})
self.assertEqual(inserted1, {
'key': 'z',
'value': 300,
'index': 0
}, 'correctly inserted and returned new item')
inserted2 = h.insert({'key': 'y', 'value': 200})
self.assertEqual(inserted2, {
'key': 'y',
'value': 200,
'index': 0
}, 'correctly inserted and returned new item')
self.assertEqual(inserted1, {
'key': 'z',
'value': 300,
'index': 1
}, 'correctly updated existing items')
inserted3 = h.insert({'key': 'x', 'value': 100})
self.assertEqual(inserted3, {
'key': 'x',
'value': 100,
'index': 0
}, 'correctly inserted and returned new item')
self.assertEqual(inserted2, {
'key': 'y',
'value': 200,
'index': 2
}, 'correctly inserted and returned item')
self.assertEqual(inserted1, {
'key': 'z',
'value': 300,
'index': 1
}, 'correctly updated existing items')
def test_get_child_indices(self):
h = IndexedHeap()
h.data = range(10)
self.assertEqual(h.get_child_indices(0), [1, 2],
'correct children for 0')
self.assertEqual(h.get_child_indices(1), [3, 4],
'correct children for 1')
self.assertEqual(h.get_child_indices(2), [5, 6],
'correct children for 2')
def test_get_parent_index(self):
h = IndexedHeap()
h.data = range(10)
self.assertEqual(h.get_parent_index(0), None, 'no parent for root')
self.assertEqual(h.get_parent_index(1), 0, '0 parent for 1')
self.assertEqual(h.get_parent_index(2), 0, '0 parent for 2')
self.assertEqual(h.get_parent_index(3), 1, '1 parent for 3')
self.assertEqual(h.get_parent_index(4), 1, '1 parent for 4')
self.assertEqual(h.get_parent_index(5), 2, '2 parent for 5')
self.assertEqual(h.get_parent_index(6), 2, '2 parent for 6')
self.assertEqual(h.get_parent_index(7), 3, '3 parent for 7')
self.assertEqual(h.get_parent_index(8), 3, '3 parent for 8')
self.assertEqual(h.get_parent_index(9), 4, '4 parent for 9')
def test_heapify(self):
h = IndexedHeap.heapify([(3, 'z'), (2, 'y'), (1, 'x')])
expected = [{
'key': 1,
'value': 'x',
'index': 0
}, {
'key': 2,
'value': 'y',
'index': 1
}, {
'key': 3,
'value': 'z',
'index': 2
}]
self.assertItemsEqual(expected, h.data, 'should create a valid heap')
def test_get_min_for_indices(self):
h = IndexedHeap()
h.data = [{'key': i, 'value': i, 'index': i} for i in range(10)]
actual = h.get_min_for_indices([9, None, None])
self.assertEqual(actual, 9, 'no children so root is min')
actual = h.get_min_for_indices([7, 8, 9])
self.assertEqual(actual, 7, 'root is the smallest')
actual = h.get_min_for_indices([2, 8, 1])
self.assertEqual(actual, 1, 'third is the smallest')
def __init__(self, max_size):
Cache.__init__(self, max_size)
self.data = {}
self.heap = IndexedHeap()
class LFUCache(Cache):
""" Least Frequently Used cache implementation.
To implement fast least frequently used key retrieval this class uses a heap.
Args:
data: dict, data structure containing the items.
heap: object, instance of LFUHeap
"""
def __init__(self, max_size):
Cache.__init__(self, max_size)
self.data = {}
self.heap = IndexedHeap()
def __len__(self):
""" Returns the number of items in the cache. """
return len(self.data)
def write(self, key, value):
""" Writes the data to the cache.
When writing, the frequency of the key is also incremented.
Complexity:
Args:
key, the key to write the data under.
value, the value to store.
"""
# Evict item if necessare before inserting.
evicted = None
if len(self.data) == self.max_size and key not in self.data:
evicted = self.evict()
# Write the cache item.
if key not in self.data:
item = {'key': 0, 'value': value, '_key': key}
self.heap.insert(item)
self.data[key] = item
else:
item = self.data[key]
item['value'] = value
self.increment_frequency(item)
return evicted
def read(self, key):
""" Reads the value for the given key from the cache.
The key gets its frequency incremented whenever it is read.
Raises:
Exception, when a cache miss occurs.
"""
if key not in self.data:
raise Exception('Cache miss for key={key}'.format(key=key))
item = self.data[key]
self.increment_frequency(item)
return item['value']
def increment_frequency(self, item):
""" Increments the frequency of the given key.
Method also normalizes frequencies whenever the values reach the top
limit for integers in python.
See: src.heap.IndexedHeap for the way objects are inserted in the heap.
Complexity: O(n) because of the index lookup.
Args:
item: format: {key, _key, value, index} where key is the frequency
by which items are sorted in the heap.
"""
index = item['index']
item = self.heap.remove(index)
item['key'] += 1
self.heap.insert(item)
def evict(self):
""" Evicts the element with the least usage frequency.
Complexity: O(log n) because of the heap.
Returns:
dict, in case a value had to be evicted. Format {key, value}.
key: str
value: anything
None, in case no eviction takes place
"""
node = self.heap.extract_min()
key = node['_key']
del self.data[key]
return {'key': node['_key'], 'value': node['value']}
def test_remove_from_single_item_heap(self):
h = IndexedHeap()
h.insert({'key': 'x', 'value': 1000})
h.remove(0)
self.assertEqual(h.data, [], 'empty data in the heap')
def test_get_child_indices(self):
h = IndexedHeap()
h.data = range(10)
self.assertEqual(h.get_child_indices(0), [1, 2], 'correct children for 0')
self.assertEqual(h.get_child_indices(1), [3, 4], 'correct children for 1')
self.assertEqual(h.get_child_indices(2), [5, 6], 'correct children for 2')
def test_remove_from_single_item_heap(self):
h = IndexedHeap()
h.insert({'key': 'x', 'value': 1000})
h.remove(0)
self.assertEqual(h.data, [], 'empty data in the heap')
class LFUCache(Cache):
""" Least Frequently Used cache implementation.
To implement fast least frequently used key retrieval this class uses a heap.
Args:
data: dict, data structure containing the items.
heap: object, instance of LFUHeap
"""
def __init__(self, max_size):
Cache.__init__(self, max_size)
self.data = {}
self.heap = IndexedHeap()
def __len__(self):
""" Returns the number of items in the cache. """
return len(self.data)
def write(self, key, value):
""" Writes the data to the cache.
When writing, the frequency of the key is also incremented.
Complexity:
Args:
key, the key to write the data under.
value, the value to store.
"""
# Evict item if necessare before inserting.
evicted = None
if len(self.data) == self.max_size and key not in self.data:
evicted = self.evict()
# Write the cache item.
if key not in self.data:
item = {'key': 0, 'value': value, '_key': key}
self.heap.insert(item)
self.data[key] = item
else:
item = self.data[key]
item['value'] = value
self.increment_frequency(item)
return evicted
def read(self, key):
""" Reads the value for the given key from the cache.
The key gets its frequency incremented whenever it is read.
Raises:
Exception, when a cache miss occurs.
"""
if key not in self.data:
raise Exception('Cache miss for key={key}'.format(key=key))
item = self.data[key]
self.increment_frequency(item)
return item['value']
def increment_frequency(self, item):
""" Increments the frequency of the given key.
Method also normalizes frequencies whenever the values reach the top
limit for integers in python.
See: src.heap.IndexedHeap for the way objects are inserted in the heap.
Complexity: O(n) because of the index lookup.
Args:
item: format: {key, _key, value, index} where key is the frequency
by which items are sorted in the heap.
"""
index = item['index']
item = self.heap.remove(index)
item['key'] += 1
self.heap.insert(item)
def evict(self):
""" Evicts the element with the least usage frequency.
Complexity: O(log n) because of the heap.
Returns:
dict, in case a value had to be evicted. Format {key, value}.
key: str
value: anything
None, in case no eviction takes place
"""
node = self.heap.extract_min()
key = node['_key']
del self.data[key]
return {'key': node['_key'], 'value': node['value']}
def test_extract_min(self):
h = IndexedHeap.heapify([('z', 3), ('y', 2), ('x', 1)])
item = h.extract_min()
expected = {'key': 'x', 'value': 1, 'index': 0}
self.assertEqual(expected, item, 'correctly extracts the min value')
def test_extract_min(self):
h = IndexedHeap.heapify([('z', 3), ('y', 2), ('x', 1)])
item = h.extract_min()
expected = {'key': 'x', 'value': 1, 'index': 0}
self.assertEqual(expected, item, 'correctly extracts the min value')
def __init__(self, max_size):
Cache.__init__(self, max_size)
self.data = {}
self.heap = IndexedHeap()