def test_streamthreshold_bytes(self):
"""test exporting a streaming threshold as bytes"""
md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e"
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
st1.add("this is a test", 100)
bytes_out = bytes(st1)
self.assertEqual(hashlib.md5(bytes_out).hexdigest(), md5_val)
def test_streamthreshold_rem_mult(self):
"""test adding elements to the stream threshold in multiple"""
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add("this is a test", 30), 30)
self.assertEqual(st1.add("this is not a test", 11), 11)
self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11})
self.assertEqual(st1.elements_added, 41)
self.assertEqual(st1.remove("this is not a test", 2), 9)
self.assertEqual(st1.meets_threshold, {"this is a test": 30})
self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_remove(self):
''' test removing elements from the stream threshold singular '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test', 5), 10)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is not a test', 9), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is a test', 20), 30)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.add('this is not a test', 2), 11)
self.assertEqual(st1.meets_threshold, {
'this is a test': 30,
'this is not a test': 11
})
self.assertEqual(st1.remove('this is a test'), 29)
self.assertEqual(st1.meets_threshold, {
'this is a test': 29,
'this is not a test': 11
})
self.assertEqual(st1.remove('this is not a test'), 10)
self.assertEqual(st1.remove('this is not a test'), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 29})
self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_rem_mult(self):
''' test adding elements to the stream threshold in multiple '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 30), 30)
self.assertEqual(st1.add('this is not a test', 11), 11)
self.assertEqual(st1.meets_threshold,
{'this is a test': 30, 'this is not a test': 11})
self.assertEqual(st1.elements_added, 41)
self.assertEqual(st1.remove('this is not a test', 2), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_export(self):
''' test exporting a stream threshold sketch '''
md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449'
filename = 'test.cms'
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
st1.add('this is a test', 100)
st1.export(filename)
md5_out = calc_file_md5(filename)
os.remove(filename)
self.assertEqual(md5_out, md5_val)
def test_streamthreshold_rem_mult(self):
''' test adding elements to the stream threshold in multiple '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 30), 30)
self.assertEqual(st1.add('this is not a test', 11), 11)
self.assertEqual(st1.meets_threshold, {
'this is a test': 30,
'this is not a test': 11
})
self.assertEqual(st1.elements_added, 41)
self.assertEqual(st1.remove('this is not a test', 2), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_str(self):
''' test the string representation of the stream threshold sketch '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 100), 100)
msg = ('Stream Threshold Count-Min Sketch:\n'
'\tWidth: 1000\n'
'\tDepth: 5\n'
'\tConfidence: 0.96875\n'
'\tError Rate: 0.002\n'
'\tElements Added: 100\n'
'\tThreshold: 10\n'
'\tNumber Meeting Threshold: 1')
self.assertEqual(str(st1), msg)
def test_streamthreshold_str(self):
''' test the string representation of the stream threshold sketch '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 100), 100)
msg = ('Stream Threshold Count-Min Sketch:\n'
'\tWidth: 1000\n'
'\tDepth: 5\n'
'\tConfidence: 0.96875\n'
'\tError Rate: 0.002\n'
'\tElements Added: 100\n'
'\tThreshold: 10\n'
'\tNumber Meeting Threshold: 1')
self.assertEqual(str(st1), msg)
def test_streamthreshold_frombytes(self):
"""testinitializing a heavy hitters from bytes"""
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
st1.add("this is a test", 100)
bytes_out = bytes(st1)
st2 = StreamThreshold.frombytes(bytes_out, threshold=10)
self.assertEqual(st2.width, 1000)
self.assertEqual(st2.depth, 5)
self.assertEqual(st2.threshold, 10)
self.assertEqual(st2.elements_added, 100)
self.assertEqual(bytes(st2), bytes(st1))
self.assertEqual(st2.check("this is a test"), 100)
# show on load that the tracking of stream threshold is gone
self.assertEqual(st2.meets_threshold, dict())
self.assertEqual(st2.add("this is a test", 1), 101)
self.assertEqual(st2.meets_threshold, {"this is a test": 101})
def test_streamthreshold_init_wd(self):
"""test initializing the stream threshold using width and depth"""
st1 = StreamThreshold(threshold=1000, width=1000, depth=5)
self.assertEqual(st1.width, 1000)
self.assertEqual(st1.depth, 5)
self.assertEqual(st1.confidence, 0.96875)
self.assertEqual(st1.error_rate, 0.002)
self.assertEqual(st1.elements_added, 0)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.threshold, 1000)
def test_streamthreshold_clear(self):
"""test clearing the stream threshold"""
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add("this is a test", 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add("this is a test", 5), 10)
self.assertEqual(st1.meets_threshold, {"this is a test": 10})
self.assertEqual(st1.add("this is not a test", 9), 9)
self.assertEqual(st1.meets_threshold, {"this is a test": 10})
self.assertEqual(st1.add("this is a test", 20), 30)
self.assertEqual(st1.meets_threshold, {"this is a test": 30})
self.assertEqual(st1.add("this is not a test", 2), 11)
self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11})
self.assertEqual(st1.elements_added, 41)
st1.clear()
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_init_ec(self):
''' test initializing the stream threshold using error rate and
confidence '''
st1 = StreamThreshold(threshold=1000,
confidence=0.96875,
error_rate=0.002)
self.assertEqual(st1.width, 1000)
self.assertEqual(st1.depth, 5)
self.assertEqual(st1.confidence, 0.96875)
self.assertEqual(st1.error_rate, 0.002)
self.assertEqual(st1.elements_added, 0)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.threshold, 1000)
def test_streamthreshold_add(self):
"""test adding elements to the stream threshold in singular"""
st1 = StreamThreshold(threshold=2, width=1000, depth=5)
self.assertEqual(st1.add("this is a test"), 1)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add("this is a test"), 2)
self.assertEqual(st1.meets_threshold, {"this is a test": 2})
self.assertEqual(st1.add("this is not a test"), 1)
self.assertEqual(st1.meets_threshold, {"this is a test": 2})
self.assertEqual(st1.add("this is a test"), 3)
self.assertEqual(st1.meets_threshold, {"this is a test": 3})
self.assertEqual(st1.add("this is not a test"), 2)
self.assertEqual(st1.add("this is still not a test"), 1)
self.assertEqual(st1.meets_threshold, {"this is a test": 3, "this is not a test": 2})
self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_export(self):
"""test exporting a stream threshold sketch"""
md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e"
with NamedTemporaryFile(dir=os.getcwd(), suffix=".cms", delete=DELETE_TEMP_FILES) as fobj:
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
st1.add("this is a test", 100)
st1.export(fobj.name)
md5_out = calc_file_md5(fobj.name)
self.assertEqual(md5_out, md5_val)
def test_streamthreshold_remove(self):
"""test removing elements from the stream threshold singular"""
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add("this is a test", 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add("this is a test", 5), 10)
self.assertEqual(st1.meets_threshold, {"this is a test": 10})
self.assertEqual(st1.add("this is not a test", 9), 9)
self.assertEqual(st1.meets_threshold, {"this is a test": 10})
self.assertEqual(st1.add("this is a test", 20), 30)
self.assertEqual(st1.meets_threshold, {"this is a test": 30})
self.assertEqual(st1.add("this is not a test", 2), 11)
self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11})
self.assertEqual(st1.remove("this is a test"), 29)
self.assertEqual(st1.meets_threshold, {"this is a test": 29, "this is not a test": 11})
self.assertEqual(st1.remove("this is not a test"), 10)
self.assertEqual(st1.remove("this is not a test"), 9)
self.assertEqual(st1.meets_threshold, {"this is a test": 29})
self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_clear(self):
''' test clearing the stream threshold '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test', 5), 10)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is not a test', 9), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is a test', 20), 30)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.add('this is not a test', 2), 11)
self.assertEqual(st1.meets_threshold, {
'this is a test': 30,
'this is not a test': 11
})
self.assertEqual(st1.elements_added, 41)
st1.clear()
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_add(self):
''' test adding elements to the stream threshold in singular '''
st1 = StreamThreshold(threshold=2, width=1000, depth=5)
self.assertEqual(st1.add('this is a test'), 1)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test'), 2)
self.assertEqual(st1.meets_threshold, {'this is a test': 2})
self.assertEqual(st1.add('this is not a test'), 1)
self.assertEqual(st1.meets_threshold, {'this is a test': 2})
self.assertEqual(st1.add('this is a test'), 3)
self.assertEqual(st1.meets_threshold, {'this is a test': 3})
self.assertEqual(st1.add('this is not a test'), 2)
self.assertEqual(st1.add('this is still not a test'), 1)
self.assertEqual(st1.meets_threshold, {
'this is a test': 3,
'this is not a test': 2
})
self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_export(self):
''' test exporting a stream threshold sketch '''
md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449'
filename = 'test.cms'
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
st1.add('this is a test', 100)
st1.export(filename)
md5_out = calc_file_md5(filename)
os.remove(filename)
self.assertEqual(md5_out, md5_val)
def test_streamthreshold_load(self):
"""test loading a stream threshold sketch from file"""
md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e"
with NamedTemporaryFile(dir=os.getcwd(), suffix=".cms", delete=DELETE_TEMP_FILES) as fobj:
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add("this is a test", 100), 100)
self.assertEqual(st1.elements_added, 100)
self.assertEqual(st1.meets_threshold, {"this is a test": 100})
st1.export(fobj.name)
md5_out = calc_file_md5(fobj.name)
self.assertEqual(md5_out, md5_val)
# try loading directly to file!
st2 = StreamThreshold(threshold=10, filepath=fobj.name)
self.assertEqual(st2.width, 1000)
self.assertEqual(st2.depth, 5)
self.assertEqual(st2.elements_added, 100)
self.assertEqual(st2.check("this is a test"), 100)
# show on load that the tracking of stream threshold is gone
self.assertEqual(st2.meets_threshold, dict())
self.assertEqual(st2.add("this is a test", 1), 101)
self.assertEqual(st2.meets_threshold, {"this is a test": 101})
def test_streamthreshold_clear(self):
''' test clearing the stream threshold '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test', 5), 10)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is not a test', 9), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is a test', 20), 30)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.add('this is not a test', 2), 11)
self.assertEqual(st1.meets_threshold,
{'this is a test': 30, 'this is not a test': 11})
self.assertEqual(st1.elements_added, 41)
st1.clear()
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_add(self):
''' test adding elements to the stream threshold in singular '''
st1 = StreamThreshold(threshold=2, width=1000, depth=5)
self.assertEqual(st1.add('this is a test'), 1)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test'), 2)
self.assertEqual(st1.meets_threshold, {'this is a test': 2})
self.assertEqual(st1.add('this is not a test'), 1)
self.assertEqual(st1.meets_threshold, {'this is a test': 2})
self.assertEqual(st1.add('this is a test'), 3)
self.assertEqual(st1.meets_threshold, {'this is a test': 3})
self.assertEqual(st1.add('this is not a test'), 2)
self.assertEqual(st1.add('this is still not a test'), 1)
self.assertEqual(st1.meets_threshold,
{'this is a test': 3, 'this is not a test': 2})
self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_load(self):
''' test loading a stream threshold sketch from file '''
md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449'
filename = 'test.cms'
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 100), 100)
self.assertEqual(st1.elements_added, 100)
self.assertEqual(st1.meets_threshold, {'this is a test': 100})
st1.export(filename)
md5_out = calc_file_md5(filename)
self.assertEqual(md5_out, md5_val)
# try loading directly to file!
st2 = StreamThreshold(threshold=10, filepath=filename)
self.assertEqual(st2.width, 1000)
self.assertEqual(st2.depth, 5)
self.assertEqual(st2.elements_added, 100)
self.assertEqual(st2.check('this is a test'), 100)
# show on load that the tracking of stream threshold is gone
self.assertEqual(st2.meets_threshold, dict())
self.assertEqual(st2.add('this is a test', 1), 101)
self.assertEqual(st2.meets_threshold, {'this is a test': 101})
os.remove(filename)
def test_streamthreshold_remove(self):
''' test removing elements from the stream threshold singular '''
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 5), 5)
self.assertEqual(st1.meets_threshold, dict())
self.assertEqual(st1.add('this is a test', 5), 10)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is not a test', 9), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 10})
self.assertEqual(st1.add('this is a test', 20), 30)
self.assertEqual(st1.meets_threshold, {'this is a test': 30})
self.assertEqual(st1.add('this is not a test', 2), 11)
self.assertEqual(st1.meets_threshold,
{'this is a test': 30, 'this is not a test': 11})
self.assertEqual(st1.remove('this is a test'), 29)
self.assertEqual(st1.meets_threshold,
{'this is a test': 29, 'this is not a test': 11})
self.assertEqual(st1.remove('this is not a test'), 10)
self.assertEqual(st1.remove('this is not a test'), 9)
self.assertEqual(st1.meets_threshold, {'this is a test': 29})
self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_load(self):
''' test loading a stream threshold sketch from file '''
md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449'
filename = 'test.cms'
st1 = StreamThreshold(threshold=10, width=1000, depth=5)
self.assertEqual(st1.add('this is a test', 100), 100)
self.assertEqual(st1.elements_added, 100)
self.assertEqual(st1.meets_threshold, {'this is a test': 100})
st1.export(filename)
md5_out = calc_file_md5(filename)
self.assertEqual(md5_out, md5_val)
# try loading directly to file!
st2 = StreamThreshold(threshold=10, filepath=filename)
self.assertEqual(st2.width, 1000)
self.assertEqual(st2.depth, 5)
self.assertEqual(st2.elements_added, 100)
self.assertEqual(st2.check('this is a test'), 100)
# show on load that the tracking of stream threshold is gone
self.assertEqual(st2.meets_threshold, dict())
self.assertEqual(st2.add('this is a test', 1), 101)
self.assertEqual(st2.meets_threshold, {'this is a test': 101})
os.remove(filename)
def test_streamthreshold_join(self):
"""test that stream threshold raises exception"""
st1 = StreamThreshold(threshold=1000, width=1000, depth=5)
st2 = StreamThreshold(threshold=1000, width=1000, depth=5)
self.assertRaises(NotSupportedError, lambda: st1.join(st2))