def test_streamthreshold_bytes(self): """test exporting a streaming threshold as bytes""" md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e" st1 = StreamThreshold(threshold=10, width=1000, depth=5) st1.add("this is a test", 100) bytes_out = bytes(st1) self.assertEqual(hashlib.md5(bytes_out).hexdigest(), md5_val)
def test_streamthreshold_rem_mult(self): """test adding elements to the stream threshold in multiple""" st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add("this is a test", 30), 30) self.assertEqual(st1.add("this is not a test", 11), 11) self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11}) self.assertEqual(st1.elements_added, 41) self.assertEqual(st1.remove("this is not a test", 2), 9) self.assertEqual(st1.meets_threshold, {"this is a test": 30}) self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_remove(self): ''' test removing elements from the stream threshold singular ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test', 5), 10) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is not a test', 9), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is a test', 20), 30) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.add('this is not a test', 2), 11) self.assertEqual(st1.meets_threshold, { 'this is a test': 30, 'this is not a test': 11 }) self.assertEqual(st1.remove('this is a test'), 29) self.assertEqual(st1.meets_threshold, { 'this is a test': 29, 'this is not a test': 11 }) self.assertEqual(st1.remove('this is not a test'), 10) self.assertEqual(st1.remove('this is not a test'), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 29}) self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_rem_mult(self): ''' test adding elements to the stream threshold in multiple ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 30), 30) self.assertEqual(st1.add('this is not a test', 11), 11) self.assertEqual(st1.meets_threshold, {'this is a test': 30, 'this is not a test': 11}) self.assertEqual(st1.elements_added, 41) self.assertEqual(st1.remove('this is not a test', 2), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_export(self): ''' test exporting a stream threshold sketch ''' md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449' filename = 'test.cms' st1 = StreamThreshold(threshold=10, width=1000, depth=5) st1.add('this is a test', 100) st1.export(filename) md5_out = calc_file_md5(filename) os.remove(filename) self.assertEqual(md5_out, md5_val)
def test_streamthreshold_rem_mult(self): ''' test adding elements to the stream threshold in multiple ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 30), 30) self.assertEqual(st1.add('this is not a test', 11), 11) self.assertEqual(st1.meets_threshold, { 'this is a test': 30, 'this is not a test': 11 }) self.assertEqual(st1.elements_added, 41) self.assertEqual(st1.remove('this is not a test', 2), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.elements_added, 39)
def test_streamthreshold_str(self): ''' test the string representation of the stream threshold sketch ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 100), 100) msg = ('Stream Threshold Count-Min Sketch:\n' '\tWidth: 1000\n' '\tDepth: 5\n' '\tConfidence: 0.96875\n' '\tError Rate: 0.002\n' '\tElements Added: 100\n' '\tThreshold: 10\n' '\tNumber Meeting Threshold: 1') self.assertEqual(str(st1), msg)
def test_streamthreshold_str(self): ''' test the string representation of the stream threshold sketch ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 100), 100) msg = ('Stream Threshold Count-Min Sketch:\n' '\tWidth: 1000\n' '\tDepth: 5\n' '\tConfidence: 0.96875\n' '\tError Rate: 0.002\n' '\tElements Added: 100\n' '\tThreshold: 10\n' '\tNumber Meeting Threshold: 1') self.assertEqual(str(st1), msg)
def test_streamthreshold_frombytes(self): """testinitializing a heavy hitters from bytes""" st1 = StreamThreshold(threshold=10, width=1000, depth=5) st1.add("this is a test", 100) bytes_out = bytes(st1) st2 = StreamThreshold.frombytes(bytes_out, threshold=10) self.assertEqual(st2.width, 1000) self.assertEqual(st2.depth, 5) self.assertEqual(st2.threshold, 10) self.assertEqual(st2.elements_added, 100) self.assertEqual(bytes(st2), bytes(st1)) self.assertEqual(st2.check("this is a test"), 100) # show on load that the tracking of stream threshold is gone self.assertEqual(st2.meets_threshold, dict()) self.assertEqual(st2.add("this is a test", 1), 101) self.assertEqual(st2.meets_threshold, {"this is a test": 101})
def test_streamthreshold_init_wd(self): """test initializing the stream threshold using width and depth""" st1 = StreamThreshold(threshold=1000, width=1000, depth=5) self.assertEqual(st1.width, 1000) self.assertEqual(st1.depth, 5) self.assertEqual(st1.confidence, 0.96875) self.assertEqual(st1.error_rate, 0.002) self.assertEqual(st1.elements_added, 0) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.threshold, 1000)
def test_streamthreshold_clear(self): """test clearing the stream threshold""" st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add("this is a test", 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add("this is a test", 5), 10) self.assertEqual(st1.meets_threshold, {"this is a test": 10}) self.assertEqual(st1.add("this is not a test", 9), 9) self.assertEqual(st1.meets_threshold, {"this is a test": 10}) self.assertEqual(st1.add("this is a test", 20), 30) self.assertEqual(st1.meets_threshold, {"this is a test": 30}) self.assertEqual(st1.add("this is not a test", 2), 11) self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11}) self.assertEqual(st1.elements_added, 41) st1.clear() self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_init_ec(self): ''' test initializing the stream threshold using error rate and confidence ''' st1 = StreamThreshold(threshold=1000, confidence=0.96875, error_rate=0.002) self.assertEqual(st1.width, 1000) self.assertEqual(st1.depth, 5) self.assertEqual(st1.confidence, 0.96875) self.assertEqual(st1.error_rate, 0.002) self.assertEqual(st1.elements_added, 0) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.threshold, 1000)
def test_streamthreshold_add(self): """test adding elements to the stream threshold in singular""" st1 = StreamThreshold(threshold=2, width=1000, depth=5) self.assertEqual(st1.add("this is a test"), 1) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add("this is a test"), 2) self.assertEqual(st1.meets_threshold, {"this is a test": 2}) self.assertEqual(st1.add("this is not a test"), 1) self.assertEqual(st1.meets_threshold, {"this is a test": 2}) self.assertEqual(st1.add("this is a test"), 3) self.assertEqual(st1.meets_threshold, {"this is a test": 3}) self.assertEqual(st1.add("this is not a test"), 2) self.assertEqual(st1.add("this is still not a test"), 1) self.assertEqual(st1.meets_threshold, {"this is a test": 3, "this is not a test": 2}) self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_export(self): """test exporting a stream threshold sketch""" md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e" with NamedTemporaryFile(dir=os.getcwd(), suffix=".cms", delete=DELETE_TEMP_FILES) as fobj: st1 = StreamThreshold(threshold=10, width=1000, depth=5) st1.add("this is a test", 100) st1.export(fobj.name) md5_out = calc_file_md5(fobj.name) self.assertEqual(md5_out, md5_val)
def test_streamthreshold_remove(self): """test removing elements from the stream threshold singular""" st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add("this is a test", 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add("this is a test", 5), 10) self.assertEqual(st1.meets_threshold, {"this is a test": 10}) self.assertEqual(st1.add("this is not a test", 9), 9) self.assertEqual(st1.meets_threshold, {"this is a test": 10}) self.assertEqual(st1.add("this is a test", 20), 30) self.assertEqual(st1.meets_threshold, {"this is a test": 30}) self.assertEqual(st1.add("this is not a test", 2), 11) self.assertEqual(st1.meets_threshold, {"this is a test": 30, "this is not a test": 11}) self.assertEqual(st1.remove("this is a test"), 29) self.assertEqual(st1.meets_threshold, {"this is a test": 29, "this is not a test": 11}) self.assertEqual(st1.remove("this is not a test"), 10) self.assertEqual(st1.remove("this is not a test"), 9) self.assertEqual(st1.meets_threshold, {"this is a test": 29}) self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_clear(self): ''' test clearing the stream threshold ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test', 5), 10) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is not a test', 9), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is a test', 20), 30) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.add('this is not a test', 2), 11) self.assertEqual(st1.meets_threshold, { 'this is a test': 30, 'this is not a test': 11 }) self.assertEqual(st1.elements_added, 41) st1.clear() self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_add(self): ''' test adding elements to the stream threshold in singular ''' st1 = StreamThreshold(threshold=2, width=1000, depth=5) self.assertEqual(st1.add('this is a test'), 1) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test'), 2) self.assertEqual(st1.meets_threshold, {'this is a test': 2}) self.assertEqual(st1.add('this is not a test'), 1) self.assertEqual(st1.meets_threshold, {'this is a test': 2}) self.assertEqual(st1.add('this is a test'), 3) self.assertEqual(st1.meets_threshold, {'this is a test': 3}) self.assertEqual(st1.add('this is not a test'), 2) self.assertEqual(st1.add('this is still not a test'), 1) self.assertEqual(st1.meets_threshold, { 'this is a test': 3, 'this is not a test': 2 }) self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_export(self): ''' test exporting a stream threshold sketch ''' md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449' filename = 'test.cms' st1 = StreamThreshold(threshold=10, width=1000, depth=5) st1.add('this is a test', 100) st1.export(filename) md5_out = calc_file_md5(filename) os.remove(filename) self.assertEqual(md5_out, md5_val)
def test_streamthreshold_load(self): """test loading a stream threshold sketch from file""" md5_val = "fb1c39dd1a73f1ef0d7fc79f60fc028e" with NamedTemporaryFile(dir=os.getcwd(), suffix=".cms", delete=DELETE_TEMP_FILES) as fobj: st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add("this is a test", 100), 100) self.assertEqual(st1.elements_added, 100) self.assertEqual(st1.meets_threshold, {"this is a test": 100}) st1.export(fobj.name) md5_out = calc_file_md5(fobj.name) self.assertEqual(md5_out, md5_val) # try loading directly to file! st2 = StreamThreshold(threshold=10, filepath=fobj.name) self.assertEqual(st2.width, 1000) self.assertEqual(st2.depth, 5) self.assertEqual(st2.elements_added, 100) self.assertEqual(st2.check("this is a test"), 100) # show on load that the tracking of stream threshold is gone self.assertEqual(st2.meets_threshold, dict()) self.assertEqual(st2.add("this is a test", 1), 101) self.assertEqual(st2.meets_threshold, {"this is a test": 101})
def test_streamthreshold_clear(self): ''' test clearing the stream threshold ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test', 5), 10) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is not a test', 9), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is a test', 20), 30) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.add('this is not a test', 2), 11) self.assertEqual(st1.meets_threshold, {'this is a test': 30, 'this is not a test': 11}) self.assertEqual(st1.elements_added, 41) st1.clear() self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.elements_added, 0)
def test_streamthreshold_add(self): ''' test adding elements to the stream threshold in singular ''' st1 = StreamThreshold(threshold=2, width=1000, depth=5) self.assertEqual(st1.add('this is a test'), 1) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test'), 2) self.assertEqual(st1.meets_threshold, {'this is a test': 2}) self.assertEqual(st1.add('this is not a test'), 1) self.assertEqual(st1.meets_threshold, {'this is a test': 2}) self.assertEqual(st1.add('this is a test'), 3) self.assertEqual(st1.meets_threshold, {'this is a test': 3}) self.assertEqual(st1.add('this is not a test'), 2) self.assertEqual(st1.add('this is still not a test'), 1) self.assertEqual(st1.meets_threshold, {'this is a test': 3, 'this is not a test': 2}) self.assertEqual(st1.elements_added, 6)
def test_streamthreshold_load(self): ''' test loading a stream threshold sketch from file ''' md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449' filename = 'test.cms' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 100), 100) self.assertEqual(st1.elements_added, 100) self.assertEqual(st1.meets_threshold, {'this is a test': 100}) st1.export(filename) md5_out = calc_file_md5(filename) self.assertEqual(md5_out, md5_val) # try loading directly to file! st2 = StreamThreshold(threshold=10, filepath=filename) self.assertEqual(st2.width, 1000) self.assertEqual(st2.depth, 5) self.assertEqual(st2.elements_added, 100) self.assertEqual(st2.check('this is a test'), 100) # show on load that the tracking of stream threshold is gone self.assertEqual(st2.meets_threshold, dict()) self.assertEqual(st2.add('this is a test', 1), 101) self.assertEqual(st2.meets_threshold, {'this is a test': 101}) os.remove(filename)
def test_streamthreshold_remove(self): ''' test removing elements from the stream threshold singular ''' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 5), 5) self.assertEqual(st1.meets_threshold, dict()) self.assertEqual(st1.add('this is a test', 5), 10) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is not a test', 9), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 10}) self.assertEqual(st1.add('this is a test', 20), 30) self.assertEqual(st1.meets_threshold, {'this is a test': 30}) self.assertEqual(st1.add('this is not a test', 2), 11) self.assertEqual(st1.meets_threshold, {'this is a test': 30, 'this is not a test': 11}) self.assertEqual(st1.remove('this is a test'), 29) self.assertEqual(st1.meets_threshold, {'this is a test': 29, 'this is not a test': 11}) self.assertEqual(st1.remove('this is not a test'), 10) self.assertEqual(st1.remove('this is not a test'), 9) self.assertEqual(st1.meets_threshold, {'this is a test': 29}) self.assertEqual(st1.elements_added, 38)
def test_streamthreshold_load(self): ''' test loading a stream threshold sketch from file ''' md5_val = '61d2ea9d0cb09b7bb284e1cf1a860449' filename = 'test.cms' st1 = StreamThreshold(threshold=10, width=1000, depth=5) self.assertEqual(st1.add('this is a test', 100), 100) self.assertEqual(st1.elements_added, 100) self.assertEqual(st1.meets_threshold, {'this is a test': 100}) st1.export(filename) md5_out = calc_file_md5(filename) self.assertEqual(md5_out, md5_val) # try loading directly to file! st2 = StreamThreshold(threshold=10, filepath=filename) self.assertEqual(st2.width, 1000) self.assertEqual(st2.depth, 5) self.assertEqual(st2.elements_added, 100) self.assertEqual(st2.check('this is a test'), 100) # show on load that the tracking of stream threshold is gone self.assertEqual(st2.meets_threshold, dict()) self.assertEqual(st2.add('this is a test', 1), 101) self.assertEqual(st2.meets_threshold, {'this is a test': 101}) os.remove(filename)
def test_streamthreshold_join(self): """test that stream threshold raises exception""" st1 = StreamThreshold(threshold=1000, width=1000, depth=5) st2 = StreamThreshold(threshold=1000, width=1000, depth=5) self.assertRaises(NotSupportedError, lambda: st1.join(st2))