def test_LFU_LFUFast(self):
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'delimiter': ','
})
p = cGeneralProfiler(reader, "LFU", cache_size=2000, num_of_threads=4)
p2 = cGeneralProfiler(reader,
"LFUFast",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
hr2 = p2.get_hit_ratio()
self.assertCountEqual(hr, hr2)
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.17430096864700317)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.82569903135299683)
p2.plotHRC("test.png", cache_unit_size=32 * 1024)
reader.close()
def test_reader_c(self):
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
"label": 5
})
p = LRUProfiler(reader)
rd = p.get_reuse_distance()
print(rd)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[2000], 0.172851974146)
hc = p.get_hit_count()
self.assertEqual(hc[20002], 0)
self.assertEqual(hc[0], 0)
mr = p.get_miss_ratio()
self.assertEqual(hr[-1], mr[-1])
rd = p.get_reuse_distance()
print(rd)
self.assertEqual(rd[1024], -1)
self.assertEqual(rd[113860], 1)
c_LRUProfiler.get_future_reuse_dist(reader.cReader)
hr = p.get_hit_ratio(begin=113852, end=113872)
self.assertEqual(hr[8], 0.2)
hr = p.get_hit_ratio(cache_size=20)
self.assertAlmostEqual(hr[1], 0.02357911)
hr = p.get_hit_ratio(cache_size=5, begin=113852, end=113872)
self.assertAlmostEqual(hr[2], 0.05)
p.plotHRC("test.png", cache_unit_size=32 * 1024)
reader.close()
def test4_cReader_r(self):
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={"header":True, "label":5, 'real_time':2})
cH = cHeatmap()
bpr = cH.get_breakpoints(reader, 'r', time_interval=1000000)
self.assertEqual(bpr[10], 53)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_pixels=24, num_of_threads=4, cache_size=2000)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_threads=4, cache_size=2000)
cH.heatmap(reader, 'r', "rd_distribution",
num_of_pixels=1000, num_of_threads=4)
cH.heatmap(reader, 'r', "future_rd_distribution",
num_of_pixels=1000, num_of_threads=4)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_pixels=100, algorithm="FIFO",
num_of_threads=4, cache_size=2000)
cH.diffHeatmap(reader, 'r', "hit_ratio_start_time_end_time",
time_interval=100000000, cache_size=200,
algorithm1="LFUFast", algorithm2="Optimal",
cache_params2=None, num_of_threads=4)
def test_Optimal(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader, "Optimal", cache_size=2000)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
reader.close()
reader = plainReader("{}/trace.txt".format(DAT_FOLDER))
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
p.plotHRC("test2.png", cache_unit_size=32 * 1024)
reader.close()
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'delimiter': ','
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()
reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
init_params={
"label": 6,
"real_time": 7,
"fmt": "<3I2H2Q"
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()
while r:
h = mmh3.hash(str(r)) % self.modulo
if h <= self.modulo * sample_ratio:
if r not in mapping_dict:
mapping_dict[r] = counter
counter += 1
writer.write((mapping_dict[r], ))
written_records += 1
r = self.reader.read_one_element()
writer.close()
return (int(self.N * sample_ratio), written_records, ofilename, fmt)
def close(self):
os.remove(self.shards_file_loc)
if __name__ == "__main__":
from mimircache.cacheReader.vscsiReader import vscsiReader
from mimircache.cacheReader.csvReader import csvReader
reader = vscsiReader("../data/trace.vscsi")
reader2 = csvReader("/home/jason/ALL_DATA/Akamai/201610.all.sort.clean",
init_params={
"header": False,
"delimiter": "\t",
"label_column": 5,
'real_time_column': 1
})
print(
tracePreprocessor(reader).prepare_for_shards(has_time=True, fmt="<LL"))