def test_reader_v(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = LRUProfiler(reader)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[2000], 0.172851974146)
hc = p.get_hit_count()
self.assertEqual(hc[20002], 0)
self.assertEqual(hc[0], 0)
mr = p.get_miss_ratio()
self.assertEqual(hr[-1], mr[-1])
#
rd = p.get_reuse_distance()
print(rd)
self.assertEqual(rd[1024], -1)
self.assertEqual(rd[113860], 1)
c_LRUProfiler.get_future_reuse_dist(reader.cReader)
hr = p.get_hit_ratio(begin=113852, end=113872)
print(hr)
self.assertEqual(hr[8], 0.2)
hr = p.get_hit_ratio(cache_size=20)
self.assertAlmostEqual(hr[1], 0.02357911)
hr = p.get_hit_ratio(cache_size=5, begin=113852, end=113872)
self.assertAlmostEqual(hr[2], 0.05)
reader.close()
def test_LRU_2(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader, "LRU_2", cache_size=2000)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.16544891893863678)
hc = p.get_hit_count()
self.assertEqual(hc[10], 164)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.83455109596252441)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
reader.close()
def test_SLRU(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader,
"SLRU",
cache_size=2000,
cache_params={"N": 2})
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.1767423003911972)
hc = p.get_hit_count()
self.assertEqual(hc[10], 117)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.8232576847076416)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
reader.close()
def test_FIFO(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader, "FIFO", cache_size=2000, num_of_threads=4)
p2 = generalProfiler(reader, 'FIFO', cache_size=2000, num_of_threads=4)
hr = p.get_hit_ratio()
hr2 = p2.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], hr2[100])
self.assertAlmostEqual(hr[100], 0.16934804618358612)
hc = p.get_hit_count()
self.assertEqual(hc[10], 449)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.83065193891525269)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
p.plotHRC("test.png", cache_unit_size=32 * 1024)
reader.close()
def test1_vReader(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
cH = cHeatmap()
bpr = cH.get_breakpoints(reader, 'r', time_interval=1000000)
self.assertEqual(bpr[10], 53)
bpr = cH.get_breakpoints(reader, 'r', num_of_pixels=1000)
# print(bpr)
bpv = cH.get_breakpoints(reader, 'v', time_interval=1000)
self.assertEqual(bpv[10], 10000)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
time_interval=10000000, num_of_threads=4, cache_size=200)
cH.heatmap(reader, 'r', "rd_distribution",
time_interval=10000000, num_of_threads=4)
cH.heatmap(reader, 'r', "future_rd_distribution",
time_interval=10000000, num_of_threads=4)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
time_interval=10000000, algorithm="FIFO",
num_of_threads=4, cache_size=2000)
cH.diffHeatmap(reader, 'r', "hit_ratio_start_time_end_time",
cache_size=200, time_interval=100000000,
algorithm1="LRU", algorithm2="Optimal",
cache_params2=None, num_of_threads=4)
def test_Optimal(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader, "Optimal", cache_size=2000)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
reader.close()
reader = plainReader("{}/trace.txt".format(DAT_FOLDER))
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
p.plotHRC("test2.png", cache_unit_size=32 * 1024)
reader.close()
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'delimiter': ','
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()
reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
init_params={
"label": 6,
"real_time": 7,
"fmt": "<3I2H2Q"
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()
while r:
h = mmh3.hash(str(r)) % self.modulo
if h <= self.modulo * sample_ratio:
if r not in mapping_dict:
mapping_dict[r] = counter
counter += 1
writer.write((mapping_dict[r], ))
written_records += 1
r = self.reader.read_one_element()
writer.close()
return (int(self.N * sample_ratio), written_records, ofilename, fmt)
def close(self):
os.remove(self.shards_file_loc)
if __name__ == "__main__":
from mimircache.cacheReader.vscsiReader import vscsiReader
from mimircache.cacheReader.csvReader import csvReader
reader = vscsiReader("../data/trace.vscsi")
reader2 = csvReader("/home/jason/ALL_DATA/Akamai/201610.all.sort.clean",
init_params={
"header": False,
"delimiter": "\t",
"label_column": 5,
'real_time_column': 1
})
print(
tracePreprocessor(reader).prepare_for_shards(has_time=True, fmt="<LL"))
def test_context_manager(self):
with vscsiReader("{}/trace.vscsi".format(DAT_FOLDER)) as reader:
self.assertEqual(reader.get_num_of_req(), 113872)