def test5_bReader(self):
reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
init_params={"label":6, "real_time":7, "fmt": "<3I2H2Q"})
cH = cHeatmap()
bpr = cH.get_breakpoints(reader, 'r', time_interval=1000000)
self.assertEqual(bpr[10], 53)
bpv = cH.get_breakpoints(reader, 'v', time_interval=1000)
self.assertEqual(bpv[10], 10000)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_pixels=100, num_of_threads=4, cache_size=2000)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_threads=4, cache_size=2000)
cH.heatmap(reader, 'r', "rd_distribution",
num_of_pixels=1000, num_of_threads=4)
cH.heatmap(reader, 'r', "future_rd_distribution",
num_of_pixels=1000, num_of_threads=4)
cH.heatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_pixels=100, algorithm="FIFO",
num_of_threads=4, cache_size=200)
cH.diffHeatmap(reader, 'r', "hit_ratio_start_time_end_time",
num_of_pixels=24, cache_size=200,
algorithm1="LRU", algorithm2="Optimal",
cache_params2=None, num_of_threads=4)
def plotHRCs(self, op=None, clf=True, save=True, figname="HRC_multi_trace.pdf", **kwargs):
if op is None:
if len(self.hrs) == 0:
self.get_hit_rate()
for i in range(len(self.hrs)):
label = self.readers[i].file_loc[self.readers[i].file_loc.rfind('/')+1:]
plt.plot(np.arange(0, self.bin_size*(len(self.hrs[i])-2), self.bin_size),
self.hrs[i][:-2], label=label)
elif op == "+" or op == "add" or op == "addition":
assert 'mix_mode' in kwargs, "please provide mix_mode when doing trace mixing/addition"
# the following only works on Linux
mixed_trace = tempfile.NamedTemporaryFile()
traceMixer.mix_asBinary(readers=self.readers, mix_mode="real_time", output=mixed_trace.name)
mixed_reader = binaryReader(mixed_trace.name, init_params={"label": 1, "fmt": "<L"})
if self.cache_name == "LRU":
profiler = LRUProfiler(mixed_reader, cache_size=self.cache_size)
else:
profiler = cGeneralProfiler(mixed_reader, cache_name=self.cache_name,
cache_size=self.cache_size, bin_size=self.bin_size,
cache_params=self.cache_params, num_of_threads=self.num_of_threads)
hr = profiler.get_hit_rate()
plt.plot(np.arange(0, self.bin_size * (len(hr) - 2), self.bin_size),
hr[:-2], label="mixed")
if save:
plt.legend(loc="best")
plt.xlabel("Cache size/items")
plt.ylabel("Hit Ratio")
plt.savefig(figname)
if clf:
plt.clf()
def test_reader_binary(self):
reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
data_type='l',
init_params={
"label": 6,
"real_time": 7,
"fmt": "<3I2H2Q"
})
lines = 0
for _ in reader:
lines += 1
self.assertEqual(lines, 113872)
reader.reset()
first_request = reader.read_one_element()
print(first_request)
self.assertEqual(int(first_request), 42932745)
def get_hit_ratio_shards(self, sample_ratio=0.01, **kwargs):
"""
experimental function
:param sample_ratio:
:param kwargs:
:return:
"""
from mimircache.cacheReader.tracePreprocesser import tracePreprocessor
kargs = {}
if 'cache_size' not in kwargs:
kargs['cache_size'] = self.cache_size
else:
kargs['cache_size'] = kwargs['cache_size']
pp = tracePreprocessor(self.reader)
N1, N2, traceName, fmt = pp.prepare_for_shards(
sample_ratio=sample_ratio, has_time=False)
correction = N2 - N1
print("correction: {}".format(correction))
# correction = 0
tempReader = binaryReader(traceName,
init_params={
"label": 1,
"fmt": fmt
})
if self.block_unit_size != 0:
print("not supported yet")
return None
else:
hit_ratio = mimircache.c_LRUProfiler.get_hit_ratio_seq_shards(
tempReader.cReader,
sample_ratio=sample_ratio,
correction=correction,
**kargs)
return hit_ratio
maxLine = range[1]
if maxLine == -1:
maxLine = len(reader)
line_count = 0
# reader.jumpN(range[0])
for line in reader:
line_count += 1
if line_count - 1 < range[0]: # -1 because we add 1 before checking
continue
if line_count - 1 >= maxLine:
break
writer.write(line)
if __name__ == "__main__":
# vReaderToPReader(sys.argv[1], sys.argv[2])
# reader = vscsiReader("../data/traces/w38_vscsi1.vscsitrace")
# reader1 = vscsiReader("../data/trace.vscsi")
# reader1 = vscsiReader("../data/traces/w106_vscsi1.vscsitrace")
# reader2 = vscsiReader("../data/traces/w100_vscsi1.vscsitrace")
# trace_mixer(reader1, reader2, mix_mode="round_robin", round_robin_n=2)
# trace_mixer(reader1, reader2, mix_mode="real_time")
# splitTrace(reader, 2, "w38Split")
# splitTraceWithTime("/var/dept/scratch/jyan254/cache/mimircache/data/traces/w65_vscsi1.vscsitrace")
with binaryReader("../data/trace.vscsi", "<3I2H2Q") as reader:
with traceBinaryWriter("test.vscsi", "<3I2H2Q") as writer:
extract_part_trace(reader, writer, (1, -1))
with binaryReader("../utils/test.vscsi", "<3I2H2Q") as r:
print(len(r))
def test_Optimal(self):
reader = vscsiReader("{}/trace.vscsi".format(DAT_FOLDER))
p = cGeneralProfiler(reader, "Optimal", cache_size=2000)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
hr = p.get_hit_ratio(begin=113852, end=113872, cache_size=5000)
self.assertAlmostEqual(hr[1], 0.2)
reader.close()
reader = plainReader("{}/trace.txt".format(DAT_FOLDER))
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
p.plotHRC("test2.png", cache_unit_size=32 * 1024)
reader.close()
reader = csvReader("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'delimiter': ','
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()
reader = binaryReader("{}/trace.vscsi".format(DAT_FOLDER),
init_params={
"label": 6,
"real_time": 7,
"fmt": "<3I2H2Q"
})
p = cGeneralProfiler(reader,
"Optimal",
cache_size=2000,
num_of_threads=4)
hr = p.get_hit_ratio()
self.assertAlmostEqual(hr[0], 0.0)
self.assertAlmostEqual(hr[100], 0.28106996417045593)
hc = p.get_hit_count()
self.assertEqual(hc[10], 180)
self.assertEqual(hc[0], 0)
mr = p.get_miss_rate()
self.assertAlmostEqual(mr[-1], 0.71893000602722168)
reader.close()