def _test_overall(self):
c = Cachecow()
c.csv("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'real_time': 2,
"delimiter": ","
})
self.assertEqual(len(c), 113872)
stat = c.characterize("short", print_stat=False)
for line in stat:
if "number of requests" in line:
self.assertEqual(int(line.split(":")[1].strip()), 113872)
elif "number of uniq obj/blocks" in line:
self.assertEqual(int(line.split(":")[1].strip()), 48973)
elif "cold miss ratio" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()), 0.4301)
elif "number of obj/block accessed only once" in line:
self.assertEqual(int(line.split(":")[1].strip()), 21048)
elif "frequency mean" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()), 2.33)
elif "time span" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()),
7199847246.0)
def get_hrc_for_file(file_name, algorithm_name):
logging.info("Working on file: {} and algorithm: {}".format(
file_name, algorithm_name))
actual_name = file_name.split("/")[-1]
actual_name = actual_name.split(".")[0]
json_file_name = get_hit_ratio_filename(actual_name, algorithm_name)
if not json_file_name:
logging.info("File {} alread has algo {}".format(
file_name, algorithm_name))
return 1
c = Cachecow()
if "vscsi1" in file_name:
trace_type = 1
else:
trace_type = 2
reader = c.vscsi(file_name, vscsi_type=trace_type)
trace_length = reader.get_num_of_uniq_req()
del reader
dict_value = c.get_hit_ratio_dict(algorithm_name,
cache_size=trace_length,
cache_params=None,
bin_size=-1)
json_fp = open(json_file_name, "w")
json.dump(obj=dict_value, fp=json_fp)
del dict_value
def _test_overall(self):
c = Cachecow()
c.csv("{}/trace.csv".format(DAT_FOLDER),
init_params={"header" :True, 'label' :5, 'real_time':2, "delimiter": ","})
self.assertEqual(len(c), 113872)
stat = c.characterize("short", print_stat=False)
for line in stat:
if "number of requests" in line:
self.assertEqual(int(line.split(":")[1].strip()), 113872)
elif "number of uniq obj/blocks" in line:
self.assertEqual(int(line.split(":")[1].strip()), 48973)
elif "cold miss ratio" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()), 0.4301)
elif "number of obj/block accessed only once" in line:
self.assertEqual(int(line.split(":")[1].strip()), 21048)
elif "frequency mean" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()), 2.33)
elif "time span" in line:
self.assertAlmostEqual(int(line.split(":")[1].strip()), 7199847246.0)
def _test_basic(self, param_sets):
c = Cachecow()
c.open('{}/trace.txt'.format(DAT_FOLDER))
for param_set in param_sets:
if "time_mode" not in param_set:
self._coretest(c, param_set)
c.close()
c = Cachecow()
c.csv("{}/trace.csv".format(DAT_FOLDER),
init_params={
"header": True,
'label': 5,
'real_time': 2
})
for param_set in param_sets:
self._coretest(c, param_set)
c.close()
c = Cachecow()
c.vscsi('{}/trace.vscsi'.format(DAT_FOLDER))
for param_set in param_sets:
self._coretest(c, param_set)
c.close()
def convert_to_csv(file_name):
logging.info("Processing File: {}".format(file_name))
type_1_dict = config.config_["TYPE_1_INDICES"]
type_2_dict = config.config_["TYPE_2_INDICES"]
i = 1
if "vscsi1" in file_name:
trace_type = 1
else:
trace_type = 2
act_name = ret_file_name_csv(filename=file_name)
c = Cachecow()
try:
reader = c.vscsi(file_path=file_name, vscsi_type=trace_type)
except AssertionError as e:
logging.info("The exception is {}".format(e))
logging.info("Excepted file name is {}".format(file_name))
return
if trace_type == 1:
data = reader.read_complete_req()
csv_file = open(act_name, mode='w')
csv_writer = csv.DictWriter(csv_file, fieldnames=type_1_dict.keys())
csv_writer.writeheader()
while data:
if i % 10000 == 0:
logging.info(
"Finished {} records of type 1 and file {}".format(
i, file_name))
temp_dict = {
"LEN": data[type_1_dict["LEN"]],
"OP_CODE": data[type_1_dict["OP_CODE"]],
"BLOCK_NUMBER": data[type_1_dict["BLOCK_NUMBER"]],
"TIME_STAMP": data[type_1_dict["TIME_STAMP"]],
}
csv_writer.writerow(temp_dict)
# print(data)
data = reader.read_complete_req()
i = i + 1
else:
csv_file = open(act_name, mode='w')
csv_writer = csv.DictWriter(csv_file, fieldnames=type_2_dict.keys())
data = reader.read_complete_req()
csv_writer.writeheader()
while data:
if i % 10000 == 0:
logging.info(
"Finished {} records of type 1 and file {}".format(
i, file_name))
temp_dict = {
"LEN": data[type_2_dict["LEN"]],
"OP_CODE": data[type_2_dict["OP_CODE"]],
"BLOCK_NUMBER": data[type_2_dict["BLOCK_NUMBER"]],
"TIME_STAMP": data[type_2_dict["TIME_STAMP"]],
"RESPONSE_TIME": data[type_2_dict["RESPONSE_TIME"]],
}
csv_writer.writerow(temp_dict)
# print(temp_dict)
data = reader.read_complete_req()
i = i + 1
return 1
# Eviction Process
eviction_process(cache, int(cache_size / 50),
int(cache_size / 500), ts)
print(cache_size, str(n_hits / length))
hit_ratios.append(n_hits / length)
t3 = time.time()
print('Time to Evaluate: ', str(t3 - t2))
'''
Comparison and Plotting
'''
# Setup PyMimircache for Comparison
c = Cachecow()
c.open('temporary/temp_trace_' + file_name + '.txt')
comparison_lst = ['Optimal', 'LRU', 'LFU', 'Random', 'SLRU', 'ARC']
# Chaining lol
comparison_hrs = [[
c.profiler(alg, cache_size=size).get_hit_ratio() for size in cache_sizes
] for alg in comparison_lst]
comparison_hrs.append(hit_ratios)
comparison_lst.append('SmarterCache')
comparison_df = pd.DataFrame(data=comparison_hrs,
index=comparison_lst,
columns=cache_sizes)
#!/usr/bin/env python3
from PyMimircache import Cachecow
if __name__ == '__main__':
c = Cachecow()
c.vscsi(
"trace.vscsi"
) # this file is in the data folder on GitHub, other data types also supported
print(c.stat())
print(c.get_reuse_distance())
print(c.get_hit_ratio_dict("LRU", cache_size=20))
c.plotHRCs(["LRU", "LFU", "Optimal"])
c.heatmap('r',
"hit_ratio_start_time_end_time",
time_interval=10000000,
cache_size=2000)
def _test_basic(self, param_sets):
c = Cachecow()
c.open('{}/trace.txt'.format(DAT_FOLDER))
for param_set in param_sets:
if "time_mode" not in param_set:
self._coretest(c, param_set)
c.close()
c = Cachecow()
c.csv("{}/trace.csv".format(DAT_FOLDER),
init_params={"header" :True, 'label' :5, 'real_time':2})
for param_set in param_sets:
self._coretest(c, param_set)
c.close()
c = Cachecow()
c.vscsi('{}/trace.vscsi'.format(DAT_FOLDER))
for param_set in param_sets:
self._coretest(c, param_set)
c.close()
# Eviction Process
eviction_process(cache, int(cache_size / 50),
int(cache_size / 500), ts)
print(cache_size, str(n_hits / length))
hit_ratios.append(n_hits / length)
t3 = time.time()
print('Time to Evaluate: ', str(t3 - t2))
'''
Comparison and Plotting
'''
# Setup PyMimircache for Comparison
c = Cachecow()
c.open('temporary/temp_trace_' + file_name + '.txt')
comparison_lst = ['Optimal', 'LRU', 'LFU', 'Random', 'SLRU', 'ARC']
hit_ratio_dicts = [
c.get_hit_ratio_dict(comparison_alg, cache_size=max_cache_size)
for comparison_alg in comparison_lst
]
# Inefficient process to convert dict kv pairs to sorted lists but oh well
comp_x = [sorted(list(hr_dict.keys())) for hr_dict in hit_ratio_dicts]
comp_hr = [sorted(list(hr_dict.values())) for hr_dict in hit_ratio_dicts]
ml_x = cache_sizes
plt.figure(0)