def find_csv_run(csv_filename):
rank = db.rank_csv
csv_obj = rank.find_one({'name': os.path.basename(csv_filename)})
if csv_obj == None:
pretty.failure("csv object not found")
return False
return csv_obj
def request_alg_run_plot_data(trace_name, algorithm, cache_size, plot,
minio_get):
alg_result = storage.find_trace_run(trace_name, algorithm, cache_size)
alg_result_id = alg_result.get("_id")
alg_result_cache_size = alg_result.get("cache_size")
alg_result_algorithm = alg_result.get("algorithm")
header = "./graph data/" + trace_name + "_" + str(
alg_result_cache_size) + "_" + str(
alg_result_algorithm) + "_" + plot + '.npz'
minio_id = str(alg_result_id) + plot
#print(minio_id)
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#success = minio_get.retrieve(header, minio_id)
success = minio_get.retrieve(tmp.name, minio_id)
if success:
pretty.success(header +
' downloaded succesfully and ready for plotting')
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
pretty.failure("plot data not loaded succesfully")
return success
def find_trace(trace_name):
workloads = db.workload
currentTrace = workloads.find_one({'trace name': trace_name})
if (currentTrace == None):
pretty.failure("trace not found")
return False
else:
return currentTrace
def find_id(trace_name):
workloads = db.workload
currentTrace = workloads.find_one({'trace name': trace_name})
print(currentTrace)
if currentTrace == None:
pretty.failure("Trace not found")
return False
iden = str(currentTrace.get("_id"))
#print(iden)
return iden
def catch_json_errors(conf):
try:
conf = json.loads(conf)
return conf
except json.JSONDecodeError as e:
pretty.failure(
"There was an error decoding the uploaded file to JSON it is possible you are missing a comma or have written a decimal value without a leading zero"
)
pretty.failure(e)
return False
def mongo_get_alg_run_plot_data(config,
plot,
trace_name,
algorithm,
cache_size,
minio_get,
app_id=None):
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
print('in mongo get plot data')
if app_id != None:
conduit.current_status[app_id] = "checking if data has been run..."
'''if os.path.exists(filename):
pretty.success("file already downloaded to disk")
if app_id != None:
conduit.current_status[app_id] = "File already processed and ready, rendering..."
return filename'''
previously_run = storage.find_trace_run_id(trace_name, algorithm,
cache_size)
print(previously_run, "previously run?")
if previously_run == False:
conduit.current_status[
app_id] = "data has not been run before... processing"
pretty.utility(
"trace run request: " + trace_name + algorithm + str(cache_size) +
" was not found in the database... inserting into database")
if app_id != None:
conduit.current_status[
app_id] = "trace run request: " + trace_name + " " + algorithm + " " + str(
cache_size
) + " was not found in the database... processing and inserting into database"
confirmed = sg.getAlgStats(algorithm, cache_size, config, trace_name,
app_id)
if confirmed:
out_file = request_alg_run_plot_data(trace_name, algorithm,
cache_size, plot, minio_get)
return out_file
else:
pretty.failure("upload to database failed")
return False
else:
conduit.current_status[
app_id] = "data has been run before... retrieving data"
out_file = request_alg_run_plot_data(trace_name, algorithm, cache_size,
plot, minio_get)
return out_file
def retrieve(self, file, id):
pretty.utility("Minio contacted >>>")
try:
self.s3.Bucket('visual-data').download_file(
id, file) #attemps to download
except ClientError:
pretty.failure("File does not exist") #message if file error
return False
except EndpointConnectionError:
pretty.failure(
"Connection was not established") #message if connection error
return False
pretty.success("Minio retrieval successful <<< ")
return True
def insert(self, file, id):
pretty.contact("IN MINIO INSERT METHOD --- CURRENTLY INSERTING " +
file)
try:
self.s3.Bucket('visual-data').upload_file(file,
id) #attempts to upload
except ClientError:
pretty.failure(
"File does not exist on your pc") #message if file error
return False
except EndpointConnectionError:
print(EndpointConnectionError)
pretty.failure("Connection was not established"
) # message if connection error
return False
pretty.success(file + " uploaded")
return True
def catch_config_errors(config):
valid_algs = [
'alecar6', 'arc', 'arcalecar', 'cacheus', 'dlirs', 'lecar', 'lfu',
'lirs', 'lirsalecar', 'lru', 'mru'
]
invalid_algs = []
for algs in set(config['algorithms']):
if algs.lower() not in valid_algs:
pretty.failure("invalid algorithm:" + algs + " detected")
invalid_algs.append(algs)
#invalid_algs = True
invalid_cache_size = []
for sizes in config['cache_sizes']:
st = str(sizes)
if sizes > 1.0:
invalid_cache_size.append(sizes)
if sizes <= 0:
invalid_cache_size.append(sizes)
return invalid_algs, invalid_cache_size
def s_minio_confirm(traceId, traceName):
toMinio = Minio_Module.Minio_Module()
hId = traceId + '-histogram'
#rqfId = traceId + 'rqf'
rqId = traceId + '-rq'
reuId = traceId + '-reuse'
timeId = traceId + '-time'
if toMinio.exist(hId):
storage.workload_confirmed(traceName, requestFrequency=1)
else:
pretty.failure("Request frequency upload for " + traceName +
" to minio failed")
return False
if toMinio.exist(rqId):
storage.workload_confirmed(traceName, request=1)
else:
pretty.failure("Request frequency upload for " + traceName +
" to minio failed")
return False
if toMinio.exist(reuId):
storage.workload_confirmed(traceName, reuseDistance=1)
else:
pretty.failure("Reuse distance upload for " + traceName +
" to minio failed")
return False
if toMinio.exist(timeId):
storage.workload_confirmed(traceName, time=1)
else:
pretty.failure("Real time array upload for " + traceName +
" to minio failed")
return False
pretty.success("s_minio_confirm" + traceName +
"confirmed uploaded to database")
return True
def as_minio_confirm(traceId, traceName, algname, cache_size):
fullTraceName = traceName + " " + algname + " " + str(cache_size)
toMinio = Minio_Module.Minio_Module()
pretty.utility("CONFIRMING IF ARRAYS FOR " + fullTraceName +
" WERE UPLOADED SUCCESSFULLY")
hit_rate_id = traceId + '-hit_rate'
miss_rate_id = traceId + '-miss_rate'
pollution_id = traceId + '-pollution'
if algname.lower() == 'dlirs':
dlirs_stack_id = traceId + '-dlirs_stack'
if toMinio.exist(dlirs_stack_id):
storage.trace_run_confirmed(traceId, dlirs=1)
else:
return False
if algname.lower() == 'lecar':
lecar_lru_id = traceId + '-lecar_lru'
if toMinio.exist(lecar_lru_id):
storage.trace_run_confirmed(traceId, lecar=1)
else:
return False
if algname.lower() == 'arc':
arc_p_id = traceId + '-arc_p'
if toMinio.exist(arc_p_id):
storage.trace_run_confirmed(traceId, arc=1)
else:
return False
if algname.lower() == 'cacheus':
cacheus_learning_rate_id = traceId + '-cacheus_learning_rate'
cacheus_lru_id = traceId + '-cacheus_lru'
cacheus_q_id = traceId + '-cacheus_q'
if toMinio.exist(cacheus_learning_rate_id) and toMinio.exist(
cacheus_lru_id) and toMinio.exist(cacheus_q_id):
storage.trace_run_confirmed(traceId, cacheus=1)
else:
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, hit_rate=1)
else:
pretty.failure("Hit rate array for " + fullTraceName +
"to minio failed")
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, miss_rate=1)
else:
pretty.failure("Miss rate array for " + fullTraceName +
"to minio failed")
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, pollution=1)
else:
pretty.failure("Pollution rate array for " + fullTraceName +
"to minio failed")
return False
pretty.success(
"==============================" + fullTraceName +
" succesfully uploaded ==========================================")
return True
def mongo_get_trace_run_plot_data(trace_name, plot, minio_get, app_id=None):
'''filename = "./graph data/"+str(trace_name) + str(plot) +'.npz'
if os.path.exists(filename):
pretty.success("file already downloaded to disk")
return filename'''
print(plot, "********************************")
print(trace_name)
trace = storage.find_trace(trace_name)
if trace == False:
pretty.utility("plot data for " + trace_name + plot +
" was not found, inserting into database...")
if app_id != None:
conduit.curret_status[
app_id] = "plot data for " + trace_name + " " + plot + " was not found, inserting into database..."
confirmed = sg.sf_singleTraceGen(trace_name, app_id)
trace = storage.find_trace(trace_name)
trace_id = trace.get("_id")
minio_id = str(trace_id) + plot
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#success = minio_get.retrieve(header, minio_id)
#success = minio_get.retrieve(tmp.name, minio_id)
if confirmed:
success = minio_get.retrieve(tmp.name, minio_id)
if success:
return tmp # tmp.name
else:
return False
else:
pretty.failure("error uploading trace data to database")
return False
else:
trace_id = trace.get("_id")
trace_ios = trace.get("requests")
#print("Mongo contacted")
#print(trace_id, " received id for", trace_name, plot)
minio_id = str(trace_id) + plot
#print("mongo_get_trace_run_plot_data", minio_id)
pretty.utility("retrieving plot data from minio...")
tmp = tempfile.NamedTemporaryFile(mode='w+b', suffix='.npz')
success = minio_get.retrieve(tmp.name, minio_id)
if success:
pretty.success("plot data for " + trace_name + plot +
" successfully downloaded")
#print(filename)
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
pretty.failure(
"data not found in minio store... processing trace data...")
confirmed = sg.sf_singleTraceGen(trace_name, minio_only=True)
success = minio_get.retrieve(tmp.name, minio_id)
if success:
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
print("error uploading data to minio")
return False
def compress_config(file):
print("in compress config")
name = "test"
np.savez_compressed(name, file)
if __name__ == '__main__':
import sys
import math
import json
import os
par = get_parser()
args, unknown = get_parser().parse_known_args()
if len(sys.argv) == 1:
pretty.failure("no arguments passed")
par.print_help()
sys.exit()
#check if the filepath passed as an argument is a valid filepath
if (os.path.exists(sys.argv[1])):
print(os.path.basename(sys.argv[1]))
with open(sys.argv[1], 'r') as f:
config = json.loads(f.read())
name = os.path.basename(sys.argv[1])
#storage.get_all_configs()
#storage.remove_all_configs()
'''if os.path.exists(config['output_path']) == False:
def visualize_head_to_head(config, flag, y_scale=None):
pretty.utility(
"generating comparison data for all pairs of algorithms and cache sizes"
)
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
current_plot = plots[flag]
#if current_plot[0] == 'algorithm' and current_plot[6] == False:
all_pairs = gen_all_pairs(config['algorithms'])
all_plots = {}
# generate all output arrays here and add them to a dictionary to be referenced for the graphing of all pairs to prevent repeatedly downloading and extracting
# the same data
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for algorithms in config['algorithms']:
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name, algorithms,
cache_size, minio_get)
if out_file:
out_file = np.load(out_file, allow_pickle=True)
out_file = out_file['arr_0']
all_plots[str(trace_name) + str(algorithms) +
str(cache_size)] = out_file
else:
pretty.failure("error downloading " + trace_name +
current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage " +
str(percent_cache))
compare = []
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
over_time = serv.get_over_time_list(trace_name, minio_get)
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(cache_size *
serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
for pairs in all_pairs:
compare.append(
all_plots[str(trace_name) + str(pairs[0]) +
str(cache_size)])
compare.append(
all_plots[str(trace_name) + str(pairs[1]) +
str(cache_size)])
over_time = serv.get_over_time_list(
trace_name, minio_get)
basename = os.path.basename(trace_name)
name = basename + " " + " cache size " + str(
cache_size) + " " + str(
pairs[0]) + " compared with " + str(pairs[1])
print(name)
graph.vis_data_multi(current_plot[1],
current_plot[3][1],
current_plot[4],
name,
compare,
pairs,
config['output_path'],
x_data=over_time,
scale=y_scale,
fill=True)
compare = []
def s_generate_graph_data(config, params, app_id):
print("s generate graph data")
minio_get = Minio_Module.Minio_Module()
current_plot = plots[params['plot']]
print(params, "s generate graph data")
if current_plot[0] == 'trace':
out_file = serv.mongo_get_trace_run_plot_data(params['trace_name'],
current_plot[2],
minio_get, app_id)
if out_file:
#out_file = np.load(out_file, allow_pickle = True)
if current_plot[1] == 'histogram':
if params['plot'] == '-g':
conduit.current_status[
app_id] = "Processing histogram, please wait... This process may take over 10 minutes."
x_axis, frequency, uniques = process_histogram(
params['plot'], out_file)
return x_axis, frequency, (str(params['trace_name']) +
' Number of Unique Accesses: ' +
str(uniques))
else:
conduit.current_status[
app_id] = "Processing histogram, please wait... This process may take over 10 minutes."
x_axis, frequency = process_histogram(
params['plot'], out_file)
conduit.current_status[app_id] = "Rendering " + params[
'trace_name'] + " graph..."
return x_axis, frequency, str(params['trace_name'])
else:
over_time = serv.get_over_time_list(params['trace_name'],
minio_get, app_id)
conduit.current_status[
app_id] = "Rendering " + params['trace_name'] + " graph..."
return over_time, out_file['arr_0'][()], str(
params['trace_name'])
else:
pretty.failure("problem with retrieving files from database")
elif current_plot[0] == 'algorithm':
out_file = serv.mongo_get_alg_run_plot_data(config, current_plot[2],
params['trace_name'],
params['algorithm'],
int(params['cache_size']),
minio_get, app_id)
plot_title = current_plot[5][1]
if out_file:
full_title = str(params['trace_name']) + " " + str(
params['algorithm']) + "_" + str(
params['cache_size']
) + " " + current_plot[4] + " " + plot_title
#out_file = np.load(out_file, allow_pickle = True)
conduit.current_status[
app_id] = "Generating over time data for" + params[
'trace_name'] + "please wait..."
over_time = serv.get_over_time_list(params['trace_name'],
minio_get, app_id)
conduit.current_status[
app_id] = "Rendering " + params['trace_name'] + " graph..."
return over_time, out_file['arr_0'][()], full_title
else:
conduit.current_status[app_id] = "error getting graph data"
return None, None, None
def visualize_data(flag, config, y_scale=None):
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
current_plot = plots[flag]
#plots that use trace data have a different, smaller loop than the plots using data from algorithms
if current_plot[0] == 'trace':
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
print(trace_name + "=======================================")
#request data from mongo
out_file = serv.mongo_get_trace_run_plot_data(
trace_name, current_plot[2], minio_get)
if out_file:
#load that data
out_file = np.load(out_file,
allow_pickle=True) #uncompress
if current_plot[1] == 'histogram':
#---
if flag == '-g':
x_axis, frequency, uniques = process_histogram(
flag, out_file)
graph.vis_data(
current_plot[1],
current_plot[3],
current_plot[4],
str(trace_name) + " " + current_plot[5] +
" Number of Uniques: " + str(uniques),
frequency,
config['output_path'],
x_data=x_axis)
else:
x_axis, frequency = process_histogram(
flag, out_file)
#print(frequency)
'''graph.vis_data(current_plot[1], current_plot[3], current_plot[4],
str(trace_name) + " " + current_plot[5], frequency, x_data = x_axis)'''
#this is hard coded as line because it will not render as a bar graph, it lags and never finishes
graph.vis_data('line',
current_plot[3],
current_plot[4],
str(trace_name) + " " +
current_plot[5],
frequency,
config['output_path'],
x_data=x_axis)
#---
else:
#---
over_time = serv.get_over_time_list(
trace_name, minio_get)
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
str(trace_name) + " " + current_plot[5],
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#---
else:
pretty.failure("error downloading" + trace_name +
'access pattern')
#if it is an algorithm data plot and that data is not algorithm specific, run a loop for all algorithms
elif current_plot[0] == 'algorithm' and current_plot[6] == False:
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for algorithms in config['algorithms']:
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name,
algorithms, cache_size, minio_get)
plot_title = current_plot[5][1]
#print(plot_title)
if out_file:
full_title = str(trace_name) + " " + str(
algorithms) + "_" + str(
cache_size) + " " + current_plot[
4] + " " + plot_title
out_file = np.load(out_file, allow_pickle=True)
over_time = serv.get_over_time_list(
trace_name, minio_get)
#print(trace_name, algorithms, cache_size)
#print("=====================")
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
full_title,
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#print("++++++++++++++++++++")
else:
pretty.failure("error downloading " +
trace_name + current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage "
+ str(percent_cache))
#else, it is an algorithm specific plot. the only difference here is that there is not looping through the list of algorithms
elif current_plot[0] == 'algorithm':
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
#all of the algorithm specific plots have the name of the algorithm before the '_'
#so that is why algorithm_used[0] is called, we must also remove the '-' that is in the zero index
algorithm_used = current_plot[2].split('_')
algorithm_used = algorithm_used[0]
algorithm_used = algorithm_used[1:len(algorithm_used)]
#pretty.utility(algorithm_used)
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name,
algorithm_used, cache_size, minio_get)
plot_title = current_plot[5][1]
#pretty.utility(plot_title)
if out_file:
full_title = str(trace_name) + " " + str(
algorithm_used) + "_" + str(
cache_size
) + " " + current_plot[4] + " " + plot_title
out_file = np.load(out_file, allow_pickle=True)
over_time = serv.get_over_time_list(
trace_name, minio_get)
pretty.utility(trace_name + algorithm_used +
str(cache_size))
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
full_title,
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#---
#
else:
pretty.failure("error downloading " + trace_name +
current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage "
+ str(percent_cache))