def get_over_time_list(trace_name, minio_get, app_id=None):
print("in get over time list")
'''filename = './graph data/' + trace_name + '_time.npz'
if os.path.exists(filename):
pretty.success("file already downloaded to disk")
data_time = np.load(('./graph data/' + trace_name + '_time.npz'), allow_pickle = True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
return over_time'''
trace = storage.find_trace(trace_name)
print(trace_name)
print(trace)
if trace == None:
print(trace_name, "could not be found in mongodb")
return False
else:
trace_id = trace.get("_id")
trace_uniques = trace.get("unique")
pretty.utility("Mongo contacted.. retrieving over time array for " +
trace_name)
#print(trace_name + "===========================")
#print(trace_id, " received id for", trace_name)
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#real time
minio_id_time = str(trace_id) + '-time'
#res = minio_get.retrieve( ('./graph data/' + trace_name + '_time.npz'), minio_id_time)
res = minio_get.retrieve(tmp.name, minio_id_time)
#due to how npz files are compressed, the data must be accessed with a key even if there is only 1 thing in the uncompressed npz file
#since there is only 1 thing and no name is specified the first thing will always have the name 'arr_0'
#similarly, the data_time[()] argument is to deal with the 0 dimensional array that is returned.
if res:
#data_time = np.load(('./graph data/' + trace_name + '_time.npz'), allow_pickle = True)
data_time = np.load(tmp.name, allow_pickle=True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
pretty.success("Mongo retrieval successful")
return over_time
else:
sg.sf_singleTraceGen(trace_name, app_id, minio_only=True)
#result = minio_get.retrieve( ('./graph data/' + trace_name + '_time.npz'), minio_id_time)
result = minio_get.retrieve(tmp.name, minio_id_time)
if result:
data_time = np.load(tmp.name, allow_pickle=True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
pretty.success("Mongo retrieval successful")
return over_time
else:
return False
def s_to_database(trace, num_req, num_unique, num_reuse, num_writes,
num_requests, req, histogram, reuse_distance, time_stamp,
minio_only):
pretty.utility("~~~~SENDING " + trace + " TO MONGO DB~~~~~~")
#print("IN S TO DATABASE METHOD")
#requestFrequencyOutput = trace + "_REQUEST_FREQ"
requestsOutput = "./graph data/" + trace + '_rq'
histogramOutput = "./graph data/" + trace + '_histogram'
reuseOutput = "./graph data/" + trace + '_reuse'
timeStampOutput = "./graph data/" + trace + '_time'
#np.savez_compressed(requestFrequencyOutput, histogram)
np.savez_compressed(requestsOutput, req)
np.savez_compressed(histogramOutput, histogram)
np.savez_compressed(reuseOutput, reuse_distance)
np.savez_compressed(timeStampOutput, time_stamp)
# np.savez_compressed(trace, req, regFreq, misses, hitPerc, ios, polOT, missRate)
# ------ db storage -------
if not minio_only:
storage.store_workload(trace, num_req, num_unique, num_reuse,
num_writes, num_requests)
mongo_trace_id = str(storage.find_id(trace))
if mongo_trace_id:
#Request frenquency ID for minio
#rqfId = mongo_trace_id + '-rqf'
#Request ID for minio
rqId = mongo_trace_id + '-rq'
#histogram ID for minio
hId = mongo_trace_id + '-histogram'
#reuse distance ID for minio
reuId = mongo_trace_id + '-reuse'
#time stamp ID for minio
timeId = mongo_trace_id + '-time'
#print("in single trace gen ", rqfId, ' ', rqId)
pretty.utility("trace being inserted to minio " + trace)
#loaded = np.load(outputString +'.npz')
'''insert both sets of output as seperate npz files'''
# it is important to note that these compressed files are written to the computer and then sent to Minio
# currently unsure how to have these compressed objects be variables in memory
toMinio = Minio_Module.Minio_Module()
#toMinio.insert(requestFrequencyOutput +'.npz', rqfId)
toMinio.insert(requestsOutput + '.npz', rqId)
toMinio.insert(histogramOutput + '.npz', hId)
toMinio.insert(reuseOutput + '.npz', reuId)
toMinio.insert(timeStampOutput + '.npz', timeId)
return mongo_trace_id
def store_flag(flag, param_list):
plots = db.plots_tracked
plots_obj = plots.find_one({'flag': flag})
if plots_obj == None:
pretty.utility("plot not found, adding to mongo")
plot_data = {}
plot_data['flag'] = flag
plot_data['params'] = param_list
plots.insert_one(plot_data)
else:
pretty.success(flag, param_list, "already present")
def printMongoResults(trace_name):
pretty.utility("retrieving previous results from MongoDB...")
trace_result = storage.find_trace(trace_name)
if trace_result != False:
print("trace name: ", trace_result['trace name'])
print("requests: ", trace_result['requests'])
print("unique: ", trace_result['unique'])
print("reuses: ", trace_result['reuses'])
print("writes: ", trace_result['writes'])
print("time: ", trace_result['time'])
else:
print(trace_name, "not found")
def mongo_get_alg_run_plot_data(config,
plot,
trace_name,
algorithm,
cache_size,
minio_get,
app_id=None):
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
print('in mongo get plot data')
if app_id != None:
conduit.current_status[app_id] = "checking if data has been run..."
'''if os.path.exists(filename):
pretty.success("file already downloaded to disk")
if app_id != None:
conduit.current_status[app_id] = "File already processed and ready, rendering..."
return filename'''
previously_run = storage.find_trace_run_id(trace_name, algorithm,
cache_size)
print(previously_run, "previously run?")
if previously_run == False:
conduit.current_status[
app_id] = "data has not been run before... processing"
pretty.utility(
"trace run request: " + trace_name + algorithm + str(cache_size) +
" was not found in the database... inserting into database")
if app_id != None:
conduit.current_status[
app_id] = "trace run request: " + trace_name + " " + algorithm + " " + str(
cache_size
) + " was not found in the database... processing and inserting into database"
confirmed = sg.getAlgStats(algorithm, cache_size, config, trace_name,
app_id)
if confirmed:
out_file = request_alg_run_plot_data(trace_name, algorithm,
cache_size, plot, minio_get)
return out_file
else:
pretty.failure("upload to database failed")
return False
else:
conduit.current_status[
app_id] = "data has been run before... retrieving data"
out_file = request_alg_run_plot_data(trace_name, algorithm, cache_size,
plot, minio_get)
return out_file
def retrieve(self, file, id):
pretty.utility("Minio contacted >>>")
try:
self.s3.Bucket('visual-data').download_file(
id, file) #attemps to download
except ClientError:
pretty.failure("File does not exist") #message if file error
return False
except EndpointConnectionError:
pretty.failure(
"Connection was not established") #message if connection error
return False
pretty.success("Minio retrieval successful <<< ")
return True
def mongo_new_paths(config):
pretty.utility("CONTACTING MONGODB... CHECKING FOR NEW PATHS")
newTraces = []
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
mongo_trace = storage.find_trace(trace_name)
if mongo_trace == False:
newTraces.append(trace_name)
else:
pretty.success("Trace: " + mongo_trace["trace name"])
pretty.success("Total number of requests: " +
str(mongo_trace['requests']))
pretty.success("Total number of unique requests: " +
str(mongo_trace['unique']))
pretty.success("Total number of reuses: " +
str(mongo_trace['reuses']))
pretty.success("Total number of writes: " +
str(mongo_trace['writes']))
pretty.success("Total time: " + str(mongo_trace['time']))
pretty.success('\n')
return newTraces
def workload_confirmed(traceName,
request=None,
requestFrequency=None,
reuseDistance=None,
time=None):
pretty.utility("CONFIRMING THAT" + traceName +
"DATA WAS UPLOADED TO MINIO")
traces = db.workload
if request != None:
pretty.success("REQUEST ARRAY CONFIRMED UPLOADED TO MINIO")
traces.update_one(
{'trace name': traceName},
{'$set': {
'requests uploaded to minio': datetime.now()
}})
if requestFrequency != None:
pretty.success("REQUEST FREQUENCY ARRAY CONFIRMED UPLOADED TO MINIO")
traces.update_one(
{'trace name': traceName},
{'$set': {
'request frequency uploaded to minio': datetime.now()
}})
if reuseDistance != None:
pretty.success("REUSE DISTANCE ARRAY CONFIRMED UPLOADED TO MINIO")
traces.update_one(
{'trace name': traceName},
{'$set': {
'reuse distance uploaded to minio': datetime.now()
}})
if time != None:
pretty.success("REAL TIME ARRAY CONFIRMED UPLOADED TO MINIO")
traces.update_one(
{'trace name': traceName},
{'$set': {
'REAL TIME uploaded to minio': datetime.now()
}})
def trace_run_confirmed(traceId,
hit_rate=None,
miss_rate=None,
pollution=None,
dlirs=None,
lecar=None,
arc=None,
cacheus=None):
#print("IN TRACE RUN CONFIRM METHOD")
alg_stats = db.trace_runs
pretty.utility("CONFIRMING THAT ALGORITHM DATA WAS UPLOADED TO MINIO")
print(traceId)
if hit_rate != None:
pretty.success("HIT RATE ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one(
{'_id': ObjectId(traceId)},
{'$set': {
'hit rate array uploaded to minio': datetime.now()
}})
if miss_rate != None:
pretty.success("MISS RATE ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one(
{'_id': ObjectId(traceId)},
{'$set': {
'miss rate array uploaded to minio': datetime.now()
}})
if pollution != None:
pretty.success("POLLUTION ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one(
{'_id': ObjectId(traceId)},
{'$set': {
'pollution array uploaded to minio': datetime.now()
}})
if dlirs != None:
pretty.success("DLIRS STACK ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one(
{'_id': ObjectId(traceId)},
{'$set': {
'dlirs stack array uploaded to minio': datetime.now()
}})
if lecar != None:
pretty.success("LECAR LRU SIZE ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one({'_id': ObjectId(traceId)}, {
'$set': {
'lecar lru size array uploaded to minio': datetime.now()
}
})
if arc != None:
pretty.success("ARC P VALUE ARRAY CONFIRMED UPLOADED TO MINIO")
alg_stats.update_one(
{'_id': ObjectId(traceId)},
{'$set': {
'arc p size array uploaded to minio': datetime.now()
}})
if cacheus != None:
pretty.success(
"CACHEUS LEARNING RATE, LRU VALUE AND Q VALUE CONFIRMED UPLOADED TO MINIO"
)
alg_stats.update_one({'_id': ObjectId(traceId)}, {
'$set': {
'cacheus learning rate array uploaded to minio':
datetime.now()
}
})
alg_stats.update_one({'_id': ObjectId(traceId)}, {
'$set': {
'cacheus lru size array uploaded to minio': datetime.now()
}
})
alg_stats.update_one({'_id': ObjectId(traceId)}, {
'$set': {
'cacheus q value array uploaded to minio': datetime.now()
}
})
def as_minio_confirm(traceId, traceName, algname, cache_size):
fullTraceName = traceName + " " + algname + " " + str(cache_size)
toMinio = Minio_Module.Minio_Module()
pretty.utility("CONFIRMING IF ARRAYS FOR " + fullTraceName +
" WERE UPLOADED SUCCESSFULLY")
hit_rate_id = traceId + '-hit_rate'
miss_rate_id = traceId + '-miss_rate'
pollution_id = traceId + '-pollution'
if algname.lower() == 'dlirs':
dlirs_stack_id = traceId + '-dlirs_stack'
if toMinio.exist(dlirs_stack_id):
storage.trace_run_confirmed(traceId, dlirs=1)
else:
return False
if algname.lower() == 'lecar':
lecar_lru_id = traceId + '-lecar_lru'
if toMinio.exist(lecar_lru_id):
storage.trace_run_confirmed(traceId, lecar=1)
else:
return False
if algname.lower() == 'arc':
arc_p_id = traceId + '-arc_p'
if toMinio.exist(arc_p_id):
storage.trace_run_confirmed(traceId, arc=1)
else:
return False
if algname.lower() == 'cacheus':
cacheus_learning_rate_id = traceId + '-cacheus_learning_rate'
cacheus_lru_id = traceId + '-cacheus_lru'
cacheus_q_id = traceId + '-cacheus_q'
if toMinio.exist(cacheus_learning_rate_id) and toMinio.exist(
cacheus_lru_id) and toMinio.exist(cacheus_q_id):
storage.trace_run_confirmed(traceId, cacheus=1)
else:
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, hit_rate=1)
else:
pretty.failure("Hit rate array for " + fullTraceName +
"to minio failed")
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, miss_rate=1)
else:
pretty.failure("Miss rate array for " + fullTraceName +
"to minio failed")
return False
if toMinio.exist(hit_rate_id):
storage.trace_run_confirmed(traceId, pollution=1)
else:
pretty.failure("Pollution rate array for " + fullTraceName +
"to minio failed")
return False
pretty.success(
"==============================" + fullTraceName +
" succesfully uploaded ==========================================")
return True
def as_to_Database(trace,
algname,
cache_size,
ios,
hits,
hit_rate,
misses,
pollution,
f_eviction_rate,
f_eviction_total,
hit_ot,
miss_ot,
pol_ot,
minio_only,
dlirs_HIR_s=None,
lecar_lru=None,
arc_p=None,
cacheus_learning_rate=None,
cacheus_lru=None,
cacheus_q=None):
#print("*************************************AS TO DATABASE**********************************")
currentTrace = trace + "_" + algname + "_" + str(cache_size)
hitrate_over_time_output = "./graph data/" + currentTrace + '-hit_rate'
missrate_over_time_output = "./graph data/" + currentTrace + '-miss_rate'
pollutionrate_over_time_output = "./graph data/" + currentTrace + '-pollution'
toMinio = Minio_Module.Minio_Module()
np.savez_compressed(hitrate_over_time_output, hit_ot)
np.savez_compressed(missrate_over_time_output, miss_ot)
np.savez_compressed(pollutionrate_over_time_output, pol_ot)
#find a collection that has current trace name, algorithm and cache size and return its ID
#use this ID for minio uploads
###new algorithm outputs###
#these are contingent on what algorithm is being run as each algorithm has different data that is important to track
'''lecar_lru_output = currentTrace + "_LECAR_LRU_OVER_TIME"
arc_p_output = currentTrace + "_ARC_P_OVER_TIME"
cacheus_learning_rate_output = currentTrace + "_CACHEUS_LEARNING_RATE_OVER_TIME"
cacheus_lru_output = currentTrace + "_CACHEUS_LRU_OVER_TIME"
cacheus_q_output = currentTrace = "_CACHEUS_Q_OVER_TIME"
np.savez_compressed(hitrate_over_time_output, hit_ot)
np.savez_compressed(missrate_over_time_output, miss_ot)
np.savez_compressed(pollutionrate_over_time_output, pol_ot)
###new algorithm outputs###
np.savez_compressed(dlirs_stack_output, dlirs_HIR_s)
np.savez_compressed(lecar_lru_output, lecar_lru)
np.savez_compressed(arc_p_output, arc_p)
np.savez_compressed(cacheus_learning_rate_output, cacheus_learning_rate)
np.savez_compressed(cacheus_lru_output, cacheus_lru)
np.savez_compressed(cacheus_q_output, cacheus_q)'''
pretty.utility("~~~~SENDING " + currentTrace + " TO MONGO DB~~~~~~")
# ------ mongodb storage ------
storage.new_store_alg(ios, hits, hit_rate, misses, pollution,
f_eviction_rate, f_eviction_total, algname,
cache_size, trace)
# ---------------------
mongo_trace_runs_id = storage.find_trace_run_id(trace, algname, cache_size)
#hit rate ID for minio
hit_rate_id = mongo_trace_runs_id + '-hit_rate'
#miss rate ID for minio
miss_rate_id = mongo_trace_runs_id + '-miss_rate'
#pollution rate ID for minio
pol_rate_id = mongo_trace_runs_id + '-pollution'
toMinio.insert(hitrate_over_time_output + '.npz', hit_rate_id)
toMinio.insert(missrate_over_time_output + '.npz', miss_rate_id)
toMinio.insert(pollutionrate_over_time_output + '.npz', pol_rate_id)
#######New alg plot data########
#dlirs HIR stack array ID for minio
'''dlirs_id = mongo_trace_runs_id + '-dlirs_stack'
#lecar LRU array for minio
lecar_lru_id = mongo_trace_runs_id + '-lecar_lru'
#arc p value array ID for minio
arc_p_id = mongo_trace_runs_id + '-arc_p'
#cacheus learning rate array ID for minio
cacheus_learning_rate_id = mongo_trace_runs_id + '-cacheus_learning_rate'
#cacheus LRU array ID for minio
cacheus_lru_id = mongo_trace_runs_id + '-cacheus_lru'
#cacheus q array ID for minio
cacheus_q_id = mongo_trace_runs_id + '-cacheus_q'''
#print("in getAlgStats ", hit_rate_id, ' ', miss_rate_id, ' ', pol_rate_id)
pretty.utility("trace arrays being inserted to mongo for " + currentTrace)
#loaded = np.load(outputString +'.npz')
'''insert both sets of output as seperate npz files'''
# it is important to note that these compressed files are written to the computer and then sent to Minio
# currently unsure how to have these compressed objects be variables in memory
if dlirs_HIR_s != None:
dlirs_stack_output = "./graph data/" + currentTrace + '-dlirs_stack'
np.savez_compressed(dlirs_stack_output, dlirs_HIR_s)
dlirs_id = mongo_trace_runs_id + '-dlirs_stack'
toMinio.insert(dlirs_stack_output + '.npz', dlirs_id)
if lecar_lru != None:
lecar_lru_output = "./graph data/" + currentTrace + '-lecar_lru'
np.savez_compressed(lecar_lru_output, lecar_lru)
lecar_lru_id = mongo_trace_runs_id + '-lecar_lru'
toMinio.insert(lecar_lru_output + '.npz', lecar_lru_id)
if arc_p != None:
arc_p_output = "./graph data/" + currentTrace + "_ARC_P_OVER_TIME"
np.savez_compressed(arc_p_output, arc_p)
arc_p_id = mongo_trace_runs_id + '-arc_p'
toMinio.insert(arc_p_output + '.npz', arc_p_id)
if (cacheus_learning_rate != None) and (cacheus_lru != None) and (cacheus_q
!= None):
cacheus_learning_rate_output = "./graph data/" + currentTrace + '-cacheus_learning_rate'
cacheus_lru_output = "./graph data/" + currentTrace + '-cacheus_lru'
cacheus_q_output = "./graph data/" + currentTrace + '-cacheus_q'
np.savez_compressed(cacheus_learning_rate_output,
cacheus_learning_rate)
np.savez_compressed(cacheus_lru_output, cacheus_lru)
np.savez_compressed(cacheus_q_output, cacheus_q)
cacheus_learning_rate_id = mongo_trace_runs_id + '-cacheus_learning_rate'
cacheus_lru_id = mongo_trace_runs_id + '-cacheus_lru'
cacheus_q_id = mongo_trace_runs_id + '-cacheus_q'
toMinio.insert(cacheus_learning_rate_output + '.npz',
cacheus_learning_rate_id)
toMinio.insert(cacheus_lru_output + '.npz', cacheus_lru_id)
toMinio.insert(cacheus_q_output + '.npz', cacheus_q_id)
#####NEW ALGORITHM ARRAYS
'''toMinio.insert(dlirs_stack_output + '.npz', dlirs_id)
toMinio.insert(lecar_lru_output + '.npz', lecar_lru_id)
toMinio.insert(arc_p_output + '.npz', arc_p_id)
toMinio.insert(cacheus_learning_rate_output + '.npz', cacheus_learning_rate_id)
toMinio.insert(cacheus_lru_output + '.npz', cacheus_lru_id)
toMinio.insert(cacheus_q_output + '.npz', cacheus_q_id)'''
return mongo_trace_runs_id
def mongo_get_trace_run_plot_data(trace_name, plot, minio_get, app_id=None):
'''filename = "./graph data/"+str(trace_name) + str(plot) +'.npz'
if os.path.exists(filename):
pretty.success("file already downloaded to disk")
return filename'''
print(plot, "********************************")
print(trace_name)
trace = storage.find_trace(trace_name)
if trace == False:
pretty.utility("plot data for " + trace_name + plot +
" was not found, inserting into database...")
if app_id != None:
conduit.curret_status[
app_id] = "plot data for " + trace_name + " " + plot + " was not found, inserting into database..."
confirmed = sg.sf_singleTraceGen(trace_name, app_id)
trace = storage.find_trace(trace_name)
trace_id = trace.get("_id")
minio_id = str(trace_id) + plot
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#success = minio_get.retrieve(header, minio_id)
#success = minio_get.retrieve(tmp.name, minio_id)
if confirmed:
success = minio_get.retrieve(tmp.name, minio_id)
if success:
return tmp # tmp.name
else:
return False
else:
pretty.failure("error uploading trace data to database")
return False
else:
trace_id = trace.get("_id")
trace_ios = trace.get("requests")
#print("Mongo contacted")
#print(trace_id, " received id for", trace_name, plot)
minio_id = str(trace_id) + plot
#print("mongo_get_trace_run_plot_data", minio_id)
pretty.utility("retrieving plot data from minio...")
tmp = tempfile.NamedTemporaryFile(mode='w+b', suffix='.npz')
success = minio_get.retrieve(tmp.name, minio_id)
if success:
pretty.success("plot data for " + trace_name + plot +
" successfully downloaded")
#print(filename)
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
pretty.failure(
"data not found in minio store... processing trace data...")
confirmed = sg.sf_singleTraceGen(trace_name, minio_only=True)
success = minio_get.retrieve(tmp.name, minio_id)
if success:
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
print("error uploading data to minio")
return False
os.mkdir(config['output_path'])
print("config acquired")
storage.insert_config(config, name)'''
else:
pretty.failure("Error: config file {} does not exist".format(
sys.argv[1]))
sys.exit()
args, unknown = get_parser().parse_known_args()
# generate trace statistics. this is set to true by default because the number of unique accesses is needed to determine cache size
# for running -a flag later
if args.test:
pretty.utility("you have called test flag")
#serv.check_config(config)
#with open("reranked_5%_clean_final_results_all.csv", 'r') as file:
#compress_config("reranked_5%_clean_final_results_all.csv")
#b = np.load('test.npz')
#print(b['arr_0'])
#df = pd.read_csv('./reranked.csv')
#compression_opts = dict(method='zip', archive_name='./reranked.csv')
#compression_opts = {'method': 'zip', 'archive_name': './reranked.csv'}
#df.to_csv('out.zip', index=False, compression='zip')
#csv_plot.plot_rank_heat_map(config)
def delete_plots_tracked(param):
pretty.utility("deleting plots with param: ", param)
plots = db.plots_tracked
plots.delete_many({'flag': param})
def visualize_head_to_head(config, flag, y_scale=None):
pretty.utility(
"generating comparison data for all pairs of algorithms and cache sizes"
)
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
current_plot = plots[flag]
#if current_plot[0] == 'algorithm' and current_plot[6] == False:
all_pairs = gen_all_pairs(config['algorithms'])
all_plots = {}
# generate all output arrays here and add them to a dictionary to be referenced for the graphing of all pairs to prevent repeatedly downloading and extracting
# the same data
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for algorithms in config['algorithms']:
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name, algorithms,
cache_size, minio_get)
if out_file:
out_file = np.load(out_file, allow_pickle=True)
out_file = out_file['arr_0']
all_plots[str(trace_name) + str(algorithms) +
str(cache_size)] = out_file
else:
pretty.failure("error downloading " + trace_name +
current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage " +
str(percent_cache))
compare = []
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
over_time = serv.get_over_time_list(trace_name, minio_get)
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(cache_size *
serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
for pairs in all_pairs:
compare.append(
all_plots[str(trace_name) + str(pairs[0]) +
str(cache_size)])
compare.append(
all_plots[str(trace_name) + str(pairs[1]) +
str(cache_size)])
over_time = serv.get_over_time_list(
trace_name, minio_get)
basename = os.path.basename(trace_name)
name = basename + " " + " cache size " + str(
cache_size) + " " + str(
pairs[0]) + " compared with " + str(pairs[1])
print(name)
graph.vis_data_multi(current_plot[1],
current_plot[3][1],
current_plot[4],
name,
compare,
pairs,
config['output_path'],
x_data=over_time,
scale=y_scale,
fill=True)
compare = []
def visualize_data(flag, config, y_scale=None):
minio_get = Minio_Module.Minio_Module()
graph = sv.StatVis()
current_plot = plots[flag]
#plots that use trace data have a different, smaller loop than the plots using data from algorithms
if current_plot[0] == 'trace':
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
print(trace_name + "=======================================")
#request data from mongo
out_file = serv.mongo_get_trace_run_plot_data(
trace_name, current_plot[2], minio_get)
if out_file:
#load that data
out_file = np.load(out_file,
allow_pickle=True) #uncompress
if current_plot[1] == 'histogram':
#---
if flag == '-g':
x_axis, frequency, uniques = process_histogram(
flag, out_file)
graph.vis_data(
current_plot[1],
current_plot[3],
current_plot[4],
str(trace_name) + " " + current_plot[5] +
" Number of Uniques: " + str(uniques),
frequency,
config['output_path'],
x_data=x_axis)
else:
x_axis, frequency = process_histogram(
flag, out_file)
#print(frequency)
'''graph.vis_data(current_plot[1], current_plot[3], current_plot[4],
str(trace_name) + " " + current_plot[5], frequency, x_data = x_axis)'''
#this is hard coded as line because it will not render as a bar graph, it lags and never finishes
graph.vis_data('line',
current_plot[3],
current_plot[4],
str(trace_name) + " " +
current_plot[5],
frequency,
config['output_path'],
x_data=x_axis)
#---
else:
#---
over_time = serv.get_over_time_list(
trace_name, minio_get)
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
str(trace_name) + " " + current_plot[5],
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#---
else:
pretty.failure("error downloading" + trace_name +
'access pattern')
#if it is an algorithm data plot and that data is not algorithm specific, run a loop for all algorithms
elif current_plot[0] == 'algorithm' and current_plot[6] == False:
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for algorithms in config['algorithms']:
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name,
algorithms, cache_size, minio_get)
plot_title = current_plot[5][1]
#print(plot_title)
if out_file:
full_title = str(trace_name) + " " + str(
algorithms) + "_" + str(
cache_size) + " " + current_plot[
4] + " " + plot_title
out_file = np.load(out_file, allow_pickle=True)
over_time = serv.get_over_time_list(
trace_name, minio_get)
#print(trace_name, algorithms, cache_size)
#print("=====================")
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
full_title,
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#print("++++++++++++++++++++")
else:
pretty.failure("error downloading " +
trace_name + current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage "
+ str(percent_cache))
#else, it is an algorithm specific plot. the only difference here is that there is not looping through the list of algorithms
elif current_plot[0] == 'algorithm':
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
for cache_size in config['cache_sizes']:
percent_cache = cache_size
uniques = serv.mongo_get_unique(trace_name)
if uniques:
cache_size = math.floor(
cache_size * serv.mongo_get_unique(trace_name))
else:
return False
if cache_size >= 10:
#all of the algorithm specific plots have the name of the algorithm before the '_'
#so that is why algorithm_used[0] is called, we must also remove the '-' that is in the zero index
algorithm_used = current_plot[2].split('_')
algorithm_used = algorithm_used[0]
algorithm_used = algorithm_used[1:len(algorithm_used)]
#pretty.utility(algorithm_used)
out_file = serv.mongo_get_alg_run_plot_data(
config, current_plot[2], trace_name,
algorithm_used, cache_size, minio_get)
plot_title = current_plot[5][1]
#pretty.utility(plot_title)
if out_file:
full_title = str(trace_name) + " " + str(
algorithm_used) + "_" + str(
cache_size
) + " " + current_plot[4] + " " + plot_title
out_file = np.load(out_file, allow_pickle=True)
over_time = serv.get_over_time_list(
trace_name, minio_get)
pretty.utility(trace_name + algorithm_used +
str(cache_size))
graph.vis_data(current_plot[1],
current_plot[3][1],
current_plot[4],
full_title,
out_file['arr_0'],
config['output_path'],
x_data=over_time)
#---
#
else:
pretty.failure("error downloading " + trace_name +
current_plot[2])
else:
pretty.utility(
trace_name +
" not big enough for current cache size percentage "
+ str(percent_cache))