def get_over_time_list(trace_name, minio_get, app_id=None):
print("in get over time list")
'''filename = './graph data/' + trace_name + '_time.npz'
if os.path.exists(filename):
pretty.success("file already downloaded to disk")
data_time = np.load(('./graph data/' + trace_name + '_time.npz'), allow_pickle = True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
return over_time'''
trace = storage.find_trace(trace_name)
print(trace_name)
print(trace)
if trace == None:
print(trace_name, "could not be found in mongodb")
return False
else:
trace_id = trace.get("_id")
trace_uniques = trace.get("unique")
pretty.utility("Mongo contacted.. retrieving over time array for " +
trace_name)
#print(trace_name + "===========================")
#print(trace_id, " received id for", trace_name)
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#real time
minio_id_time = str(trace_id) + '-time'
#res = minio_get.retrieve( ('./graph data/' + trace_name + '_time.npz'), minio_id_time)
res = minio_get.retrieve(tmp.name, minio_id_time)
#due to how npz files are compressed, the data must be accessed with a key even if there is only 1 thing in the uncompressed npz file
#since there is only 1 thing and no name is specified the first thing will always have the name 'arr_0'
#similarly, the data_time[()] argument is to deal with the 0 dimensional array that is returned.
if res:
#data_time = np.load(('./graph data/' + trace_name + '_time.npz'), allow_pickle = True)
data_time = np.load(tmp.name, allow_pickle=True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
pretty.success("Mongo retrieval successful")
return over_time
else:
sg.sf_singleTraceGen(trace_name, app_id, minio_only=True)
#result = minio_get.retrieve( ('./graph data/' + trace_name + '_time.npz'), minio_id_time)
result = minio_get.retrieve(tmp.name, minio_id_time)
if result:
data_time = np.load(tmp.name, allow_pickle=True)
data_time = data_time['arr_0']
data_time = data_time[()]
over_time = normalize_real_time(data_time)
pretty.success("Mongo retrieval successful")
return over_time
else:
return False
def mongo_get_unique(trace_name):
currentTrace = storage.find_trace(trace_name)
#print(currentTrace)
if currentTrace != False:
number_of_uniques = currentTrace.get("unique")
print("number of uniques gotten", number_of_uniques)
return number_of_uniques
else:
return False
'''else:
def printMongoResults(trace_name):
pretty.utility("retrieving previous results from MongoDB...")
trace_result = storage.find_trace(trace_name)
if trace_result != False:
print("trace name: ", trace_result['trace name'])
print("requests: ", trace_result['requests'])
print("unique: ", trace_result['unique'])
print("reuses: ", trace_result['reuses'])
print("writes: ", trace_result['writes'])
print("time: ", trace_result['time'])
else:
print(trace_name, "not found")
def mongo_new_paths(config):
pretty.utility("CONTACTING MONGODB... CHECKING FOR NEW PATHS")
newTraces = []
for traces in config['traces']:
for trace_name in sg.generateTraceNames(traces):
mongo_trace = storage.find_trace(trace_name)
if mongo_trace == False:
newTraces.append(trace_name)
else:
pretty.success("Trace: " + mongo_trace["trace name"])
pretty.success("Total number of requests: " +
str(mongo_trace['requests']))
pretty.success("Total number of unique requests: " +
str(mongo_trace['unique']))
pretty.success("Total number of reuses: " +
str(mongo_trace['reuses']))
pretty.success("Total number of writes: " +
str(mongo_trace['writes']))
pretty.success("Total time: " + str(mongo_trace['time']))
pretty.success('\n')
return newTraces
def getAlgStats(algname,
cache_size,
config,
trace_name,
app_id=None,
minio_only=False):
print("in get alg stats")
print("==============================================")
#alg = get_algorithm(algname)(cache_size, **self.alg_args)
alg_args = parseConfig(config, algname)
avg_pollution = 0
misses = 0
ios = 1
#general alg data
lba_map = {}
hitPerc = [] #percentage of hits over time
missRate = [] #miss rate over time
timestamps = []
count = 0
evict = 0
polOT = [] #pollution over time
#special alg data: depending on which algorithm is called, other information is collected as the algorithm is run
dlirs_HIR_s = []
lecar_lru = []
arc_p = []
cacheus_learning_rate = []
cacheus_lru = []
cacheus_q = []
# ------------ plot for hit rate over learning rate
lecarLR = -1 # for any lecar w. fixed learning rate
# evictions data
all_evictions = 0
evRate = [] # total number of evictions @ point in time
evictions = [] # number of evictions per requested LBA -SH
#trace_type = identify_trace(trace_name)
#trace_reader = get_trace_reader(trace_type)
if app_id != None:
conduit.current_status[app_id] = "getting data from sftp server..."
if sftp_data.get(trace_name) == None:
print("getting data from sftp server...")
filename, trace_data = sftp_r.direct_query(trace_name)
sftp_data[trace_name] = [filename, trace_data]
print("finished getting data from sftp server...")
print(filename)
else:
filename = sftp_data[trace_name][0]
trace_data = sftp_data[trace_name][1]
pretty.success("*\n*\n*\n*\n*\n*\n*\nREUSE")
print("getting data from sftp server...")
filename, trace_data = sftp_r.direct_query(trace_name)
print("finished getting data from sftp server...")
# this is new ----------
reader = sf_filereader.identify_trace(filename, trace_data)
print(filename, " ***************************** ********************** ")
print(algname)
alg = get_algorithm(algname)(cache_size, **alg_args)
print(alg)
timestamps.append(0) #time
init_timestamp = time.perf_counter()
trace = storage.find_trace(trace_name)
trace_requests = trace.get("requests")
if app_id != None:
conduit.current_status[
app_id] = "processing " + trace_name + algname + str(
cache_size) + " this can take over 10 minutes..."
ios, misses, hitPerc, missRate, polOT, dlirs_HIR_s, arc_p, cacheus_learning_rate, cacheus_lru, cacheus_q, lecar_lru = readerLoop(
reader, alg, hitPerc, missRate, polOT, algname, lba_map,
trace_requests)
lecarLR = alg.learning_rate if algname.lower() == 'lecar' else -1
avg_pollution = np.mean(alg.pollution.Y)
ios = reader.requests
hits = ios - misses
hitRate = round(100 * (hits / ios), 2)
finalEvictionR = round(100 * (all_evictions / ios)) #final eviction rate
finalEvictionT = len(evictions)
print(
"\nResults: {:<10} size={:<8} hits={}, ios = {}, misses={}, hitrate={:4}% avg_pollution={:4}% {}"
.format(algname, cache_size, hits, ios, misses, hitRate,
round(avg_pollution, 2), trace_name, *alg_args.items()))
print(dlirs_HIR_s, lecar_lru, arc_p, cacheus_learning_rate, cacheus_lru,
cacheus_q)
mongo_trace_runs_id = serv.as_to_Database(
trace_name, algname, cache_size, ios, hits, hitRate, misses,
round(avg_pollution, 2), finalEvictionR, finalEvictionT, hitPerc,
missRate, polOT, minio_only, dlirs_HIR_s, lecar_lru, arc_p,
cacheus_learning_rate, cacheus_lru, cacheus_q)
if app_id != None:
conduit.current_status[
app_id] = "processing " + trace_name + algname + str(
cache_size) + " complete"
print("---------------------MONGO TRACE RUNS ID BEING SENT TO CONFIRM",
mongo_trace_runs_id)
#if not minio_only:
confirmed = serv.as_minio_confirm(mongo_trace_runs_id, trace_name, algname,
cache_size)
if confirmed:
return mongo_trace_runs_id
else:
return -1
def testAlgStats(algname, cache_size, config, trace_name):
print("in get alg stats")
#alg = get_algorithm(algname)(cache_size, **self.alg_args)
alg_args = parseConfig(config, algname)
avg_pollution = 0
misses = 0
ios = 1
#general alg data
lba_map = {}
hitPerc = [] #percentage of hits over time
missRate = [] #miss rate over time
timestamps = []
count = 0
evict = 0
polOT = [] #pollution over time
#special alg data: depending on which algorithm is called, other information is collected as the algorithm is run
dlirs_HIR_s = []
lecar_lru = []
arc_p = []
cacheus_learning_rate = []
cacheus_lru = []
cacheus_q = []
# ------------ plot for hit rate over learning rate
lecarLR = -1 # for any lecar w. fixed learning rate
# evictions data
all_evictions = 0
evRate = [] # total number of evictions @ point in time
evictions = [] # number of evictions per requested LBA -SH
#trace_type = identify_trace(trace_name)
#trace_reader = get_trace_reader(trace_type)
# this is new ----------
reader = fr.identify_trace(trace_name)
alg = get_algorithm(algname)(cache_size, **alg_args)
timestamps.append(0) #time
init_timestamp = time.perf_counter()
trace = storage.find_trace(trace_name)
trace_requests = trace.get("requests")
ios, misses, hitPerc, missRate, polOT, dlirs_HIR_s, arc_p, cacheus_learning_rate, cacheus_lru, cacheus_q, lecar_lru = readerLoop(
reader, alg, hitPerc, missRate, polOT, algname, lba_map,
trace_requests)
lecarLR = alg.learning_rate if algname.lower() == 'lecar' else -1
avg_pollution = np.mean(alg.pollution.Y)
ios = reader.requests
hits = ios - misses
hitRate = round(100 * (hits / ios), 2)
finalEvictionR = round(100 * (all_evictions / ios)) #final eviction rate
finalEvictionT = len(evictions)
print(
"\nResults: {:<10} size={:<8} hits={}, ios = {}, misses={}, hitrate={:4}% avg_pollution={:4}% {}"
.format(algname, cache_size, hits, ios, misses, hitRate,
round(avg_pollution, 2), trace_name, *alg_args.items()))
def mongo_get_trace_run_plot_data(trace_name, plot, minio_get, app_id=None):
'''filename = "./graph data/"+str(trace_name) + str(plot) +'.npz'
if os.path.exists(filename):
pretty.success("file already downloaded to disk")
return filename'''
print(plot, "********************************")
print(trace_name)
trace = storage.find_trace(trace_name)
if trace == False:
pretty.utility("plot data for " + trace_name + plot +
" was not found, inserting into database...")
if app_id != None:
conduit.curret_status[
app_id] = "plot data for " + trace_name + " " + plot + " was not found, inserting into database..."
confirmed = sg.sf_singleTraceGen(trace_name, app_id)
trace = storage.find_trace(trace_name)
trace_id = trace.get("_id")
minio_id = str(trace_id) + plot
tmp = tempfile.NamedTemporaryFile(suffix='.npz')
#success = minio_get.retrieve(header, minio_id)
#success = minio_get.retrieve(tmp.name, minio_id)
if confirmed:
success = minio_get.retrieve(tmp.name, minio_id)
if success:
return tmp # tmp.name
else:
return False
else:
pretty.failure("error uploading trace data to database")
return False
else:
trace_id = trace.get("_id")
trace_ios = trace.get("requests")
#print("Mongo contacted")
#print(trace_id, " received id for", trace_name, plot)
minio_id = str(trace_id) + plot
#print("mongo_get_trace_run_plot_data", minio_id)
pretty.utility("retrieving plot data from minio...")
tmp = tempfile.NamedTemporaryFile(mode='w+b', suffix='.npz')
success = minio_get.retrieve(tmp.name, minio_id)
if success:
pretty.success("plot data for " + trace_name + plot +
" successfully downloaded")
#print(filename)
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
pretty.failure(
"data not found in minio store... processing trace data...")
confirmed = sg.sf_singleTraceGen(trace_name, minio_only=True)
success = minio_get.retrieve(tmp.name, minio_id)
if success:
file = np.load(tmp.name, allow_pickle=True)
return file # tmp.name
else:
print("error uploading data to minio")
return False