def load_file_time(traceID, typeID, sizerate=0.1, p=1):
readReq = 0
# print(traceID)
historyHit = 0
historyReq = 0
size = int(sizerate*uclnDict[traceID])
# print(sizerate*uclnDict[traceID], size)
ssd = PLRU(size, p)
fin = open(getPath(traceID, typeID), 'r')
lines = fin.readlines()
print("load file finished")
logFile = open(logFilename, "a")
print(traceID, p, sizerate, size, sep=",", end=",", file=logFile)
for line in lines:
items = line.split(' ')
reqtype = int(items[0])
blockid = int(items[2])
if reqtype == 1:
ssd.delete_cache(blockid)
else:
if readReq % PERIODLEN == 0 and readReq!=0:
localHitRatio = 1.0*(ssd.hit-historyHit)/(readReq-historyReq)
historyHit = ssd.hit
historyReq = readReq
print(localHitRatio, sep=",", end=",", file=logFile)
readReq += 1
ssd.is_hit(blockid)
ssd.update_cache(blockid)
fin.close()
print("size", size, p)
print("total hit rate", 1.0*ssd.hit/readReq, ssd.update)
print(1.0*ssd.hit/readReq, ssd.update, 1.0*ssd.hit/ssd.update, sep=',', file=logFile)
logFile.close()
def __init__(self, trace, bsizeRatio, csizeRatio, ucln, p, policy):
self.trace = trace
self.bsizeRatio = bsizeRatio
self.cacheSizeRatio = csizeRatio
# self.p = p
self.ucln = ucln
# policy = nrsamples, hit throt, +-s, +-p
self.policy = policy
(bp, cp) = p
self.baseline = PLRU(int(bsizeRatio*ucln), bp)
# print("base", int(bsizeRatio*ucln), bp)
self.baseline2 = PLRU(int(csizeRatio*ucln), bp)
# print("base2", int(csizeRatio*ucln), bp)
self.cache = PLRU(int(csizeRatio*ucln), cp)
# print("cache", int(csizeRatio*ucln), cp)
self.req = 0
self.lastBaseUpdate = 0
self.lastCacheUpdate = 0
self.init_samples()
def init_samples(self):
self.samples = []
self.lastBaseUpdate += self.baseline.get_update()
self.lastCacheUpdate += self.cache.get_update()
self.cache.update = 0
self.baseline.update = 0
for i in range(-self.policy["nrsamples"], self.policy["nrsamples"]):
for j in range(-self.policy["nrsamples"], self.policy["nrsamples"]):
if i == 0 and j == 0:
continue
# print("i=", i, ",j=", j)
sizeRatio = self.cacheSizeRatio + i * self.policy["deltas"]
p = self.cache.p + j * self.policy["deltap"]
# print("sr=", sizeRatio, ",p=", p)
if is_valid_sp(sizeRatio, p):
size = int(sizeRatio*self.ucln)
# print("valid", size, p)
s = PLRU(int(sizeRatio*self.ucln), p)
s.copy(self.cache, size, p)
self.samples.append(s)
def do_single_trace(traceName, parameters):
#load trace
fin = open(traceName, 'r')
reqs = load_trace(fin)
fin.close()
#initial
baseline = PLRU(parameters.size, parameters.p)
ssd = PLRU(parameters.size, parameters.p)
samples = SampleSSDs(parameters, ssd)
log = Log(parameters)
for (rw, req) in reqs:
random.seed()
roll = random.random()
#?what about write request?
update_cache(rw, req, baseline, roll)
(hit, update) = update_cache(rw, req, ssd, roll)
samples.update_cache(roll, req, rw, hit, update)
if log.tick():
modify_config(samples, ssd, baseline, parameters, log)
log.print(baseline, ssd)
def mtc_test_size_p(path, traceID, totalTimeLength, timeStart, sizerate, p):
lines = []
uclnDict = {}
req = 0
readReq = 0
load_lines(path, traceID, totalTimeLength, timeStart, lines, uclnDict)
size = int(sizerate * len(uclnDict))
# 稀疏周期，size过小
if size <= 100:
return
print("size=", size)
ssd = PLRU(size, p)
for line in lines:
(time, rw, blockid) = parse_line(line, "gen")
req += 1
hit = ssd.is_hit(blockid)
if rw == 0:
readReq += 1
if rw == 1 and hit:
ssd.add_update()
ssd.update_cache(blockid)
print(traceID, "size", size, p)
print("total hit rate", 1.0 * ssd.hit / req, "update", ssd.update)
global g
if ssd.update > 1.0 * size * g * totalTimeLength:
cost = 1.0 * ssd.update / g / totalTimeLength
else:
cost = size
print(ssd.update, size, g * totalTimeLength,
1.0 * size * g * totalTimeLength, 1.0 * ssd.update / size, cost)
logFile = open(logFilename, "a")
print(traceID,
timeStart / totalTimeLength,
totalTimeLength / danwei,
sizerate,
size,
p,
1.0 * ssd.hit / req,
ssd.update,
req,
round(1.0 * readReq / req, 3),
cost,
sep=',',
file=logFile)
logFile.close()
def handle_csv_time(fileid, filename, order, time, pattern, ssd, fileIdx):
print("enter handle_csv_time", fileid, order, time, fileIdx)
flag = False
readcount = 1
writecount = 1
readsize = 1
writesize = 1
# outname = filename +'.req'
count = fileIdx
totalDict = {}
readDict = {}
writeDict = {}
lba = [[sys.maxsize, 0], [sys.maxsize, 0], [sys.maxsize, 0]]
infile = open(filename, 'r')
# outfile = open(outname, 'w')
nrreq = 0
lines = infile.readlines()
if pattern == "warm":
if fileIdx == 0:
timeStart = -1
timeEnd = 10**25
else:
line = lines[fileIdx]
timeBase = get_time(line)
timeStart = timeBase + order * time
timeEnd = timeStart + time
if ssd == None:
ssd = PLRU(int(0.1 * uclnDict[fileid]), 1)
oldhit = -1
oldupdate = -1
else:
oldhit = ssd.hit
oldupdate = ssd.update
else:
if order < 0 and fileIdx == 0:
line = lines[-1]
else:
line = lines[fileIdx]
line = line.strip().split(',')
timeBase = int(line[0])
if fileIdx > 0:
timeStart = timeBase
else:
timeStart = timeBase + order * time
timeEnd = timeStart + time
print("time", timeBase / danwei, timeStart / danwei, timeEnd / danwei)
for line in lines[fileIdx:]:
count += 1
line = line.strip().split(',')
timestamp = int(line[0])
# print(timestamp, timestamp < timeStart, timestamp > timeEnd)
if timestamp < timeStart:
continue
elif timestamp > timeEnd:
break
# first coming
if not flag:
lineStart = count
flag = True
block_id = int((float(line[4])) / block_size)
block_end = int((float(line[4]) + float(line[5]) - 1) / block_size)
# if count % 100000 == 0:
# print(count)
if line[3] == 'Write':
rw = 1
writecount += 1
writesize += block_end - block_id + 1
elif line[3] == 'Read':
rw = 0
readcount += 1
readsize += block_end - block_id + 1
else:
rw = 2
if lba[2][0] > block_id:
lba[2][0] = block_id
if lba[2][1] < block_end:
lba[2][1] = block_end
if lba[rw][0] > block_id:
lba[rw][0] = block_id
if lba[rw][1] < block_end:
lba[rw][1] = block_end
for i in range(block_id, block_end + 1):
# print('{0} {1} {2}'.format(rw,line[2],i),file=outfile)
# print>>outfile, '{0} {1} {2}'.format(rw,line[2],i)
totalDict[i] = True
if rw == 0:
readDict[i] = True
elif rw == 1:
writeDict[i] = True
if pattern == "warm":
nrreq += 1
hit = ssd.is_hit(i)
if line[3] == 'Write' and hit:
ssd.add_update()
ssd.update_cache(i)
if pattern == "warm" and oldhit < 0 and ssd.is_full():
timeBase = timestamp
timeStart = timeBase + order * time
timeEnd = timeStart + time
nrreq = 0
ssd.oldhit = ssd.hit
ssd.oldupdate = ssd.update
readsize = 0
writesize = 0
flag = False
# print("read write", readcount, writecount, readcount/writecount, readsize, writesize, readsize/writesize, sep=',')
# print("ucln", len(totalDict), len(readDict), len(writeDict),
# lba[2][1]-lba[2][0]+1, lba[0][1]-lba[0][0]+1, lba[1][1]-lba[1][0]+1, sep=',')
lineStop = count
print(fileid,
pattern,
order,
time / danwei,
readsize,
writesize,
round(1.0 * readsize / (readsize + writesize), 2),
sep=',',
end=',',
file=logFile)
infile.close()
# outfile.close()
if pattern != "warm":
nrreq = 0
assert len(totalDict) > 0
ssd = PLRU(int(0.1 * len(totalDict)), 1)
infile = open(filename, 'r')
# outfile = open(outname, 'w')
lines = infile.readlines()
for line in lines[lineStart - 1:lineStop]:
line = line.strip().split(',')
timestamp = int(line[0])
# if timestamp < timeStart:
# continue
# elif timestamp > timeEnd:
# break
block_id = int((float(line[4])) / block_size)
block_end = int((float(line[4]) + float(line[5]) - 1) / block_size)
if count % 100000 == 0:
print(count)
for req in range(block_id, block_end + 1):
nrreq += 1
hit = ssd.is_hit(req)
if line[3] == 'Write' and hit:
ssd.add_update()
ssd.update_cache(req)
hit = ssd.hit
update = ssd.update
else:
hit = ssd.hit - oldhit
update = ssd.update - oldupdate
print(fileid, hit, nrreq, 1.0 * hit / nrreq, update)
print(nrreq,
ssd.size,
hit,
1.0 * hit / nrreq,
update,
sep=',',
file=logFile)
logFile.flush()
return (ssd, count)
class Cache(object):
"""docstring for Cache"""
def __init__(self, trace, bsizeRatio, csizeRatio, ucln, p, policy):
self.trace = trace
self.bsizeRatio = bsizeRatio
self.cacheSizeRatio = csizeRatio
# self.p = p
self.ucln = ucln
# policy = nrsamples, hit throt, +-s, +-p
self.policy = policy
(bp, cp) = p
self.baseline = PLRU(int(bsizeRatio*ucln), bp)
# print("base", int(bsizeRatio*ucln), bp)
self.baseline2 = PLRU(int(csizeRatio*ucln), bp)
# print("base2", int(csizeRatio*ucln), bp)
self.cache = PLRU(int(csizeRatio*ucln), cp)
# print("cache", int(csizeRatio*ucln), cp)
self.req = 0
self.lastBaseUpdate = 0
self.lastCacheUpdate = 0
self.init_samples()
def init_samples(self):
self.samples = []
self.lastBaseUpdate += self.baseline.get_update()
self.lastCacheUpdate += self.cache.get_update()
self.cache.update = 0
self.baseline.update = 0
for i in range(-self.policy["nrsamples"], self.policy["nrsamples"]):
for j in range(-self.policy["nrsamples"], self.policy["nrsamples"]):
if i == 0 and j == 0:
continue
# print("i=", i, ",j=", j)
sizeRatio = self.cacheSizeRatio + i * self.policy["deltas"]
p = self.cache.p + j * self.policy["deltap"]
# print("sr=", sizeRatio, ",p=", p)
if is_valid_sp(sizeRatio, p):
size = int(sizeRatio*self.ucln)
# print("valid", size, p)
s = PLRU(int(sizeRatio*self.ucln), p)
s.copy(self.cache, size, p)
self.samples.append(s)
# sizeRatio = self.cacheSizeRatio - i * self.policy["deltas"]
# p = self.cache.p - j * self.policy["deltap"]
# print("sr=", sizeRatio, ",p=", p)
# if is_valid_sp(sizeRatio, p):
# size = int(sizeRatio*self.ucln)
# print("valid", size, p)
# s = PLRU(size, p)
# s.copy(self.cache, size, p)
# s.update = 0
# self.samples.append(s)
# 修改
def do_req_help(self, cache, rw, blkid, roll):
hit = cache.is_hit(blkid)
if rw==1 and hit:
cache.add_update() #写【驱逐时写回】
(evicted, update) = cache.update_cache(blkid, roll)
if update==-1: # 不需要更新
update=False
else:
update=True
return (hit, update, evicted) #返回被驱逐的块以及是否命中
#
# 极端情况：某个trace在第一个周期内没有req，在get_potential中调用此函数
# 需要加个条件判断
def exceed_throt(self, hit, num):
if self.req == 0:
return False
baseline = self.baseline.get_hit()
h = 1.0*(baseline - hit)/self.req
# if (hit!=baseline):
# print("baseline", baseline, ",cache", hit, ",Dratio=", h)
if h > num:
return True
return False
# def get_close_potentials(self):
# # print("enter get close potentials")
# sizeRatio = self.cacheSizeRatio
# size = int(sizeRatio*self.ucln)
# p = self.cache.p
# pt1 = None
# pt2 = None
# for pt in self.samples:
# if pt.size == size and pt.p==p+self.policy["deltap"]:
# pt1 = pt
# 这个地方代码写的有问题
# elif pt.size == size+self.policy["deltas"] and pt.p == p:
# pt2 = pt
# if pt1!=None and pt2!=None:
# return (pt1, pt2)
# return (pt1, pt2)
# 修改
def do_req(self, rw, blkid):
random.seed()
roll = random.random()
self.req += 1
self.do_req_help(self.baseline, rw, blkid, roll)
self.do_req_help(self.baseline2, rw, blkid, roll)
#++
(hit, update, evicted)=self.do_req_help(self.cache, rw, blkid, roll)
for s in self.samples:
self.do_req_help(s, rw, blkid, roll)
if self.exceed_throt(self.cache.hit, self.policy["throt"]):
return (True, hit, update, evicted)
return (False, hit, update, evicted)
# def do_req(self, rw, blkid):
# random.seed()
# roll = random.random()
# self.req += 1
# self.do_req_help(self.baseline, rw, blkid, roll)
# self.do_req_help(self.baseline2, rw, blkid, roll)
# self.do_req_help(self.cache, rw, blkid, roll)
# for s in self.samples:
# self.do_req_help(s, rw, blkid, roll)
# # print("self=", self)
# # 命中率过低
# if self.exceed_throt(self.cache.hit, self.policy["throt"]):
# return True
# # print("after self=", self)
# # (p1, p2) = self.get_close_potentials()
# # if p1==None and p2==None:
# # return (False, self.policy["deltas"], self.policy["deltap"])
# # elif p1==None:
# # return (True, (p2.size-self.cache.size, p2.p-self.cache.p), self.policy["deltas"])
# # elif p2==None:
# # return (True, (p1.size-self.cache.size, p1.p-self.cache.p), None)
# # if p1.get_hit() > p2.get_hit():
# # return (True, (p1.size-self.cache.size, p1.p-self.cache.p), (p2.size-self.cache.size, p2.p-self.cache.p))
# # return (True, (p2.size-self.cache.size, p2.p-self.cache.p), (p1.size-self.cache.size, p1.p-self.cache.p))
# return False
def get_hit_scheme_help(self, scheme):
(s, p) = scheme
for item in self.samples:
if item.size == s and item.p == p:
return item.get_hit()
return -1
# 命中率不足时被调用，返回比当前配置和baseline【命中率高一个级别】的(deltas,dtp)列表
# 返回按照命中率从高到低排序的[(deltas, deltap, hit)]
def get_hit_scheme(self):
l = []
for item in [self.cache, self.baseline]:
for change in [(int(self.policy["deltas"]*self.ucln),0), (0, self.policy["deltap"])]:
newSize = item.size
newP = item.p
while True:
newSize += change[0]
newP += change[1]
# print(item.size, item.p)
# print(newSize, newP)
# print("end")
if is_valid_sp(1.0*newSize/self.ucln, newP):
hit = self.get_hit_scheme_help((newSize, newP))
# 如果待改的配置不是potential，默认比item大1
if hit == -1:
hit = item.get_hit()+1
l.append((newSize-self.cache.size, newP-self.cache.p, hit))
break
elif hit<=item.get_hit():
continue
else:
l.append((newSize-self.cache.size, newP-self.cache.p, hit))
break
else: #s或者p已经加到越界
break
# l不可能为空
l.sort(key=lambda item:item[2], reverse=True)
return l
#修改
# 确定更改，不存在更改无效的情况
def change_config(self, s, p):
# size = s+self.cache.size
# p = p+self.cache.p
s = min(int(self.ucln), s)
(dellist, freenode)=self.cache.change_size(s)
self.cache.change_p(p)
return (dellist, freenode)
# self.cacheSizeRatio = round(1.0*s/self.ucln, 2)
# print(self.trace, s, p, self.ucln, self.req, self.cacheSizeRatio, self.cacheSizeRatio>=1)
# assert self.cacheSizeRatio < 1
# if self.cacheSizeRatio >= 1:
# print("trace", self.trace, ",sr=", self.cacheSizeRatio, ",s=", s, "p=", p)
# sys.exit(-1)
# print("sr=", self.cacheSizeRatio)
# give potentials inside the hit range
# remove bad ones (both s and p are larger)
def get_potential(self):
# potentials = []
results = []
# print("debug", self.req, self.)
for sample in self.samples:
if self.exceed_throt(sample.get_hit(),self.policy["hitThrot"]):
continue
# potentials.append(sample)
# 注释掉优化后加
results.append((sample.get_size(), sample.get_p(), sample.get_hit(), sample.update))
# print("sample", sample)
# 这段是想要优化的，先注释掉吧
# for i in range(len(potentials)):
# sign = False
# for j in range(len(potentials)):
# # print(i, j, potentials[i].size, potentials[j].size, potentials[i].update, potentials[j].update)
# # print(sign)
# if i==j:
# continue
# if potentials[i].size >= potentials[j].size and potentials[i].update >= potentials[j].update:
# sign = True
# break
# # print(sign)
# if not sign:
# # print("debug", potentials[i].get_size(), potentials[i].get_update(), potentials[i].get_p())
# results.append(potentials[i])
if self.exceed_throt(self.cache.get_hit(),self.policy["hitThrot"]):
sample = self.cache
results.append((sample.get_size(), sample.get_p(), sample.get_hit(), sample.update))
sample = self.baseline
results.append((sample.get_size(), sample.get_p(), sample.get_hit(), sample.update))
return results
# 因为中间计算时把cache的update减去了，这里加回来
def finish(self):
self.baseline.update += self.lastBaseUpdate
self.cache.update += self.lastCacheUpdate
def load_file(traceID, typeID, sizerate=0.1, p=1, mode='w'):
readReq = 0
size = int(sizerate * uclnDict[traceID])
ssd = PLRU(size, p)
fin = open(getPath(traceID, typeID), 'r')
lines = fin.readlines()
req = 0
myupdate = 0
minTime = None
maxTime = None
print("load file finished")
for line in lines:
items = line.strip().split(' ')
reqtype = int(items[0])
blockid = int(items[2])
if mode == 'r':
if reqtype == 1:
ssd.delete_cache(blockid)
else:
readReq += 1
if readReq % 1000000 == 0:
print(readReq)
req += 1
ssd.is_hit(blockid)
ssd.update_cache(blockid)
else:
req += 1
hit = ssd.is_hit(blockid)
if reqtype == 0:
readReq += 1
if reqtype == 1 and hit:
ssd.add_update()
myupdate += 1
ssd.update_cache(blockid)
if not hit and ssd.get_top_n(1) == [blockid]:
myupdate += 1
fin.close()
print(traceID, "size", size, p, "myupdate", myupdate)
print("total hit rate", 1.0 * ssd.hit / req, ssd.update)
logFile = open(logFilename, "a")
print(traceID,
p,
sizerate,
size,
1.0 * ssd.hit / req,
ssd.update,
req,
round(1.0 * readReq / req, 3),
minTime,
maxTime,
sep=',',
file=logFile)
logFile.close()