def setUp(self):
config = """
[Cluster]
builder: ('simulator.cluster', 'ClusterBuilder')
servers: ['Ampere']
[Configurator]
builder: ('system.configurator', 'ConfiguratorBuilder')
dependencies: ['Cluster', 'Predictor']
loopPeriod: 20
windowSize: 20
gamma: 1.0
[DVFSController]
builder: ('system.DVFSController', 'DVFSControllerBuilder')
[Ampere]
idlePower: [66.3,70.5,72.7]
busyPower: [81.5,101.8,109.8]
peakPerformance: [170.0,309.9,346.8]
freqs: [1.0,1.8,2.0]
"""
cr = ConfigReader(None)
sio = StringIO(config)
cr.readfp(sio)
cluster = ClusterBuilder().build(cr)
self.configurator = ConfiguratorBuilder().build(cr, cluster, None)
def main():
filename = '../tmp/config.txt'
c = ConfigReader()
c.parse(filename)
cluster = c.readCluster()
maxPerf = getMaxPerformance(cluster.availableServers)
outline = readOutlineFile('../tmp/outline.txt')
trace = generateTrace(outline, maxPerf, 20)
if Gnuplot is not None:
data = Data(trace, with_='l')
g = Gnuplot()
g('set ytics 0, 100')
g('set xtics 0, 20')
g('set grid')
g.plot(data)
raw_input()
file = open('../tmp/trace_gen.txt','w')
for value in trace:
file.write('%f\n' % value)
file.close()
def main():
monitorFilename = '2011-06-07_11h44m46s/monitor.txt'
data = file2data(27, monitorFilename, 0)
configFilename = '2011-06-07_11h44m46s/wc98_1_freqs.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
cluster.servers[Server.ON] = cluster.availableServers
del cluster.servers[Server.OFF][:]
pe = PowerEmulator(cluster)
map = {'ampere': 12, 'coulomb': 15, 'hertz': 18, 'joule': 21, 'ohm': 24}
#offsets
FREQ = 0
FCAP = 1
UTIL = 2
#absolute
POWER = 2 - 1
for i in range(len(data[0])):
# take utilization and freqs
for server in cluster:
hostname = server.hostname
server.processor.util = data[map[hostname] + UTIL][i] / 100.0
server.processor.freq = data[map[hostname] + FREQ][i] * 1000000.0
if i < len(data[0]) - 1:
print data[POWER][i + 1], pe.calcPower()
def main():
monitorFilename = '2011-06-07_11h44m46s/monitor.txt'
data = file2data(27, monitorFilename, 0)
configFilename = '2011-06-07_11h44m46s/wc98_1_freqs.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
cluster.servers[Server.ON] = cluster.availableServers
del cluster.servers[Server.OFF][:]
pe = PowerEmulator(cluster)
map = {'ampere': 12, 'coulomb': 15, 'hertz': 18, 'joule': 21, 'ohm': 24}
#offsets
FREQ = 0
FCAP = 1
UTIL = 2
#absolute
POWER = 2 - 1
for i in range(len(data[0])):
# take utilization and freqs
for server in cluster:
hostname = server.hostname
server.processor.util = data[map[hostname] + UTIL][i] / 100.0
server.processor.freq = data[map[hostname] + FREQ][i] * 1000000.0
if i < len(data[0]) - 1:
print data[POWER][i+1], pe.calcPower()
def main():
filename = '../../tmp/nasa_3.txt'
step = 20
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data, cluster.availableServers, maxUtil=1.0)
data = []
acc = 0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) % step == 0:
data.append(float(acc) / step)
acc = 0
rest = len(raw_data) % step
if rest > 0:
data.append(float(acc) / rest)
#print gridMethod2_h3(raw_data, window=step, step=0.05, Yalpha=0.8, method=MAPE)
#print naive(data,MSE)
print gridMethod2_h2(data, 0.05, MSE)
def main():
filename = '../../tmp/wc98_1.txt'
method = MSE
precision = 0.05
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data,cluster.availableServers,maxUtil=1.0)
for pred_step in range(1, 500):
"""
Take the averages at pred_step windows
"""
data = []
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) % pred_step == 0:
data.append(acc / pred_step)
acc = 0
rest = len(raw_data) % pred_step
if rest > 0:
data.append(acc / rest)
#take the optimized Holt parameters:
"""
pars = gridOpt(data, precision, method)
holt = HoltFilter(alpha=pars[0], beta=pars[1])
"""
holt = AdaptableHoltFilter(alpha=1.0, beta=0.05, windowSize=10, optMethod=method)
#apply Holt and Naive to the real data
holtError = 0.0
naiveError = 0.0
acc = 0.0
avg = 0.0
pred = 0.0
for i in range(len(raw_data)):
if (i + 1) > pred_step: #don't count the first error
naiveError += error(avg, raw_data[i], method)
holtError += error(pred, raw_data[i], method)
acc += raw_data[i]
if (i + 1) % pred_step == 0:
avg = acc / pred_step
pred = holt.apply(avg)
acc = 0.0
holtError /= len(raw_data) - pred_step
naiveError /= len(raw_data) - pred_step
print pred_step, naiveError, holtError
def setUp(self):
config = """
[Cluster]
builder: ('simulator.cluster', 'ClusterBuilder')
servers: ['Ampere', 'Coulomb', 'Hertz', 'Joule', 'Ohm']
[Configurator]
builder: ('system.configurator', 'ConfiguratorBuilder')
dependencies: ['Cluster', 'Predictor']
loopPeriod: 20
windowSize: 20
gamma: 1.0
[DVFSController]
builder: ('system.DVFSController', 'DVFSControllerBuilder')
[Ampere]
hostname: ampere
idlePower: [66.3,70.5,72.7]
busyPower: [81.5,101.8,109.8]
peakPerformance: [170.0,309.9,346.8]
freqs: [1.0,1.8,2.0]
[Coulomb]
hostname: coulomb
idlePower: [67.4,70.9,72.4,73.8,75.2]
busyPower: [75.2,89.0,94.5,100.9,107.7]
peakPerformance: [84.0,152.5,168.6,184.5,199.8]
freqs: [1.0,1.8,2.0,2.2,2.4]
[Hertz]
hostname: hertz
idlePower: [63.9,67.2,68.7,69.9,71.6]
busyPower: [71.6,85.5,90.7,96.5,103.2]
peakPerformance: [82.8,151.3,167.6,183.8,198.5]
freqs: [1.0,1.8,2.0,2.2,2.4]
[Joule]
hostname: joule
idlePower: [66.6,73.8,76.9,80.0]
busyPower: [74.7,95.7,103.1,110.6]
peakPerformance: [82.0,148.3,163.3,179.4]
freqs: [1.0,1.8,2.0,2.2]
[Ohm]
hostname: ohm
idlePower: [65.8,68.5,70.6,72.3,74.3,76.9]
busyPower: [82.5,99.2,107.3,116.6,127.2,140.1]
peakPerformance: [164.8,301.5,337.9,370.9,404.4,439.5]
freqs: [1.0,1.8,2.0,2.2,2.4,2.6]
"""
cr = ConfigReader(None)
sio = StringIO(config)
cr.readfp(sio)
cluster = ClusterBuilder().build(cr)
self.configurator = ConfiguratorBuilder().build(cr, cluster, None)
def main(argv):
config = ConfigReader()
config.parse('config.txt')
cluster = config.readCluster()
server = cluster._availableServers[0]
server.processor.detect()
controller = ContinuousController(server.processor)
controller._setup()
dz = DeadzoneController()
dz.setDeadzone((QOS_TARGET - 0.03, QOS_TARGET + 0.03))
dz.setPoint(QOS_TARGET)
lost_min = server.energyModel.peakPerf[0] / (1 - QOS_TARGET)
time = int(argv[0])
count = 0
accLost, oldaccLost = 0.0, 0.0
accReplied, oldaccReplied = 0.0, 0.0
qos, qos_avg = 1.0, 1.0
lost_avg = 0.0
print '#1:Time 2:QoS 3:Lost 4:Frequency 5:QoS_Avg 6:Lost_Avg'
while count < time:
tmp = readFile(LOST_FILENAME)
if len(tmp) > 0: accLost = int(tmp)
tmp = readFile(REPLIED_FILENAME)
if len(tmp) > 0: accReplied = int(tmp)
if count > 0:
lost = accLost - oldaccLost
lost_avg = ALPHA * max(lost_min, lost) + (1 - ALPHA) * lost_avg
replied = accReplied - oldaccReplied
if replied > 0.0:
qos = float(replied - lost) / replied
qos_avg = ALPHA * qos + (1 - ALPHA) * qos_avg
action = dz.action(qos_avg)
if action != 0 and (count % ACTUATE == 0):
target_replied = lost_avg / (1.0 - QOS_TARGET)
freq = server.energyModel.getFreq(target_replied)
controller.setSpeed(freq)
print '%3d %1.3f %3.0f %1.3f %1.3f %3.0f' % (
count, qos, lost, controller.getFreqCap() / 1000000.0, qos_avg,
lost_avg)
oldaccReplied = accReplied
oldaccLost = accLost
sleep(1)
count += 1
controller.stop()
def main():
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
configurator = ConfiguratorBuilder().build(c, cluster, None)
servers = cluster.availableServers * 100000
maxPerf = getMaxPerformance(servers)
cProfile.runctx('configurator.getConfiguration(maxPerf, servers, 1.0)',globals=globals(),locals=locals())
def main(argv):
config = ConfigReader()
config.parse('config.txt')
cluster = config.readCluster()
server = cluster._availableServers[0]
server.processor.detect()
controller = ContinuousController(server.processor)
controller._setup()
dz = DeadzoneController()
dz.setDeadzone((QOS_TARGET - 0.03, QOS_TARGET + 0.03))
dz.setPoint(QOS_TARGET)
lost_min = server.energyModel.peakPerf[0] / (1 - QOS_TARGET)
time = int(argv[0])
count = 0
accLost, oldaccLost = 0.0, 0.0
accReplied, oldaccReplied = 0.0, 0.0
qos, qos_avg = 1.0, 1.0
lost_avg = 0.0
print '#1:Time 2:QoS 3:Lost 4:Frequency 5:QoS_Avg 6:Lost_Avg'
while count < time:
tmp = readFile(LOST_FILENAME)
if len(tmp) > 0: accLost = int(tmp)
tmp = readFile(REPLIED_FILENAME)
if len(tmp) > 0: accReplied = int(tmp)
if count > 0:
lost = accLost - oldaccLost
lost_avg = ALPHA * max(lost_min, lost) + (1 - ALPHA) * lost_avg
replied = accReplied - oldaccReplied
if replied > 0.0:
qos = float(replied - lost) / replied
qos_avg = ALPHA * qos + (1 - ALPHA) * qos_avg
action = dz.action(qos_avg)
if action != 0 and (count % ACTUATE == 0):
target_replied = lost_avg / (1.0 - QOS_TARGET)
freq = server.energyModel.getFreq(target_replied)
controller.setSpeed(freq)
print '%3d %1.3f %3.0f %1.3f %1.3f %3.0f' % (count, qos, lost, controller.getFreqCap() / 1000000.0, qos_avg, lost_avg)
oldaccReplied = accReplied
oldaccLost = accLost
sleep(1)
count += 1
controller.stop()
def main():
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
configurator = ConfiguratorBuilder().build(c, cluster, None)
servers = cluster.availableServers * 100000
maxPerf = getMaxPerformance(servers)
cProfile.runctx('configurator.getConfiguration(maxPerf, servers, 1.0)',
globals=globals(),
locals=locals())
def main():
try:
with open(EXPS_PATH,'r') as f:
for line in f:
if len(line) > 0 and line[0] not in ['#','\n']: # ignore blank lines (\n) and comments
config = ConfigReader(line.rstrip(' \n')) # remove the trailing spaces and '\n' on the line
config.parse()
exp = BaseExperimentBuilder().build(config)
logging.info('Starting: %s' % exp)
exp.startExperiment()
exp.saveExperiment()
if exp.getStatus() != Experiment.SUCCESS:
break
sleep(3) # delay between experiments
except KeyboardInterrupt:
logging.warning('User aborted...')
exp.abortExperiment()
return
logging.info('Finished!')
def main():
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
configurator = ConfiguratorBuilder().build(c, cluster, None)
tests = 1000
maxMultiplier = 200
step = 20
for mult in xrange(1,maxMultiplier,step):
servers = cluster.availableServers * mult
maxPerf = getMaxPerformance(servers)
ti = time.time()
for i in xrange(tests):
configurator.getConfiguration(maxPerf, servers, 1.0)
tf = time.time()
print len(servers), (tf-ti) * 1000.0 / tests
def main():
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
configurator = ConfiguratorBuilder().build(c, cluster, None)
tests = 1000
maxMultiplier = 200
step = 20
for mult in xrange(1, maxMultiplier, step):
servers = cluster.availableServers * mult
maxPerf = getMaxPerformance(servers)
ti = time.time()
for i in xrange(tests):
configurator.getConfiguration(maxPerf, servers, 1.0)
tf = time.time()
print len(servers), (tf - ti) * 1000.0 / tests
def main():
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
print 'Frequency Pick Efficiency','*' * 10
pickList = []
for server in cluster:
dynPot = [busy - idle for busy, idle in zip(server.energyModel.busyPower,server.energyModel.idlePower)]
deltaPerf = [server.energyModel.peakPerf[0]]
deltaPerf += [server.energyModel.peakPerf[i+1] for i in range(len(server.energyModel.peakPerf)-1)]
#deltaPerf = server.energyModel.peakPerf[:]
eff = [deltaPerf[i] / dynPot[i] for i in range(len(dynPot))]
pickList += [(server.hostname, eff[i], i) for i in range(len(eff))]
print server.hostname, eff
pickList = sorted(pickList, key=lambda element: element[1], reverse=True)
for element in pickList:
print element
"""
def main():
try:
with open(EXPS_PATH, 'r') as f:
for line in f:
if len(line) > 0 and line[0] not in [
'#', '\n'
]: # ignore blank lines (\n) and comments
config = ConfigReader(
line.rstrip(' \n')
) # remove the trailing spaces and '\n' on the line
config.parse()
exp = BaseExperimentBuilder().build(config)
logging.info('Starting: %s' % exp)
exp.startExperiment()
exp.saveExperiment()
if exp.getStatus() != Experiment.SUCCESS:
break
sleep(3) # delay between experiments
except KeyboardInterrupt:
logging.warning('User aborted...')
exp.abortExperiment()
return
logging.info('Finished!')
def main():
traceFilename = '../../tmp/wc98_1.txt'
configFilename = '../../tmp/dynGamma.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
predictor = cReader.readPredictor()
lb = cReader.readLoadBalancer(cluster)
cluster.loadBalancer = lb
config = cluster._availableServers[:]
bestConfig = config[:]
predConfig = config[:]
reactiveConfig = config[:]
bestConfigurator = cReader.readConfigurator(cluster, DummyFilter())
bestConfigurator.setEfficientServerList(cluster._availableServers)
bestConfigurator.gamma = 1.0 #do not change!!
predConfigurator = cReader.readConfigurator(cluster, predictor)
predConfigurator.setEfficientServerList(cluster._availableServers)
predConfigurator._gamma = 1.0
reactiveConfigurator = cReader.readConfigurator(cluster, DummyFilter())
reactiveConfigurator.setEfficientServerList(cluster._availableServers)
#reactiveConfigurator._gamma = 0.55
step = predConfigurator.windowSize
#reads the trace file
data = file2data(1, traceFilename)[0]
scale(data, cluster.availableServers)
predCounter, reactiveCounter, bestCounter = 0, 0, 0 #configuration counter
pred_upCounter, pred_downCounter = 0, 0
best_upCounter, best_downCounter = 0, 0
reactive_upCounter, reactive_downCounter = 0, 0
bestPerf = getMaxPerformance(bestConfig)
reactivePerf = getMaxPerformance(bestConfig)
predPerf = getMaxPerformance(bestConfig)
avgs = []
reactivePerfs = []
predPerfs = []
bestPerfs = []
gammas = []
for i in range(len(data) / step):
avgData = 0.0
for j in range(step):
avgData += data[i * step + j]
avgData /= step
avgs.append(avgData)
reactivePerfs.append(reactivePerf)
predPerfs.append(predPerf)
optimumConfig = None
'''
Will use the efficient list to create the optimumConfig
'''
optimumConfig = [bestConfigurator._efficientServerList[0]]
k = 1
while getMaxPerformance(optimumConfig) < avgData:
optimumConfig.append(bestConfigurator._efficientServerList[k])
k += 1
'''
Calculates the appropriate configuration, 'knowing' that avgData is true
'''
if i > 0: #calculates false alarms
tmp = bestConfigurator.getConfiguration(avgData, bestConfig,
bestConfigurator._gamma)
if tmp != bestConfig:
bestCounter += 1
bestConfig = tmp
bestPerf = getMaxPerformance(bestConfig)
bestPerfs.append(bestPerf)
'''
Compares the chosen configuration with a least powerful one
'''
if avgData > bestPerf:
best_downCounter += 1
elif getMaxPerformance(bestConfig) > getMaxPerformance(
optimumConfig):
best_upCounter += 1
if avgData > reactivePerf:
reactive_downCounter += 1
elif getMaxPerformance(reactiveConfig) > getMaxPerformance(
optimumConfig):
reactive_upCounter += 1
if avgData > predPerf:
pred_downCounter += 1
elif getMaxPerformance(predConfig) > getMaxPerformance(
optimumConfig):
pred_upCounter += 1
elif len(bestConfig) < len(predConfig):
pred_upCounter += 1
prediction = predictor.apply(avgData)
#actuation = prediction
actuation = max(prediction, avgData)
tmp = predConfigurator.getConfiguration(actuation, predConfig,
predConfigurator._gamma)
if set(tmp) != set(predConfig):
predCounter += 1
predConfig = tmp
predPerf = getMaxPerformance(predConfig)
if reactiveConfigurator.dynGamma:
idealConfig = bestConfigurator.getConfiguration(
avgData, reactiveConfig, reactiveConfigurator.MAX_GAMMA)
idealPerf = round(getMaxPerformance(idealConfig), 5)
currPerf = round(getMaxPerformance(reactiveConfig), 5)
reactiveConfigurator._gamma = reactiveConfigurator.adjustGamma(
currPerf - idealPerf)
gammas.append(reactiveConfigurator._gamma)
tmp = reactiveConfigurator.getConfiguration(
avgData, reactiveConfig, reactiveConfigurator._gamma)
if set(tmp) != set(reactiveConfig):
reactiveCounter += 1
reactiveConfig = tmp
reactivePerf = getMaxPerformance(reactiveConfig)
print 'Best - False alarms: up = %d, down = %d' % (best_upCounter,
best_downCounter)
print 'Predictive - False alarms: up = %d, down = %d' % (pred_upCounter,
pred_downCounter)
print 'Reactive - False alarms: up = %d, down = %d' % (
reactive_upCounter, reactive_downCounter)
print 'Best Configs = %d, Predictive Configs = %d, Reactive Configs = %d' %\
(bestCounter, predCounter, reactiveCounter)
g = Gnuplot(debug=0)
g('reset')
#g('set size 0.65,0.65')
g('set encoding iso_8859_1')
g('set terminal postscript eps enhanced monochrome')
g('set output "carlos2.ps"')
g('set key top left')
g('set xlabel "Tempo (s)"')
g('set xrange [0:%d]' % (len(data) / step))
spacing = 15
tics = ','.join([
'"%d" %d' % (x * step * spacing, x * spacing)
for x in range(1,
len(data) / (step * spacing) + 1)
])
g('set xtics (%s)' % tics)
g('set ylabel "Requisições"'.encode('iso_8859_1'))
g('set grid')
#plot incoming requests avg
with_plot = 'steps'
#plot configurations
pi1 = Data(avgs, with_=with_plot, title='Carga')
pi2 = Data(reactivePerfs, with_=with_plot, title='Reativo')
pi_gamma = Data(gammas, with_=with_plot, title='Gamma')
#pi3 = Data(predPerfs, with_=with_plot,title='Preditivo')
#pi4 = Data(bestPerfs, with_=with_plot,title='Perfect')
#g.plot(pi1,pi2)
#g.plot(pi_gamma)
#raw_input()
print 'Done'
def main():
traceFilename = '../../tmp/nasa_3.txt'
configFilename = '../../tmp/probSub.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
lb = cReader.readLoadBalancer(cluster)
cluster.loadBalancer = lb
bestConfigurator = cReader.readConfigurator(cluster, DummyFilter())
bestConfigurator.setEfficientServerList(cluster._availableServers)
bestConfigurator.gamma = 1.0 #do not change!!
reactiveConfigurator = cReader.readConfigurator(cluster, DummyFilter())
reactiveConfigurator.setEfficientServerList(cluster._availableServers)
step = bestConfigurator.windowSize
#reads the trace file
data = file2data(1, traceFilename)[0]
scale(data, cluster.availableServers)
gamma = 0.0
while round(gamma,5) <= MAX_GAMMA:
config = cluster._availableServers[:]
bestConfig = config[:]
reactiveConfig = config[:]
reactiveConfigurator._gamma = gamma
reactiveCounter, bestCounter = 0, 0 #configuration counter
best_upCounter, best_downCounter = 0, 0
reactive_upCounter, reactive_downCounter = 0, 0
bestPerf = getMaxPerformance(bestConfig)
reactivePerf = getMaxPerformance(bestConfig)
subestimated = 0.0
for i in range(len(data) / step):
avgData = 0.0
for j in range(step):
avgData += data[i * step + j]
avgData /= step
optimumConfig = None
'''
Will use the efficient list to create the optimumConfig
'''
optimumConfig = [bestConfigurator._efficientServerList[0]]
k = 1
while getMaxPerformance(optimumConfig) < avgData:
optimumConfig.append(bestConfigurator._efficientServerList[k])
k += 1
'''
Calculates the appropriate configuration, 'knowing' that avgData is true
'''
if i > 0: #calculates false alarms
tmp = bestConfigurator.getConfiguration(avgData, bestConfig, bestConfigurator._gamma)
if tmp != bestConfig:
bestCounter += 1
bestConfig = tmp
bestPerf = getMaxPerformance(bestConfig)
'''
Compares the chosen configuration with a least powerful one
'''
if avgData > bestPerf:
best_downCounter += 1
elif getMaxPerformance(bestConfig) > getMaxPerformance(optimumConfig):
best_upCounter += 1
if avgData > reactivePerf:
reactive_downCounter += 1
elif getMaxPerformance(reactiveConfig) > getMaxPerformance(optimumConfig):
reactive_upCounter += 1
for j in range(step):
subestimated += max(data[i * step + j] - reactivePerf, 0)
if reactiveConfigurator.dynGamma:
idealConfig = bestConfigurator.getConfiguration(avgData, reactiveConfig, reactiveConfigurator.MAX_GAMMA)
idealPerf = round(getMaxPerformance(idealConfig),5)
currPerf = round(getMaxPerformance(reactiveConfig),5)
reactiveConfigurator._gamma = reactiveConfigurator.adjustGamma(currPerf - idealPerf)
tmp = reactiveConfigurator.getConfiguration(avgData, reactiveConfig, reactiveConfigurator._gamma)
if set(tmp) != set(reactiveConfig):
reactiveCounter += 1
reactiveConfig = tmp
reactivePerf = getMaxPerformance(reactiveConfig)
#print 'Gamma = %f' % gamma
#print 'Best - super = %d, sub = %d' % (best_upCounter, best_downCounter)
#print 'Reactive - super = %d, sub = %d' % (reactive_upCounter, reactive_downCounter)
#print 'Best Configs = %d, Reactive Configs = %d' %\
# (bestCounter, reactiveCounter)
print gamma, 1.0 - reactive_downCounter / (float(len(data) - step) / step), reactiveCounter
gamma += GAMMA_INCREMENT
def main():
reader = ConfigReader('../tmp/config.txt')
exp = reader.readExperiment()
exp._perfRegulator.actuate = actuate
exp.startExperiment()
print 'end!'
def main():
traceFilename = '../../tmp/nasa_3.txt'
configFilename = '../../tmp/probSub.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
lb = cReader.readLoadBalancer(cluster)
cluster.loadBalancer = lb
bestConfigurator = cReader.readConfigurator(cluster, DummyFilter())
bestConfigurator.setEfficientServerList(cluster._availableServers)
bestConfigurator.gamma = 1.0 #do not change!!
reactiveConfigurator = cReader.readConfigurator(cluster, DummyFilter())
reactiveConfigurator.setEfficientServerList(cluster._availableServers)
step = bestConfigurator.windowSize
#reads the trace file
data = file2data(1, traceFilename)[0]
scale(data, cluster.availableServers)
gamma = 0.0
while round(gamma, 5) <= MAX_GAMMA:
config = cluster._availableServers[:]
bestConfig = config[:]
reactiveConfig = config[:]
reactiveConfigurator._gamma = gamma
reactiveCounter, bestCounter = 0, 0 #configuration counter
best_upCounter, best_downCounter = 0, 0
reactive_upCounter, reactive_downCounter = 0, 0
bestPerf = getMaxPerformance(bestConfig)
reactivePerf = getMaxPerformance(bestConfig)
subestimated = 0.0
for i in range(len(data) / step):
avgData = 0.0
for j in range(step):
avgData += data[i * step + j]
avgData /= step
optimumConfig = None
'''
Will use the efficient list to create the optimumConfig
'''
optimumConfig = [bestConfigurator._efficientServerList[0]]
k = 1
while getMaxPerformance(optimumConfig) < avgData:
optimumConfig.append(bestConfigurator._efficientServerList[k])
k += 1
'''
Calculates the appropriate configuration, 'knowing' that avgData is true
'''
if i > 0: #calculates false alarms
tmp = bestConfigurator.getConfiguration(
avgData, bestConfig, bestConfigurator._gamma)
if tmp != bestConfig:
bestCounter += 1
bestConfig = tmp
bestPerf = getMaxPerformance(bestConfig)
'''
Compares the chosen configuration with a least powerful one
'''
if avgData > bestPerf:
best_downCounter += 1
elif getMaxPerformance(bestConfig) > getMaxPerformance(
optimumConfig):
best_upCounter += 1
if avgData > reactivePerf:
reactive_downCounter += 1
elif getMaxPerformance(reactiveConfig) > getMaxPerformance(
optimumConfig):
reactive_upCounter += 1
for j in range(step):
subestimated += max(data[i * step + j] - reactivePerf, 0)
if reactiveConfigurator.dynGamma:
idealConfig = bestConfigurator.getConfiguration(
avgData, reactiveConfig, reactiveConfigurator.MAX_GAMMA)
idealPerf = round(getMaxPerformance(idealConfig), 5)
currPerf = round(getMaxPerformance(reactiveConfig), 5)
reactiveConfigurator._gamma = reactiveConfigurator.adjustGamma(
currPerf - idealPerf)
tmp = reactiveConfigurator.getConfiguration(
avgData, reactiveConfig, reactiveConfigurator._gamma)
if set(tmp) != set(reactiveConfig):
reactiveCounter += 1
reactiveConfig = tmp
reactivePerf = getMaxPerformance(reactiveConfig)
#print 'Gamma = %f' % gamma
#print 'Best - super = %d, sub = %d' % (best_upCounter, best_downCounter)
#print 'Reactive - super = %d, sub = %d' % (reactive_upCounter, reactive_downCounter)
#print 'Best Configs = %d, Reactive Configs = %d' %\
# (bestCounter, reactiveCounter)
print gamma, 1.0 - reactive_downCounter / (float(len(data) - step) /
step), reactiveCounter
gamma += GAMMA_INCREMENT
def main():
traceFilename = '../../tmp/wc98_1.txt'
configFilename = '../../tmp/dynGamma.txt'
cReader = ConfigReader()
cReader.parse(configFilename)
cluster = cReader.readCluster()
predictor = cReader.readPredictor()
lb = cReader.readLoadBalancer(cluster)
cluster.loadBalancer = lb
config = cluster._availableServers[:]
bestConfig = config[:]
predConfig = config[:]
reactiveConfig = config[:]
bestConfigurator = cReader.readConfigurator(cluster, DummyFilter())
bestConfigurator.setEfficientServerList(cluster._availableServers)
bestConfigurator.gamma = 1.0 #do not change!!
predConfigurator = cReader.readConfigurator(cluster, predictor)
predConfigurator.setEfficientServerList(cluster._availableServers)
predConfigurator._gamma = 1.0
reactiveConfigurator = cReader.readConfigurator(cluster, DummyFilter())
reactiveConfigurator.setEfficientServerList(cluster._availableServers)
#reactiveConfigurator._gamma = 0.55
step = predConfigurator.windowSize
#reads the trace file
data = file2data(1, traceFilename)[0]
scale(data, cluster.availableServers)
predCounter, reactiveCounter, bestCounter = 0, 0, 0 #configuration counter
pred_upCounter, pred_downCounter = 0, 0
best_upCounter, best_downCounter = 0, 0
reactive_upCounter, reactive_downCounter = 0, 0
bestPerf = getMaxPerformance(bestConfig)
reactivePerf = getMaxPerformance(bestConfig)
predPerf = getMaxPerformance(bestConfig)
avgs = []
reactivePerfs = []
predPerfs = []
bestPerfs = []
gammas = []
for i in range(len(data) / step):
avgData = 0.0
for j in range(step):
avgData += data[i * step + j]
avgData /= step
avgs.append(avgData)
reactivePerfs.append(reactivePerf)
predPerfs.append(predPerf)
optimumConfig = None
'''
Will use the efficient list to create the optimumConfig
'''
optimumConfig = [bestConfigurator._efficientServerList[0]]
k = 1
while getMaxPerformance(optimumConfig) < avgData:
optimumConfig.append(bestConfigurator._efficientServerList[k])
k += 1
'''
Calculates the appropriate configuration, 'knowing' that avgData is true
'''
if i > 0: #calculates false alarms
tmp = bestConfigurator.getConfiguration(avgData, bestConfig, bestConfigurator._gamma)
if tmp != bestConfig:
bestCounter += 1
bestConfig = tmp
bestPerf = getMaxPerformance(bestConfig)
bestPerfs.append(bestPerf)
'''
Compares the chosen configuration with a least powerful one
'''
if avgData > bestPerf:
best_downCounter += 1
elif getMaxPerformance(bestConfig) > getMaxPerformance(optimumConfig):
best_upCounter += 1
if avgData > reactivePerf:
reactive_downCounter += 1
elif getMaxPerformance(reactiveConfig) > getMaxPerformance(optimumConfig):
reactive_upCounter += 1
if avgData > predPerf:
pred_downCounter += 1
elif getMaxPerformance(predConfig) > getMaxPerformance(optimumConfig):
pred_upCounter += 1
elif len(bestConfig) < len(predConfig):
pred_upCounter += 1
prediction = predictor.apply(avgData)
#actuation = prediction
actuation = max(prediction, avgData)
tmp = predConfigurator.getConfiguration(actuation, predConfig, predConfigurator._gamma)
if set(tmp) != set(predConfig):
predCounter += 1
predConfig = tmp
predPerf = getMaxPerformance(predConfig)
if reactiveConfigurator.dynGamma:
idealConfig = bestConfigurator.getConfiguration(avgData, reactiveConfig, reactiveConfigurator.MAX_GAMMA)
idealPerf = round(getMaxPerformance(idealConfig),5)
currPerf = round(getMaxPerformance(reactiveConfig),5)
reactiveConfigurator._gamma = reactiveConfigurator.adjustGamma(currPerf - idealPerf)
gammas.append(reactiveConfigurator._gamma)
tmp = reactiveConfigurator.getConfiguration(avgData, reactiveConfig, reactiveConfigurator._gamma)
if set(tmp) != set(reactiveConfig):
reactiveCounter += 1
reactiveConfig = tmp
reactivePerf = getMaxPerformance(reactiveConfig)
print 'Best - False alarms: up = %d, down = %d' % (best_upCounter, best_downCounter)
print 'Predictive - False alarms: up = %d, down = %d' % (pred_upCounter, pred_downCounter)
print 'Reactive - False alarms: up = %d, down = %d' % (reactive_upCounter, reactive_downCounter)
print 'Best Configs = %d, Predictive Configs = %d, Reactive Configs = %d' %\
(bestCounter, predCounter, reactiveCounter)
g = Gnuplot(debug=0)
g('reset')
#g('set size 0.65,0.65')
g('set encoding iso_8859_1')
g('set terminal postscript eps enhanced monochrome')
g('set output "carlos2.ps"')
g('set key top left')
g('set xlabel "Tempo (s)"')
g('set xrange [0:%d]' % (len(data) / step))
spacing = 15
tics = ','.join(['"%d" %d' % (x * step * spacing, x * spacing) for x in range(1, len(data) / (step * spacing) + 1)])
g('set xtics (%s)' % tics)
g('set ylabel "Requisições"'.encode('iso_8859_1'))
g('set grid')
#plot incoming requests avg
with_plot = 'steps'
#plot configurations
pi1 = Data(avgs, with_=with_plot,title='Carga')
pi2 = Data(reactivePerfs, with_=with_plot,title='Reativo')
pi_gamma = Data(gammas, with_=with_plot,title='Gamma')
#pi3 = Data(predPerfs, with_=with_plot,title='Preditivo')
#pi4 = Data(bestPerfs, with_=with_plot,title='Perfect')
#g.plot(pi1,pi2)
#g.plot(pi_gamma)
#raw_input()
print 'Done'
def main():
filename = '../../tmp/wc98_1.txt'
method = MSE
precision = 0.05
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data, cluster.availableServers, maxUtil=1.0)
for pred_step in range(1, 500):
"""
Take the averages at pred_step windows
"""
data = []
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) % pred_step == 0:
data.append(acc / pred_step)
acc = 0
rest = len(raw_data) % pred_step
if rest > 0:
data.append(acc / rest)
#take the optimized Holt parameters:
"""
pars = gridOpt(data, precision, method)
holt = HoltFilter(alpha=pars[0], beta=pars[1])
"""
holt = AdaptableHoltFilter(alpha=1.0,
beta=0.05,
windowSize=10,
optMethod=method)
#apply Holt and Naive to the real data
holtError = 0.0
naiveError = 0.0
acc = 0.0
avg = 0.0
pred = 0.0
for i in range(len(raw_data)):
if (i + 1) > pred_step: #don't count the first error
naiveError += error(avg, raw_data[i], method)
holtError += error(pred, raw_data[i], method)
acc += raw_data[i]
if (i + 1) % pred_step == 0:
avg = acc / pred_step
pred = holt.apply(avg)
acc = 0.0
holtError /= len(raw_data) - pred_step
naiveError /= len(raw_data) - pred_step
print pred_step, naiveError, holtError
def main():
filename = '../../tmp/nasa_3.txt'
method = MAE
pred_step = 10 # applies predictions every given seconds
precision = 0.05
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data,cluster.availableServers,maxUtil=1.0)
data = []
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) % pred_step == 0:
data.append(acc / pred_step)
acc = 0
rest = len(raw_data) % pred_step
if rest > 0:
data.append(acc / rest)
"""
Calculate the optimum pars
"""
opt_pars = gridOpt(data, precision, method)
"""
Print header
"""
print "#trace: " + filename
print "#pred_step: %d" % pred_step
print "#precision: %f" % precision
print '#error: %s' % errorsDict[method]
print '#opt-alpha: %f, opt-beta: %f' % (opt_pars[0],opt_pars[1])
print '#1-adapt_step #2-adapt_error #3-opt_error'
"""
Calculate the adaptive error
"""
for adapt_step in range(pred_step, 500):
adapt_window_size = adapt_step # adapt parameters using a window of the given size
opt_holt = HoltFilter(alpha=opt_pars[0],beta=opt_pars[0])
opt_pred = 0.0
opt_error = 0.0
adapt_window = []
'''
Holt's initialization doesn't matter, because we'll consider
the error only after the first adaption
'''
adapt_holt = HoltFilter()
adapt_pred = 0.0
adapt_error = 0.0
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) > adapt_step: # calculate error according to method, skipping the first window
adapt_error += error(adapt_pred, raw_data[i], method)
opt_error += error(opt_pred, raw_data[i], method)
if (i + 1) % pred_step == 0: # got a data point
adapt_window.append(acc / pred_step)
if len(adapt_window) > adapt_window_size:
adapt_window.pop(0)
if (i + 1) % adapt_step == 0: # do adaption
adapt_pars = gridOpt(adapt_window, precision, MSE)
adapt_holt.alpha = adapt_pars[0]
adapt_holt.beta = adapt_pars[1]
if (i + 1) % pred_step == 0: # do prediction after adaptation
avg = acc / pred_step
adapt_pred = adapt_holt.apply(avg)
opt_pred = opt_holt.apply(avg)
acc = 0.0
print adapt_step, adapt_error / (len(raw_data) - adapt_step), opt_error / (len(raw_data) - adapt_step)
def main():
filename = '../../tmp/nasa_3.txt'
method = MAE
pred_step = 10 # applies predictions every given seconds
precision = 0.05
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data, cluster.availableServers, maxUtil=1.0)
data = []
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (i + 1) % pred_step == 0:
data.append(acc / pred_step)
acc = 0
rest = len(raw_data) % pred_step
if rest > 0:
data.append(acc / rest)
"""
Calculate the optimum pars
"""
opt_pars = gridOpt(data, precision, method)
"""
Print header
"""
print "#trace: " + filename
print "#pred_step: %d" % pred_step
print "#precision: %f" % precision
print '#error: %s' % errorsDict[method]
print '#opt-alpha: %f, opt-beta: %f' % (opt_pars[0], opt_pars[1])
print '#1-adapt_step #2-adapt_error #3-opt_error'
"""
Calculate the adaptive error
"""
for adapt_step in range(pred_step, 500):
adapt_window_size = adapt_step # adapt parameters using a window of the given size
opt_holt = HoltFilter(alpha=opt_pars[0], beta=opt_pars[0])
opt_pred = 0.0
opt_error = 0.0
adapt_window = []
'''
Holt's initialization doesn't matter, because we'll consider
the error only after the first adaption
'''
adapt_holt = HoltFilter()
adapt_pred = 0.0
adapt_error = 0.0
acc = 0.0
for i in range(len(raw_data)):
acc += raw_data[i]
if (
i + 1
) > adapt_step: # calculate error according to method, skipping the first window
adapt_error += error(adapt_pred, raw_data[i], method)
opt_error += error(opt_pred, raw_data[i], method)
if (i + 1) % pred_step == 0: # got a data point
adapt_window.append(acc / pred_step)
if len(adapt_window) > adapt_window_size:
adapt_window.pop(0)
if (i + 1) % adapt_step == 0: # do adaption
adapt_pars = gridOpt(adapt_window, precision, MSE)
adapt_holt.alpha = adapt_pars[0]
adapt_holt.beta = adapt_pars[1]
if (i + 1) % pred_step == 0: # do prediction after adaptation
avg = acc / pred_step
adapt_pred = adapt_holt.apply(avg)
opt_pred = opt_holt.apply(avg)
acc = 0.0
print adapt_step, adapt_error / (
len(raw_data) - adapt_step), opt_error / (len(raw_data) -
adapt_step)
def main():
filename = '../../tmp/nasa_3.txt'
step = 20
method = MSE
f = open(filename)
lines = f.read()
f.close()
raw_data = [float(token) for token in lines.splitlines()]
c = ConfigReader()
c.parse('../../tmp/config.txt')
cluster = c.readCluster()
scale(raw_data, cluster.availableServers, maxUtil=1.0)
#holt1 = HoltFilter(alpha=0.50, beta=0.10) #jeito ruim
holt2 = HoltFilter(alpha=0.65, beta=0.05) #com média
'''
# apply holt1
pred = 0
error1 = 0.0
for i in range(len(raw_data)):
if (i + 1) > step:
error1 += error(pred, raw_data[i], method)
holt1.apply(raw_data[i]) # update holt
if (i + 1) % step == 0: # every 'step' apply holt
pred = holt1.S + step * holt1.B
error1 /= len(raw_data) - step
'''
# apply holt2
acc = 0.0
pred = 0
error2 = 0.0
for i in range(len(raw_data)):
if (i + 1) > step:
error2 += error(pred, raw_data[i], method)
acc += raw_data[i]
if (i + 1) % step == 0: # every 'step' apply holt
pred = holt2.apply(float(acc) / step)
acc = 0.0
error2 /= len(raw_data) - step
print '#step: %f' % step
print '#error: %s' % errorsDict[method]
#print '#holt1_error: %f' % error1
print '#holt2_error: %f' % error2
print '#1-alpha #2-error'
alpha = 1.0
dec = 0.01
while alpha > 0.0:
# apply holt3
acc = 0
pred = 0.0
error3 = 0.0
holt3_pars = gridMethod2_h3(raw_data, window=step, step=0.05, Yalpha=alpha, method=method)
holt3 = HoltFilter(alpha=holt3_pars[0], beta=holt3_pars[1]) #com exp avg
#holt3 = HoltFilter(alpha=0.55, beta=0.05) #com exp avg
Y = raw_data[0] # initialization of exponential average
for i in range(len(raw_data)):
Y = alpha * raw_data[i] + (1 - alpha) * Y # smooth the data
if (i + 1) > step:
error3 += error(pred, raw_data[i], method)
if (i + 1) % step == 0: # every 'step' apply holt
pred = holt3.apply(Y)
acc = 0
print alpha, error3 / (len(raw_data) - step)
alpha -= dec