def _line(self, measurement): self._numMeas = self._numMeas + 1 if self._numMeas % 50 == 0: self._header(overwrite=False) err = [] valid = measurement.isValid(err) txt = "%-10s; %-12s; %-8s; %-7d; %-10.5f; %-10.5f; %-10.5f; %-10.5f; %-10.5f; %-12.5f; %-12.5f; %-12.5f; %-8s; %-10s" \ % (formatCurrTime(fmt="%H:%M:%S"), self.vmType.readableName, self.readableName, measurement.numberOfUsers(), measurement.getAnomaly(), measurement.cpuUtil(), measurement.ramUtil(), measurement.normaliseCpuUtil(), measurement.normaliseRAMUtil(), measurement.getAvgPrevNormCPU(), measurement.normaliseCpuCapacity(), measurement.normaliseRAMCapacity(), str(valid), str(err) ) with open(self.statFile, "a+") as f: f.write(txt + "\n") return txt
def _line(self, measurement): self._numMeas = self._numMeas + 1 if self._numMeas % 50 == 0: self._header(overwrite=False) err = [] valid = measurement.isValid(err) txt = "%-10s; %-12s; %-8s; %-7d; %-10.5f; %-10.5f; %-10.5f; %-10.5f; %-10.5f; %-12.5f; %-12.5f; %-12.5f; %-8s; %-10s" \ % (formatCurrTime(fmt="%H:%M:%S"), self.vmType.readableName, self.readableName, measurement.numberOfUsers(), measurement.getAnomaly(), measurement.cpuUtil(), measurement.ramUtil(), measurement.normaliseCpuUtil(), measurement.normaliseRAMUtil(), measurement.getAvgPrevNormCPU(), measurement.normaliseCpuCapacity(), measurement.normaliseRAMCapacity(), str(valid), str(err) ) with open(self.statFile, "a+") as f: f.write(txt+"\n") return txt
def __init__(self, readableName, address, pemFile, vmType, monitoringScript, billingPolicy, password = None, userName="******", htm=None, startTimeSecs=None): """ Constr. @param readableName: see superclass. @param address: see superclass. @param pemFile: see superclass. @param vmType: The type of the VM. Must not be None. Must be an instance of VMType. @param monitoringScript: the location of the monitoring script. Must not be None. Must be valid. @param billingPolicy: the billing policy. Must not be null. @param userName: see superclass. @param htm: The HTM to use, or None if a new HTM should be created. @param startTimeSecs: the time, when the VM was started in milliseconds. If None - the VM is considered starting now. """ super(AppServer, self).__init__(readableName = readableName, address = address, pemFile = pemFile, userName = userName, password = password) assert htm is None or isinstance(htm, HTMWrapper), "Invalid HTM type %s" % type(htm) assert monitoringScript is not None and os.path.isfile(monitoringScript), "Invalid monitoring script %s" % (monitoringScript) assert isinstance(vmType, VMType), "Invalid VM type %s" % (vmType) assert startTimeSecs is None or startTimeSecs > 0, "Invalid start time %s" % (startTimeSecs) assert billingPolicy is not None, "Billing policty is None" self.monitoringScript = monitoringScript self.vmType = vmType self.htm = htm if htm is not None else HTMWrapper() self.lastMeasurement = None self.startTimeSecs = startTimeSecs if startTimeSecs is not None else currentTimeSecs() self.billingPolicy = billingPolicy # Start the monitoring remotePath = os.path.join("/home", self.userName, os.path.os.path.basename(self.monitoringScript)) log.info("Copying Monitoring File: %s to %s:%s", self.monitoringScript, self.readableName, remotePath) sftp = self.getSSHClient().open_sftp() sftp.put(self.monitoringScript, remotePath) sftp.close() log.info("Starting the monitoring script on: %s" % (self.readableName)) self.execRemoteCommand(command="bash {0} &> /dev/null".format(remotePath), asynch=True) self.statFile=os.path.expanduser("~/%s-%s.txt" % (readableName, formatCurrTime("%d-%measurement-%Y-%H:%M:%S")) ) log.info("Record measurements in: %s" %(self.statFile)) self._header() self._numMeas = 0
def selectVMType(fann, vmTypes, scalingStatFile, minUsers, maxUsers, serverName, delta=5, maxUtil=0.7): with open(scalingStatFile, "a+") as f: txt = "\n\n=== %s: Selecting VM type for \"%s\", minU(%d) maxU(%d)" % (formatCurrTime(), serverName, minUsers, maxUsers) f.write(txt+"\n") bestVMt = None bestCost = maxint; # For all vms for vmt in vmTypes: cpuCapacity = VMType.inferNormalisedCPUCapacity(vmt, vmTypes) ramCapacity = vmt.normalisedRAMCapacity() # Find how many users can it take n = minUsers uCapacity = 1 while True: cpu = maxint ram = maxint if n > maxUsers: runMin = fann.run(minUsers) cpuMin = runMin[0] ramMin = runMin[1] runMax = fann.run(maxUsers) cpuMax = runMax[0] ramMax = runMax[1] cpuPerUser = (cpuMax - cpuMin) / float(maxUsers - minUsers) ramPerUser = (ramMax - ramMin) / float(maxUsers - minUsers) cpu = cpuMax + (n - maxUsers) * cpuPerUser ram = ramMax + (n - maxUsers) * ramPerUser else : run = fann.run(n) cpu = run[0] ram = run[1] with open(scalingStatFile, "a+") as f: txt = "%-10s : VM type=%-15s, n=%-4d, CPUpred=%.4f, RAMpred=%.4f, CPUCap=%.4f, RAMCap=%.4f " % \ (formatCurrTime(), vmt.readableName, n, cpu, ram, maxUtil*cpuCapacity, maxUtil*ramCapacity ) f.write(txt+"\n") if cpu < maxUtil*cpuCapacity and ram < maxUtil*ramCapacity: uCapacity = n else: break n = n + delta # Compute the cost per user uCost = vmt.costPerTimeUnit / uCapacity txt1 ="Definition of VMType \"%s\": CPU(%.4f), RAM(%.4f), Cost (%.4f) " % (vmt.readableName, cpuCapacity, ramCapacity, vmt.costPerTimeUnit) txt2 = "Capacity of VMType \"%s\": CPU(%.4f), RAM(%.4f), Users(%d), Cost per user(%.6f) " % (vmt.readableName, cpu, ram, uCapacity, uCost) log.info(txt1) log.info(txt2) with open(scalingStatFile, "a+") as f: f.write(txt1+"\n") f.write(txt2+"\n=======\n") # Choose the best type if uCost < bestCost: bestCost = uCost bestVMt = vmt with open(scalingStatFile, "a+") as f: f.write("--> Selected VM type:" + bestVMt.readableName +"\n") return bestVMt
from autoscale.FANNWrapper import FANNWrapper from autoscale.VMType import VMType from autoscale.Util import convertMem, sigmoid, nextEpoch, getMk, getLrk, formatCurrTime, statHeader, statLine, printTest from autoscale import VMFactory log = logging.getLogger(__name__) ##== Get command line arguements inputUnits = int(sys.argv[1]) if len(sys.argv) > 1 else 1 hiddenUnits = int(sys.argv[2]) if len(sys.argv) > 2 else 250 lr = float(sys.argv[3]) if len(sys.argv) > 3 else 0.001 epochCode = int(sys.argv[4]) if len(sys.argv) > 4 else 1 trainingStatFile = os.path.expanduser( "~/RESULTS-NN(%d-%d-%d) LR(%.4f) EpCode(%d)-%s.txt" % (inputUnits, hiddenUnits, 2, lr, epochCode, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) trainingResFile = os.path.expanduser( "~/NN(%d-%d-%d) LR(%.4f) EpCode(%d)-%s.txt" % (inputUnits, hiddenUnits, 2, lr, epochCode, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) scalingStatFile = os.path.expanduser( "~/Scale-%s.txt" % (formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) ##== Set up auxiliary result files printTest(trainingStatFile, None, overwrite=True) statHeader(trainingResFile, overwrite=True) ##== Configure the level, handler and format for the loggers rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG)
def _measName(self): return "{0}: {1}".format(self.readableName, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))
def __init__(self, readableName, address, pemFile, vmType, monitoringScript, billingPolicy, password=None, userName="******", htm=None, startTimeSecs=None): """ Constr. @param readableName: see superclass. @param address: see superclass. @param pemFile: see superclass. @param vmType: The type of the VM. Must not be None. Must be an instance of VMType. @param monitoringScript: the location of the monitoring script. Must not be None. Must be valid. @param billingPolicy: the billing policy. Must not be null. @param userName: see superclass. @param htm: The HTM to use, or None if a new HTM should be created. @param startTimeSecs: the time, when the VM was started in milliseconds. If None - the VM is considered starting now. """ super(AppServer, self).__init__(readableName=readableName, address=address, pemFile=pemFile, userName=userName, password=password) assert htm is None or isinstance( htm, HTMWrapper), "Invalid HTM type %s" % type(htm) assert monitoringScript is not None and os.path.isfile( monitoringScript), "Invalid monitoring script %s" % ( monitoringScript) assert isinstance(vmType, VMType), "Invalid VM type %s" % (vmType) assert startTimeSecs is None or startTimeSecs > 0, "Invalid start time %s" % ( startTimeSecs) assert billingPolicy is not None, "Billing policty is None" self.monitoringScript = monitoringScript self.vmType = vmType self.htm = htm if htm is not None else HTMWrapper() self.lastMeasurement = None self.startTimeSecs = startTimeSecs if startTimeSecs is not None else currentTimeSecs( ) self.billingPolicy = billingPolicy # Start the monitoring remotePath = os.path.join( "/home", self.userName, os.path.os.path.basename(self.monitoringScript)) log.info("Copying Monitoring File: %s to %s:%s", self.monitoringScript, self.readableName, remotePath) sftp = self.getSSHClient().open_sftp() sftp.put(self.monitoringScript, remotePath) sftp.close() log.info("Starting the monitoring script on: %s" % (self.readableName)) self.execRemoteCommand( command="bash {0} &> /dev/null".format(remotePath), asynch=True) self.statFile = os.path.expanduser( "~/%s-%s.txt" % (readableName, formatCurrTime("%d-%measurement-%Y-%H:%M:%S"))) log.info("Record measurements in: %s" % (self.statFile)) self._header() self._numMeas = 0
def _measName(self): return "{0}: {1}".format( self.readableName, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))
from autoscale.FANNWrapper import FANNWrapper from autoscale.VMType import VMType from workload.Workload import Workload from workload.ClientFactory import ClientFactory from autoscale.Util import convertMem, sigmoid, nextEpoch, getMk, getLrk, formatCurrTime, statHeader, statLine, printTest log = logging.getLogger(__name__) ##== Get command line arguements inputUnits = int(sys.argv[1]) if len(sys.argv) > 1 else 1 hiddenUnits = int(sys.argv[2]) if len(sys.argv) > 2 else 250 lr = float(sys.argv[3]) if len(sys.argv) > 3 else 0.001 epochCode = int(sys.argv[4]) if len(sys.argv) > 4 else 1 trainingStatFile=os.path.expanduser("~/RESULTS-NN(%d-%d-%d) LR(%.4f) EpCode(%d)-%s.txt" % (inputUnits, hiddenUnits, 2, lr, epochCode, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) trainingResFile=os.path.expanduser("~/NN(%d-%d-%d) LR(%.4f) EpCode(%d)-%s.txt" % (inputUnits, hiddenUnits, 2, lr, epochCode, formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) scalingStatFile=os.path.expanduser("~/Scale-%s.txt" % ( formatCurrTime(fmt='%d-%measurement-%Y %H:%M:%S'))) ##== Set up auxiliary result files printTest(trainingStatFile, None, overwrite=True) statHeader(trainingResFile, overwrite=True) ##== Configure the level, handler and format for the loggers rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) ch = logging.StreamHandler(sys.stdout) ch.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s', datefmt="%H:%M:%S")
def selectVMType(fann, vmTypes, scalingStatFile, minUsers, maxUsers, serverName, delta=5, maxUtil=0.7): with open(scalingStatFile, "a+") as f: txt = "\n\n=== %s: Selecting VM type for \"%s\", minU(%d) maxU(%d)" % ( formatCurrTime(), serverName, minUsers, maxUsers) f.write(txt + "\n") bestVMt = None bestCost = maxint # For all vms for vmt in vmTypes: cpuCapacity = VMType.inferNormalisedCPUCapacity(vmt, vmTypes) ramCapacity = vmt.normalisedRAMCapacity() # Find how many users can it take n = minUsers uCapacity = 1 while True: cpu = maxint ram = maxint if n > maxUsers: runMin = fann.run(minUsers) cpuMin = runMin[0] ramMin = runMin[1] runMax = fann.run(maxUsers) cpuMax = runMax[0] ramMax = runMax[1] cpuPerUser = (cpuMax - cpuMin) / float(maxUsers - minUsers) ramPerUser = (ramMax - ramMin) / float(maxUsers - minUsers) cpu = cpuMax + (n - maxUsers) * cpuPerUser ram = ramMax + (n - maxUsers) * ramPerUser else: run = fann.run(n) cpu = run[0] ram = run[1] with open(scalingStatFile, "a+") as f: txt = "%-10s : VM type=%-15s, n=%-4d, CPUpred=%.4f, RAMpred=%.4f, CPUCap=%.4f, RAMCap=%.4f " % \ (formatCurrTime(), vmt.readableName, n, cpu, ram, maxUtil*cpuCapacity, maxUtil*ramCapacity ) f.write(txt + "\n") if cpu < maxUtil * cpuCapacity and ram < maxUtil * ramCapacity: uCapacity = n else: break n = n + delta # Compute the cost per user uCost = vmt.costPerTimeUnit / uCapacity txt1 = "Definition of VMType \"%s\": CPU(%.4f), RAM(%.4f), Cost (%.4f) " % ( vmt.readableName, cpuCapacity, ramCapacity, vmt.costPerTimeUnit) txt2 = "Capacity of VMType \"%s\": CPU(%.4f), RAM(%.4f), Users(%d), Cost per user(%.6f) " % ( vmt.readableName, cpu, ram, uCapacity, uCost) log.info(txt1) log.info(txt2) with open(scalingStatFile, "a+") as f: f.write(txt1 + "\n") f.write(txt2 + "\n=======\n") # Choose the best type if uCost < bestCost: bestCost = uCost bestVMt = vmt with open(scalingStatFile, "a+") as f: f.write("--> Selected VM type:" + bestVMt.readableName + "\n") return bestVMt