Esempio n. 1
0
def create_model(csv_processor, input_file_path, filter_enabled):
    name_pair_class = get_name_pair_class(input_file_path)
    if not name_pair_class:
        return None
    model = CostModel(name_pair_class,
                      csv_processor,
                      input_file_path,
                      filter_enabled=filter_enabled)
    model.train()
    return model
Esempio n. 2
0
import sys

sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")
from cost_model import CostModel

base_cost_model = CostModel(profiler_path="base_profile",
                            hlo_module_path="base_hlo.pb")
search_cost_model = CostModel(profiler_path="search_profile",
                              hlo_module_path="search_hlo.pb")

print("Base cost model:")
base_cost_model.test_accuracy()

print("Search cost model:")
search_cost_model.test_accuracy()
Esempio n. 3
0
import sys
sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")
from cost_model import CostModel

cost_model = CostModel()
cost_model.test_accuracy()
Esempio n. 4
0
def run(VM_parameter_unit, threshold_percentage, uptime, min_VM, shift, provider, prediction_type,
	vm_price_per_hour, vm_init_data, actualFileName, scaleFileName, costFileName):
    listVM = list()

    open(scaleFileName, 'w').close()
    open(costFileName, 'w').close()
    scaleCSV = open(scaleFileName,"rw+")
    costCSV = open(costFileName,"rw+")

    scaleCSV.write("Time,VM Count\n" )
    costCSV.write( "Time,Total Cost\n" )

    #print(provider)

    digix_cordinates = array.array('d')
    digiy_cordinates = array.array('d')
    x_coordinates = array.array('d')
    y_coordinates = array.array('d')
    digiy_coord_actual = array.array('d')
    modely_coord_actual = array.array('d')

    #Read vlaues while converting to VM_UNITS
    ifile = open(actualFileName, "rb")
    reader = csv.reader(ifile)
    rownum = 0
    for row in reader:
      x_coordinates.append(float(row[0]))
      y_coordinates.append(float(row[1])/(VM_parameter_unit))
      rownum += 1
    ifile.close()
    ##print(x_coordinates)
    ##print(y_coordinates)

    # Regression
    xdata = np.array(x_coordinates)
    ydata = np.array(y_coordinates)

    #Plot row data
    rowdata = Line2D(xdata, ydata)

    #plot regression line of data
    #plt1.plot(xdata, quad(xdata,popt[0],popt[1],popt[2], popt[3]), '-')
    #plot EMA
    line2d = Line2D(xdata, EMA.ema(ydata ,3))

    #plot stratos
    line2d_stratos = Line2D(xdata[2:len(xdata)]+1,Stratos.Stratos(ydata))

    #Initialize min_VMs
    #VM = [23,42] #Todo fill with randoms

    for j in range(0, min_VM):
        # id = randint(1,999)
        t = vm_init_data[j]#take this as arg?
        vm = VM(initTime=-t, endTime=-1)
        listVM.append(vm)

    #Plot number of VMs required
    vm_count = min_VM
    yvalueset = []
    if(provider == "default" and prediction_type == "reactive"):
        for i in drange(uptime, max(xdata) - shift + uptime- 1, 0.1):
            #print(i-uptime)
            z = getValue(line2d, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z/threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, i)

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata) #requirement
        digi_line  = Line2D(digixdata, actualydata) #actual

    elif(provider == "default" and prediction_type == "stratos"):
        request_count = 0
        sampling_distance = 0.1
        for i in drange(uptime, max(xdata) - shift + uptime -4, sampling_distance):
            print(i)
            line2d = line2d_stratos
            z = getValue(line2d_stratos, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z/threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
                request_count = 0
            elif vm_change < 0:
                request_count += 1
                if request_count > 2.0/sampling_distance :
                    vm_count -= removeVMs(listVM, -vm_change, provider, i)
                    request_count = 0

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata) #requirement
        digi_line  = Line2D(digixdata+4, actualydata) #actual

    elif(provider == "aws" and prediction_type == "stratos"):
        request_count = 0
        sampling_distance = 0.1
        for i in drange(uptime, max(xdata) - shift + uptime -4, 0.1):
            print(i)
            line2d = line2d_stratos
            z = getValue(line2d, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z/threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
                request_count = 0
            elif vm_change < 0:
                request_count += 1
                if request_count > 2.0/sampling_distance:
                    vm_count -= removeVMs(listVM, -vm_change, provider, i)

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata) #requirement
        digi_line  = Line2D(digixdata+4, actualydata) #actual

    elif(provider == "aws" and prediction_type == "reactive"):
            for i in drange(uptime, max(xdata) - shift + uptime- 1, 0.1):
                #print(i-uptime)
                z = getValue(line2d, i - uptime)
                yvalueset.append(z)
                ##print(z)
                new_vm_count = math.ceil(z/threshold_percentage)
                if new_vm_count < min_VM:
                    new_vm_count = min_VM

                vm_change = int(math.ceil(new_vm_count - vm_count))
                if vm_change > 0:
                    vm_count += startVMs(listVM, vm_change, i)
                elif vm_change < 0:
                    vm_count -= removeVMs(listVM, -vm_change, provider, i)

                digix_cordinates.append(i)
                digiy_cordinates.append(new_vm_count)
                digiy_coord_actual.append(vm_count)
                scaleCSV.seek(0, 2)
                scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
            digixdata = np.array(digix_cordinates)
            digiydata = np.array(digiy_cordinates)
            actualydata = np.array(digiy_coord_actual)
            lineAllocate = Line2D(digixdata, digiydata) #requirement
            digi_line  = Line2D(digixdata, actualydata) #actual

    elif(provider == "default" and prediction_type == "proactive"):
        digix, digiy = CostModel.run(actualFileName)
        for i in range(0, len(digix)):
            new_vm_count = digiy[i]
            if i != 0:
                vm_change = int(math.ceil(digiy[i] - len([i for x in listVM if x.endTime == -1])))
            else :
                vm_change = int(math.ceil(digiy[0] - len([i for x in listVM if x.endTime == -1])))

            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, digix[i])
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
            modely_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))

        digixdata = np.array(digix)
        digiydata = np.array(digiy)
        lineAllocate = Line2D(digixdata, digiydata) #requirement
        digi_line  = Line2D(digixdata, modely_coord_actual) #actual

    elif(provider == "aws" and prediction_type == "proactive"):
        digix, digiy = CostModel.run(actualFileName)
        #print(digix)
        #print(digiy)
        for i in range(0, len(digix)):
            new_vm_count = digiy[i]
            if i != 0:
                vm_change = int(math.ceil(digiy[i] - len([i for x in listVM if x.endTime == -1])))
            else :
                vm_change = int(math.ceil(digiy[0] - len([i for x in listVM if x.endTime == -1])))

            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, digix[i])
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
            modely_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))

        digixdata = np.array(digix)
        digiydata = np.array(digiy)
        lineAllocate = Line2D(digixdata, digiydata) #requirement
        digi_line  = Line2D(digixdata, modely_coord_actual) #actual

    #for vm in listVM:
        #print("VM_id %3s: %5.1f - %5.1f = %5.1f%s" % (vm.id, vm.initTime, vm.endTime,(vm.endTime if vm.endTime > 0 else i) - vm.initTime, ("" if vm.endTime > 0 else " up")))

    costValues = array.array('d')
    for k in drange(0, max(xdata), 1):
        cost = calculateAWSCost(listVM, k, vm_price_per_hour)
        costValues.append(cost)
        costCSV.seek(0, 2)
        costCSV.write("%.3f,%.3f\n" %(k,cost))

    costydata = np.array(costValues)
    costxdata = np.arange(0, max(xdata), 1)

    cost_line = Line2D(costxdata,costydata)

    yvalues = np.array(yvalueset)
    #e = mse(digiydata, yvalues)

    start = max(min(line2d.get_xdata()), min(lineAllocate.get_xdata()))
    end   = min(max(line2d.get_xdata()), max(lineAllocate.get_xdata()));
    #calculateViolation(predictLine=line2d, allocateline=lineAllocate, startTime= start , endTime= end)
    return rowdata, line2d, digi_line,cost_line
Esempio n. 5
0
import sys

sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")

import tensorflow as tf
from flask import Flask, render_template, request, jsonify
from cost_model import CostModel
import tensorflow.compiler.xla.service.hlo_pb2 as hlo_pb2
import threading

app = Flask(__name__)

my_lock = threading.Lock()
cost_model = CostModel()


@app.route('/predict', methods=['POST'])
def launch():
    if request.method == 'POST':
        print("get request")
        data = request.get_data()
        hlo_proto = hlo_pb2.HloProto()
        hlo_proto.ParseFromString(data)
        hlo_module = hlo_proto.hlo_module

        with my_lock:
            try:
                estimated_time = cost_model.estimate_time(hlo_module)
            except Exception as err:
Esempio n. 6
0
def run(VM_parameter_unit, threshold_percentage, uptime, min_VM, shift,
        provider, prediction_type, vm_price_per_hour, vm_init_data,
        actualFileName, scaleFileName, costFileName):
    listVM = list()

    open(scaleFileName, 'w').close()
    open(costFileName, 'w').close()
    scaleCSV = open(scaleFileName, "rw+")
    costCSV = open(costFileName, "rw+")

    scaleCSV.write("Time,VM Count\n")
    costCSV.write("Time,Total Cost\n")

    #print(provider)

    digix_cordinates = array.array('d')
    digiy_cordinates = array.array('d')
    x_coordinates = array.array('d')
    y_coordinates = array.array('d')
    digiy_coord_actual = array.array('d')
    modely_coord_actual = array.array('d')

    #Read vlaues while converting to VM_UNITS
    ifile = open(actualFileName, "rb")
    reader = csv.reader(ifile)
    rownum = 0
    for row in reader:
        x_coordinates.append(float(row[0]))
        y_coordinates.append(float(row[1]) / (VM_parameter_unit))
        rownum += 1
    ifile.close()
    ##print(x_coordinates)
    ##print(y_coordinates)

    # Regression
    xdata = np.array(x_coordinates)
    ydata = np.array(y_coordinates)

    #Plot row data
    rowdata = Line2D(xdata, ydata)

    #plot regression line of data
    #plt1.plot(xdata, quad(xdata,popt[0],popt[1],popt[2], popt[3]), '-')
    #plot EMA
    line2d = Line2D(xdata, EMA.ema(ydata, 3))

    #plot stratos
    line2d_stratos = Line2D(xdata[2:len(xdata)] + 1, Stratos.Stratos(ydata))

    #Initialize min_VMs
    #VM = [23,42] #Todo fill with randoms

    for j in range(0, min_VM):
        # id = randint(1,999)
        t = vm_init_data[j]  #take this as arg?
        vm = VM(initTime=-t, endTime=-1)
        listVM.append(vm)

    #Plot number of VMs required
    vm_count = min_VM
    yvalueset = []
    if (provider == "default" and prediction_type == "reactive"):
        for i in drange(uptime, max(xdata) - shift + uptime - 1, 0.1):
            #print(i-uptime)
            z = getValue(line2d, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z / threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, i)

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata, actualydata)  #actual

    elif (provider == "default" and prediction_type == "stratos"):
        request_count = 0
        sampling_distance = 0.1
        for i in drange(uptime,
                        max(xdata) - shift + uptime - 4, sampling_distance):
            print(i)
            line2d = line2d_stratos
            z = getValue(line2d_stratos, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z / threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
                request_count = 0
            elif vm_change < 0:
                request_count += 1
                if request_count > 2.0 / sampling_distance:
                    vm_count -= removeVMs(listVM, -vm_change, provider, i)
                    request_count = 0

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata + 4, actualydata)  #actual

    elif (provider == "aws" and prediction_type == "stratos"):
        request_count = 0
        sampling_distance = 0.1
        for i in drange(uptime, max(xdata) - shift + uptime - 4, 0.1):
            print(i)
            line2d = line2d_stratos
            z = getValue(line2d, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z / threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
                request_count = 0
            elif vm_change < 0:
                request_count += 1
                if request_count > 2.0 / sampling_distance:
                    vm_count -= removeVMs(listVM, -vm_change, provider, i)

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata + 4, actualydata)  #actual

    elif (provider == "aws" and prediction_type == "reactive"):
        for i in drange(uptime, max(xdata) - shift + uptime - 1, 0.1):
            #print(i-uptime)
            z = getValue(line2d, i - uptime)
            yvalueset.append(z)
            ##print(z)
            new_vm_count = math.ceil(z / threshold_percentage)
            if new_vm_count < min_VM:
                new_vm_count = min_VM

            vm_change = int(math.ceil(new_vm_count - vm_count))
            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, i)
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, i)

            digix_cordinates.append(i)
            digiy_cordinates.append(new_vm_count)
            digiy_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
        digixdata = np.array(digix_cordinates)
        digiydata = np.array(digiy_cordinates)
        actualydata = np.array(digiy_coord_actual)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata, actualydata)  #actual

    elif (provider == "default" and prediction_type == "proactive"):
        digix, digiy = CostModel.run(actualFileName)
        for i in range(0, len(digix)):
            new_vm_count = digiy[i]
            if i != 0:
                vm_change = int(
                    math.ceil(digiy[i] -
                              len([i for x in listVM if x.endTime == -1])))
            else:
                vm_change = int(
                    math.ceil(digiy[0] -
                              len([i for x in listVM if x.endTime == -1])))

            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, digix[i])
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
            modely_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))

        digixdata = np.array(digix)
        digiydata = np.array(digiy)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata, modely_coord_actual)  #actual

    elif (provider == "aws" and prediction_type == "proactive"):
        digix, digiy = CostModel.run(actualFileName)
        #print(digix)
        #print(digiy)
        for i in range(0, len(digix)):
            new_vm_count = digiy[i]
            if i != 0:
                vm_change = int(
                    math.ceil(digiy[i] -
                              len([i for x in listVM if x.endTime == -1])))
            else:
                vm_change = int(
                    math.ceil(digiy[0] -
                              len([i for x in listVM if x.endTime == -1])))

            if vm_change > 0:
                vm_count += startVMs(listVM, vm_change, digix[i])
            elif vm_change < 0:
                vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
            modely_coord_actual.append(vm_count)
            scaleCSV.seek(0, 2)
            scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))

        digixdata = np.array(digix)
        digiydata = np.array(digiy)
        lineAllocate = Line2D(digixdata, digiydata)  #requirement
        digi_line = Line2D(digixdata, modely_coord_actual)  #actual

    #for vm in listVM:
    #print("VM_id %3s: %5.1f - %5.1f = %5.1f%s" % (vm.id, vm.initTime, vm.endTime,(vm.endTime if vm.endTime > 0 else i) - vm.initTime, ("" if vm.endTime > 0 else " up")))

    costValues = array.array('d')
    for k in drange(0, max(xdata), 1):
        cost = calculateAWSCost(listVM, k, vm_price_per_hour)
        costValues.append(cost)
        costCSV.seek(0, 2)
        costCSV.write("%.3f,%.3f\n" % (k, cost))

    costydata = np.array(costValues)
    costxdata = np.arange(0, max(xdata), 1)

    cost_line = Line2D(costxdata, costydata)

    yvalues = np.array(yvalueset)
    #e = mse(digiydata, yvalues)

    start = max(min(line2d.get_xdata()), min(lineAllocate.get_xdata()))
    end = min(max(line2d.get_xdata()), max(lineAllocate.get_xdata()))
    #calculateViolation(predictLine=line2d, allocateline=lineAllocate, startTime= start , endTime= end)
    return rowdata, line2d, digi_line, cost_line
Esempio n. 7
0
#f00, (plt1) = plt.subplots(1,1,sharey=True)
#f01, (plt4) = plt.subplots(1,1,sharey=True)
#f11, (plt11) = plt.subplots(1,1,sharey=True)
#f13, (plt3) = plt.subplots(1,1,sharey=True)
#f_stratos0, (plt_str_1) = plt.subplots(1,1,sharey=True)
#f_stratos1, (plt_str_2) = plt.subplots(1,1,sharey=True)
f2, (func_plot) = plt.subplots(1, 1)
f3, (plt2) = plt.subplots(1, 1)

violation_x = array.array('d')
violation_y = array.array('d')

#plot cost function
for n in drange(0, 100, 0.01):
    violation_x.append(n)
    violation_y.append(CostModel.SLA_func(n))

func_plot.plot(violation_x, violation_y)

#filename = "../datasets/predicted_static/predicted.csv"
filename = "aws_rubispredicted.csv"
filename2 = "aws_rubisactual.csv"
rowdata, predicted, digi_line, cost_line = run(
    VM_PARAM_UNIT, REACTIVE_THREASHOLD, 0, MIN_VM, 0, "default", "reactive",
    M3_MEDIUM_HOURLY_PRICE, [0, 0], filename, "data/reactive_scale.csv",
    "data/normal_cost.csv")

#plt1.plot(rowdata.get_xdata(), rowdata.get_ydata(), "*") #rowdata
plt1.plot(predicted.get_xdata(), predicted.get_ydata())  #EMA predicted
plt1.plot(digi_line.get_xdata(),
          digi_line.get_ydata())  #Reactive Blind Killing