def create_model(csv_processor, input_file_path, filter_enabled):
name_pair_class = get_name_pair_class(input_file_path)
if not name_pair_class:
return None
model = CostModel(name_pair_class,
csv_processor,
input_file_path,
filter_enabled=filter_enabled)
model.train()
return model
import sys
sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")
from cost_model import CostModel
base_cost_model = CostModel(profiler_path="base_profile",
hlo_module_path="base_hlo.pb")
search_cost_model = CostModel(profiler_path="search_profile",
hlo_module_path="search_hlo.pb")
print("Base cost model:")
base_cost_model.test_accuracy()
print("Search cost model:")
search_cost_model.test_accuracy()
import sys
sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")
from cost_model import CostModel
cost_model = CostModel()
cost_model.test_accuracy()
def run(VM_parameter_unit, threshold_percentage, uptime, min_VM, shift, provider, prediction_type,
vm_price_per_hour, vm_init_data, actualFileName, scaleFileName, costFileName):
listVM = list()
open(scaleFileName, 'w').close()
open(costFileName, 'w').close()
scaleCSV = open(scaleFileName,"rw+")
costCSV = open(costFileName,"rw+")
scaleCSV.write("Time,VM Count\n" )
costCSV.write( "Time,Total Cost\n" )
#print(provider)
digix_cordinates = array.array('d')
digiy_cordinates = array.array('d')
x_coordinates = array.array('d')
y_coordinates = array.array('d')
digiy_coord_actual = array.array('d')
modely_coord_actual = array.array('d')
#Read vlaues while converting to VM_UNITS
ifile = open(actualFileName, "rb")
reader = csv.reader(ifile)
rownum = 0
for row in reader:
x_coordinates.append(float(row[0]))
y_coordinates.append(float(row[1])/(VM_parameter_unit))
rownum += 1
ifile.close()
##print(x_coordinates)
##print(y_coordinates)
# Regression
xdata = np.array(x_coordinates)
ydata = np.array(y_coordinates)
#Plot row data
rowdata = Line2D(xdata, ydata)
#plot regression line of data
#plt1.plot(xdata, quad(xdata,popt[0],popt[1],popt[2], popt[3]), '-')
#plot EMA
line2d = Line2D(xdata, EMA.ema(ydata ,3))
#plot stratos
line2d_stratos = Line2D(xdata[2:len(xdata)]+1,Stratos.Stratos(ydata))
#Initialize min_VMs
#VM = [23,42] #Todo fill with randoms
for j in range(0, min_VM):
# id = randint(1,999)
t = vm_init_data[j]#take this as arg?
vm = VM(initTime=-t, endTime=-1)
listVM.append(vm)
#Plot number of VMs required
vm_count = min_VM
yvalueset = []
if(provider == "default" and prediction_type == "reactive"):
for i in drange(uptime, max(xdata) - shift + uptime- 1, 0.1):
#print(i-uptime)
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z/threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, actualydata) #actual
elif(provider == "default" and prediction_type == "stratos"):
request_count = 0
sampling_distance = 0.1
for i in drange(uptime, max(xdata) - shift + uptime -4, sampling_distance):
print(i)
line2d = line2d_stratos
z = getValue(line2d_stratos, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z/threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
request_count = 0
elif vm_change < 0:
request_count += 1
if request_count > 2.0/sampling_distance :
vm_count -= removeVMs(listVM, -vm_change, provider, i)
request_count = 0
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata+4, actualydata) #actual
elif(provider == "aws" and prediction_type == "stratos"):
request_count = 0
sampling_distance = 0.1
for i in drange(uptime, max(xdata) - shift + uptime -4, 0.1):
print(i)
line2d = line2d_stratos
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z/threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
request_count = 0
elif vm_change < 0:
request_count += 1
if request_count > 2.0/sampling_distance:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata+4, actualydata) #actual
elif(provider == "aws" and prediction_type == "reactive"):
for i in drange(uptime, max(xdata) - shift + uptime- 1, 0.1):
#print(i-uptime)
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z/threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, actualydata) #actual
elif(provider == "default" and prediction_type == "proactive"):
digix, digiy = CostModel.run(actualFileName)
for i in range(0, len(digix)):
new_vm_count = digiy[i]
if i != 0:
vm_change = int(math.ceil(digiy[i] - len([i for x in listVM if x.endTime == -1])))
else :
vm_change = int(math.ceil(digiy[0] - len([i for x in listVM if x.endTime == -1])))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, digix[i])
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
modely_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix)
digiydata = np.array(digiy)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, modely_coord_actual) #actual
elif(provider == "aws" and prediction_type == "proactive"):
digix, digiy = CostModel.run(actualFileName)
#print(digix)
#print(digiy)
for i in range(0, len(digix)):
new_vm_count = digiy[i]
if i != 0:
vm_change = int(math.ceil(digiy[i] - len([i for x in listVM if x.endTime == -1])))
else :
vm_change = int(math.ceil(digiy[0] - len([i for x in listVM if x.endTime == -1])))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, digix[i])
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
modely_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" %(i, vm_count*VM_parameter_unit))
digixdata = np.array(digix)
digiydata = np.array(digiy)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, modely_coord_actual) #actual
#for vm in listVM:
#print("VM_id %3s: %5.1f - %5.1f = %5.1f%s" % (vm.id, vm.initTime, vm.endTime,(vm.endTime if vm.endTime > 0 else i) - vm.initTime, ("" if vm.endTime > 0 else " up")))
costValues = array.array('d')
for k in drange(0, max(xdata), 1):
cost = calculateAWSCost(listVM, k, vm_price_per_hour)
costValues.append(cost)
costCSV.seek(0, 2)
costCSV.write("%.3f,%.3f\n" %(k,cost))
costydata = np.array(costValues)
costxdata = np.arange(0, max(xdata), 1)
cost_line = Line2D(costxdata,costydata)
yvalues = np.array(yvalueset)
#e = mse(digiydata, yvalues)
start = max(min(line2d.get_xdata()), min(lineAllocate.get_xdata()))
end = min(max(line2d.get_xdata()), max(lineAllocate.get_xdata()));
#calculateViolation(predictLine=line2d, allocateline=lineAllocate, startTime= start , endTime= end)
return rowdata, line2d, digi_line,cost_line
import sys
sys.path.append("../")
sys.path.append("../../")
sys.path.append("../../protobuf/")
import tensorflow as tf
from flask import Flask, render_template, request, jsonify
from cost_model import CostModel
import tensorflow.compiler.xla.service.hlo_pb2 as hlo_pb2
import threading
app = Flask(__name__)
my_lock = threading.Lock()
cost_model = CostModel()
@app.route('/predict', methods=['POST'])
def launch():
if request.method == 'POST':
print("get request")
data = request.get_data()
hlo_proto = hlo_pb2.HloProto()
hlo_proto.ParseFromString(data)
hlo_module = hlo_proto.hlo_module
with my_lock:
try:
estimated_time = cost_model.estimate_time(hlo_module)
except Exception as err:
def run(VM_parameter_unit, threshold_percentage, uptime, min_VM, shift,
provider, prediction_type, vm_price_per_hour, vm_init_data,
actualFileName, scaleFileName, costFileName):
listVM = list()
open(scaleFileName, 'w').close()
open(costFileName, 'w').close()
scaleCSV = open(scaleFileName, "rw+")
costCSV = open(costFileName, "rw+")
scaleCSV.write("Time,VM Count\n")
costCSV.write("Time,Total Cost\n")
#print(provider)
digix_cordinates = array.array('d')
digiy_cordinates = array.array('d')
x_coordinates = array.array('d')
y_coordinates = array.array('d')
digiy_coord_actual = array.array('d')
modely_coord_actual = array.array('d')
#Read vlaues while converting to VM_UNITS
ifile = open(actualFileName, "rb")
reader = csv.reader(ifile)
rownum = 0
for row in reader:
x_coordinates.append(float(row[0]))
y_coordinates.append(float(row[1]) / (VM_parameter_unit))
rownum += 1
ifile.close()
##print(x_coordinates)
##print(y_coordinates)
# Regression
xdata = np.array(x_coordinates)
ydata = np.array(y_coordinates)
#Plot row data
rowdata = Line2D(xdata, ydata)
#plot regression line of data
#plt1.plot(xdata, quad(xdata,popt[0],popt[1],popt[2], popt[3]), '-')
#plot EMA
line2d = Line2D(xdata, EMA.ema(ydata, 3))
#plot stratos
line2d_stratos = Line2D(xdata[2:len(xdata)] + 1, Stratos.Stratos(ydata))
#Initialize min_VMs
#VM = [23,42] #Todo fill with randoms
for j in range(0, min_VM):
# id = randint(1,999)
t = vm_init_data[j] #take this as arg?
vm = VM(initTime=-t, endTime=-1)
listVM.append(vm)
#Plot number of VMs required
vm_count = min_VM
yvalueset = []
if (provider == "default" and prediction_type == "reactive"):
for i in drange(uptime, max(xdata) - shift + uptime - 1, 0.1):
#print(i-uptime)
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z / threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, actualydata) #actual
elif (provider == "default" and prediction_type == "stratos"):
request_count = 0
sampling_distance = 0.1
for i in drange(uptime,
max(xdata) - shift + uptime - 4, sampling_distance):
print(i)
line2d = line2d_stratos
z = getValue(line2d_stratos, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z / threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
request_count = 0
elif vm_change < 0:
request_count += 1
if request_count > 2.0 / sampling_distance:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
request_count = 0
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata + 4, actualydata) #actual
elif (provider == "aws" and prediction_type == "stratos"):
request_count = 0
sampling_distance = 0.1
for i in drange(uptime, max(xdata) - shift + uptime - 4, 0.1):
print(i)
line2d = line2d_stratos
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z / threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
request_count = 0
elif vm_change < 0:
request_count += 1
if request_count > 2.0 / sampling_distance:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata + 4, actualydata) #actual
elif (provider == "aws" and prediction_type == "reactive"):
for i in drange(uptime, max(xdata) - shift + uptime - 1, 0.1):
#print(i-uptime)
z = getValue(line2d, i - uptime)
yvalueset.append(z)
##print(z)
new_vm_count = math.ceil(z / threshold_percentage)
if new_vm_count < min_VM:
new_vm_count = min_VM
vm_change = int(math.ceil(new_vm_count - vm_count))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, i)
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, i)
digix_cordinates.append(i)
digiy_cordinates.append(new_vm_count)
digiy_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix_cordinates)
digiydata = np.array(digiy_cordinates)
actualydata = np.array(digiy_coord_actual)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, actualydata) #actual
elif (provider == "default" and prediction_type == "proactive"):
digix, digiy = CostModel.run(actualFileName)
for i in range(0, len(digix)):
new_vm_count = digiy[i]
if i != 0:
vm_change = int(
math.ceil(digiy[i] -
len([i for x in listVM if x.endTime == -1])))
else:
vm_change = int(
math.ceil(digiy[0] -
len([i for x in listVM if x.endTime == -1])))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, digix[i])
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
modely_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix)
digiydata = np.array(digiy)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, modely_coord_actual) #actual
elif (provider == "aws" and prediction_type == "proactive"):
digix, digiy = CostModel.run(actualFileName)
#print(digix)
#print(digiy)
for i in range(0, len(digix)):
new_vm_count = digiy[i]
if i != 0:
vm_change = int(
math.ceil(digiy[i] -
len([i for x in listVM if x.endTime == -1])))
else:
vm_change = int(
math.ceil(digiy[0] -
len([i for x in listVM if x.endTime == -1])))
if vm_change > 0:
vm_count += startVMs(listVM, vm_change, digix[i])
elif vm_change < 0:
vm_count -= removeVMs(listVM, -vm_change, provider, digix[i])
modely_coord_actual.append(vm_count)
scaleCSV.seek(0, 2)
scaleCSV.write("%.3f,%.3f\n" % (i, vm_count * VM_parameter_unit))
digixdata = np.array(digix)
digiydata = np.array(digiy)
lineAllocate = Line2D(digixdata, digiydata) #requirement
digi_line = Line2D(digixdata, modely_coord_actual) #actual
#for vm in listVM:
#print("VM_id %3s: %5.1f - %5.1f = %5.1f%s" % (vm.id, vm.initTime, vm.endTime,(vm.endTime if vm.endTime > 0 else i) - vm.initTime, ("" if vm.endTime > 0 else " up")))
costValues = array.array('d')
for k in drange(0, max(xdata), 1):
cost = calculateAWSCost(listVM, k, vm_price_per_hour)
costValues.append(cost)
costCSV.seek(0, 2)
costCSV.write("%.3f,%.3f\n" % (k, cost))
costydata = np.array(costValues)
costxdata = np.arange(0, max(xdata), 1)
cost_line = Line2D(costxdata, costydata)
yvalues = np.array(yvalueset)
#e = mse(digiydata, yvalues)
start = max(min(line2d.get_xdata()), min(lineAllocate.get_xdata()))
end = min(max(line2d.get_xdata()), max(lineAllocate.get_xdata()))
#calculateViolation(predictLine=line2d, allocateline=lineAllocate, startTime= start , endTime= end)
return rowdata, line2d, digi_line, cost_line
#f00, (plt1) = plt.subplots(1,1,sharey=True)
#f01, (plt4) = plt.subplots(1,1,sharey=True)
#f11, (plt11) = plt.subplots(1,1,sharey=True)
#f13, (plt3) = plt.subplots(1,1,sharey=True)
#f_stratos0, (plt_str_1) = plt.subplots(1,1,sharey=True)
#f_stratos1, (plt_str_2) = plt.subplots(1,1,sharey=True)
f2, (func_plot) = plt.subplots(1, 1)
f3, (plt2) = plt.subplots(1, 1)
violation_x = array.array('d')
violation_y = array.array('d')
#plot cost function
for n in drange(0, 100, 0.01):
violation_x.append(n)
violation_y.append(CostModel.SLA_func(n))
func_plot.plot(violation_x, violation_y)
#filename = "../datasets/predicted_static/predicted.csv"
filename = "aws_rubispredicted.csv"
filename2 = "aws_rubisactual.csv"
rowdata, predicted, digi_line, cost_line = run(
VM_PARAM_UNIT, REACTIVE_THREASHOLD, 0, MIN_VM, 0, "default", "reactive",
M3_MEDIUM_HOURLY_PRICE, [0, 0], filename, "data/reactive_scale.csv",
"data/normal_cost.csv")
#plt1.plot(rowdata.get_xdata(), rowdata.get_ydata(), "*") #rowdata
plt1.plot(predicted.get_xdata(), predicted.get_ydata()) #EMA predicted
plt1.plot(digi_line.get_xdata(),
digi_line.get_ydata()) #Reactive Blind Killing