def findLottery(vote, classHeights, solverSettings):
'''
Returns a Lottery satisfying all constraints specified by the classHeights parameter
@type vote: vote.society.Vote
@type classHeights: dict(vote.society.ChoiceClass, float)
@type solverSettings: vote.solver.settings.SolverSettings
@rtype: vote.society.Lottery
@raise ValueError: If the constraints are not satisfiable
'''
classNames = getUniqueNames(classHeights.keys(), prefix="Class ")
choiceNames = getUniqueNames(vote.getChoices(), prefix="Choice ")
problem = LpProblem("Lambda", LpMaximize)
choiceVariables = LpVariable.dicts("p", choiceNames.values(), lowBound=0)
problem += lpSum(choiceVariables) <= 1, "Distribution"
for choiceClass, height in classHeights.items():
problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \
height, classNames[choiceClass] + " height"
problem.setObjective(lpSum(choiceVariables.values()))
checkPulpStatus(problem.solve(solverSettings.getSolver()))
choiceValues = dict()
for choice, choiceName in choiceNames.items():
choiceValues[choice.getObject()] = choiceVariables[choiceName].value()
return Lottery(choiceValues, solverSettings)
def update_with_soft_affinity_constraints_and_objective(self,variables,prob, num_hosts, num_instances):
#Adding column sum variables whose value is 1 if there is any instance on that host
column_sum_var = []
for i in range(num_hosts):
column_sum_var.append(pulp.LpVariable("Normalised_Column_Sum_Host_"+str(i), 0, 1, constants.LpInteger))
#Adding normalisation constraint
for i in range(num_hosts):
prob += pulp.lpSum([variables[i][j]] for j in range(num_instances)) <= num_instances*column_sum_var[i]
prob += column_sum_var[i] <= pulp.lpSum([variables[i][j]] for j in range(num_instances))
z_variables =[]
#Adding 'z' variables
for i in range(num_hosts):
for j in range(num_hosts):
if i != j:
z_variables.append(pulp.LpVariable("Z_variable_Col_"+str(i)+"Col_"+str(j), 0, 1, constants.LpInteger))
temp = 0
for i in range(num_hosts):
for j in range(num_hosts):
if i != j:
prob += column_sum_var[i] + column_sum_var[j] <= z_variables[temp] + 1
prob += 2 * z_variables[temp]<=column_sum_var[i] + column_sum_var[j]
#print str(temp) + " " + str(z_variables[temp])
temp = temp + 1
#Adding the objective
prob+=z_variables[0] * 0 + z_variables[1] * 3 + z_variables[2] * 1 + z_variables[4] * 3 + z_variables[5] * 5 + z_variables[8] * 4
return prob
def findLottery(vote, classHeights, solverSettings):
'''
Returns a Lottery satisfying all constraints specified by the classHeights parameter
@type vote: vote.society.Vote
@type classHeights: dict(vote.society.ChoiceClass, float)
@type solverSettings: vote.solver.settings.SolverSettings
@rtype: vote.society.Lottery
@raise ValueError: If the constraints are not satisfiable
'''
classNames = getUniqueNames(classHeights.keys(), prefix="Class ")
choiceNames = getUniqueNames(vote.getChoices(), prefix="Choice ")
problem = LpProblem("Lambda", LpMaximize)
choiceVariables = LpVariable.dicts("p", choiceNames.values(), lowBound=0)
problem += lpSum(choiceVariables) <= 1, "Distribution"
for choiceClass, height in classHeights.items():
problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \
height, classNames[choiceClass] + " height"
problem.setObjective(lpSum(choiceVariables.values()))
checkPulpStatus(problem.solve(solverSettings.getSolver()))
choiceValues = dict()
for choice, choiceName in choiceNames.items():
choiceValues[choice.getObject()] = choiceVariables[choiceName].value()
return Lottery(choiceValues, solverSettings)
def _objective(xs, betas, route, msg_load, hosting_cost):
# We want to minimize communication and hosting costs
# Objective is the communication + hosting costs
comm = LpAffineExpression()
for c1, a1, c2, a2 in betas:
comm += route(a1, a2) * msg_load(c1, c2) * betas[(c1, a1, c2, a2)]
costs = lpSum([hosting_cost(a, c) * xs[(c, a)] for c, a in xs])
return lpSum([RATIO_HOST_COMM * comm , (1-RATIO_HOST_COMM) * costs])
def update_with_strict_affinity_constraints_and_objective(self,variables,prob, num_hosts, num_instances):
#Adding column sum variables whose value is 1 if there is any instance on that host
column_sum_var = []
for i in range(num_hosts):
column_sum_var.append(pulp.LpVariable("Normalised_Column_Sum_Host_"+str(i), 0, 1, constants.LpInteger))
#Adding normalisation constraint
for i in range(num_hosts):
prob += pulp.lpSum([variables[i][j]] for j in range(num_instances)) <= num_instances*column_sum_var[i]
prob += column_sum_var[i] <= pulp.lpSum([variables[i][j]] for j in range(num_instances))
prob += pulp.lpSum([column_sum_var[i]] for i in range(num_hosts)) == 1
return prob
def calcola_orario(self, model, dati, str_aux):
skd = str_aux.get_schedulazione()
# Funzione obiettivo
model += lpSum(skd[(c,m,a,g,s)] for c in dati.get_corsi() for m in dati.get_moduli() for a in dati.get_aule()
for g in dati.get_giorni() for s in dati.get_slot())
model.solve()
return pl.LpStatus[model.status]
def createConstraints(problem, variable):
problem += lpSum(choiceVariables) <= 1, "Distribution"
for choiceClass, height in state.getCurrentClassHeights().items():
problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \
height, classNames[choiceClass] + " height"
for agent in state.getActiveAgents():
problem += createLpSum(state.getCurrentAgentChoiceClass(agent),
choiceNames, choiceVariables) >= \
state.getAgentHeight(agent) + variable * state.getAgentSpeed(agent), \
agentNames[agent] + " push"
def createConstraints(problem, variable):
problem += lpSum(choiceVariables) <= 1, "Distribution"
for choiceClass, height in state.getCurrentClassHeights().items():
problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \
height, classNames[choiceClass] + " height"
for agent in state.getActiveAgents():
problem += createLpSum(state.getCurrentAgentChoiceClass(agent),
choiceNames, choiceVariables) >= \
state.getAgentHeight(agent) + variable * state.getAgentSpeed(agent), \
agentNames[agent] + " push"
def update_with_strict_affinity_constraints_and_objective(
self, variables, prob, num_hosts, num_instances):
#Adding column sum variables whose value is 1 if there is any instance on that host
column_sum_var = []
for i in range(num_hosts):
column_sum_var.append(
pulp.LpVariable("Normalised_Column_Sum_Host_" + str(i), 0, 1,
constants.LpInteger))
#Adding normalisation constraint
for i in range(num_hosts):
prob += pulp.lpSum([variables[i][j]] for j in range(
num_instances)) <= num_instances * column_sum_var[i]
prob += column_sum_var[i] <= pulp.lpSum(
[variables[i][j]] for j in range(num_instances))
prob += pulp.lpSum([column_sum_var[i]] for i in range(num_hosts)) == 1
return prob
def update_with_soft_affinity_constraints_and_objective(
self, variables, prob, num_hosts, num_instances):
#Adding column sum variables whose value is 1 if there is any instance on that host
column_sum_var = []
for i in range(num_hosts):
column_sum_var.append(
pulp.LpVariable("Normalised_Column_Sum_Host_" + str(i), 0, 1,
constants.LpInteger))
#Adding normalisation constraint
for i in range(num_hosts):
prob += pulp.lpSum([variables[i][j]] for j in range(
num_instances)) <= num_instances * column_sum_var[i]
prob += column_sum_var[i] <= pulp.lpSum(
[variables[i][j]] for j in range(num_instances))
z_variables = []
#Adding 'z' variables
for i in range(num_hosts):
for j in range(num_hosts):
if i != j:
z_variables.append(
pulp.LpVariable(
"Z_variable_Col_" + str(i) + "Col_" + str(j), 0, 1,
constants.LpInteger))
temp = 0
for i in range(num_hosts):
for j in range(num_hosts):
if i != j:
prob += column_sum_var[i] + column_sum_var[
j] <= z_variables[temp] + 1
prob += 2 * z_variables[temp] <= column_sum_var[
i] + column_sum_var[j]
#print str(temp) + " " + str(z_variables[temp])
temp = temp + 1
#Adding the objective
prob += z_variables[0] * 0 + z_variables[1] * 3 + z_variables[
2] * 1 + z_variables[4] * 3 + z_variables[5] * 5 + z_variables[
8] * 4
return prob
def imposta_vincoli_obbligatori(self, model, dati, str_aux):
skd=str_aux.get_schedulazione()
# Vincolo che evita il sovrapporsi di moduli insegnati dagli stessi docenti
for d in str_aux.get_docenti():
for s in dati.get_slot():
for g in dati.get_giorni():
model += lpSum(skd[(c,m,a,g,s)]*str_aux.get_titolari_moduli()[(m,d)] for m in dati.get_moduli() for a in dati.get_aule() for c in dati.get_corsi()) <= 1
# Vincolo che fissa il numero di slot da assegnare per un modulo di un'attività didattica in una settimana
for m in dati.get_moduli():
model += lpSum(skd[(c,m,a,g,s)] for a in dati.get_aule() for g in dati.get_giorni() for s in dati.get_slot() for c in dati.get_corsi()) == (m.get_num_sessioni()*m.get_dur_sessioni())
# Vincolo che impedisce che ad un aula venga assegnato più di un corso in uno slot di un dato giorno
for a in dati.get_aule():
for g in dati.get_giorni():
for s in dati.get_slot():
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() for c in dati.get_corsi()) <= 1
# Un corso per un dato giorno in un dato slot può essere assegnato ad una sola aula
for c in dati.get_corsi():
for m in dati.get_moduli():
for g in dati.get_giorni():
for s in dati.get_slot():
model += lpSum(skd[(c, m, a, g, s)] for a in dati.get_aule()) <= 1
# Vincoli che permettono di assegnare ad un corso solo aule compatibili con la numerosità del corso stesso
for c in dati.get_corsi():
for m in dati.get_moduli():
for g in dati.get_giorni():
for s in dati.get_slot():
for a in dati.get_aule():
if str_aux.get_compatibilita_aule()[(m,a)] == 0:
model += skd[(c,m,a,g,s)] == 0
# Vincolo che non consente la sovrapposizione di moduli inerenti ad attività didattiche previste allo stesso anno di corso
for c in dati.get_corsi():
for g in dati.get_giorni():
for s in dati.get_slot():
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 1 for a in dati.get_aule()) <= 1
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 2 for a in dati.get_aule()) <= 1
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 3 for a in dati.get_aule()) <= 1
# Rende impossibili gli abbinamenti di assegnazioni di moduli di codici corso diversi dal codice del corso in esame
for c in dati.get_corsi():
for g in dati.get_giorni():
for s in dati.get_slot():
for m in dati.get_moduli():
for a in dati.get_aule():
if c.get_codice() != m.get_cod_corso():
model += skd[(c,m,a,g,s)] == 0
def get_optimal_rackspace(cpu_usage, mem_usage, optimal=True, debug=False):
"""
Calculates the price of optimal resources allocation over a certain time span (with monthly granularity).
Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function.
"""
assert(len(cpu_usage) == len(mem_usage))
prob = LpProblem("Rackspace cost optimization", LpMinimize)
# variables
## 1h instances
per_h_ondems = []
for p in range(len(cpu_usage)):
per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()]
category = LpInteger if optimal else LpContinuous
vars = LpVariable.dicts("rackspace", per_h_ondems, 0, None, cat=category)
# objective function
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]), "Total cost of running the infra (wrt to CPU/RAM)"
# constraints
## demand constraints
for p in range(len(cpu_usage)):
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][2] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CU demand period %s" %p
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand period %s" %p
prob.solve()
if debug:
for v in prob.variables():
if v.varValue != 0.0:
print v.name, "=", v.varValue
print "Total Cost of the solution = ", value(prob.objective)
return value(prob.objective)
def imposta_vincoli_addizionali(self, model, dati, str_aux, vincoli, cod_cds):
if vincoli["chkPreferenzeDocenti"] == 1:
skd=str_aux.get_schedulazione()
for l in dati.get_logistica():
s = l[2]
for m in dati.get_moduli():
if (m.get_offerta_id() == l[0] and m.get_id() == l[1]):
break
for c in dati.get_corsi():
if c.get_id() == m.get_corso_id():
break
g = dati.get_giorni()
model += lpSum(skd[(c,m,a,g[l[3]-1],dati.get_slot()[s-1])] for a in dati.get_aule()) == 1
if vincoli["posizioniFisse"] == None:
None
else:
skd = str_aux.get_schedulazione()
global posizioniFisse
if (cod_cds == -1):
posizioniFisse = vincoli["posizioniFisse"]
else:
posizioniFisse = [pf for pf in vincoli["posizioniFisse"] if pf['corso_id']==int(cod_cds)]
for p in posizioniFisse:
if (p["corso_id"] != -1 and p["modulo_id"] != -1 and
p["aula_id"] != -1 and p["giorno_id"] != -1 and
p["slot_id"] != -1):
for c in dati.get_corsi():
if c.get_id() == p["corso_id"]:
break
for m in dati.get_moduli():
if m.get_id() == p["modulo_id"]:
break
for a in dati.get_aule():
if a.get_id() == p["aula_id"]:
break
for g in dati.get_giorni():
if g.get_id() == p["giorno_id"]:
break
for s in dati.get_slot():
if s.get_id() == p["slot_id"]:
break
model += skd[(c,m,a,g,s)] == 1
def solve(self, hosts, filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties.
"""
host_instance_combinations = []
num_instances = filter_properties['num_instances']
num_hosts = len(hosts)
instance_uuids = filter_properties.get('instance_uuids') or [
'(unknown_uuid)' + str(i) for i in xrange(num_instances)]
filter_properties.setdefault('solver_cache', {})
filter_properties['solver_cache'].update(
{'cost_matrix': [],
'constraint_matrix': []})
cost_matrix = self._get_cost_matrix(hosts, filter_properties)
cost_matrix = self._adjust_cost_matrix(cost_matrix)
constraint_matrix = self._get_constraint_matrix(hosts,
filter_properties)
# Create dictionaries mapping temporary host/instance keys to
# hosts/instance_uuids. These temorary keys are to be used in the
# solving process since we need a convention of lp variable names.
host_keys = ['Host' + str(i) for i in xrange(num_hosts)]
host_key_map = dict(zip(host_keys, hosts))
instance_num_keys = ['InstanceNum' + str(i) for
i in xrange(num_instances + 1)]
instance_num_key_map = dict(zip(instance_num_keys,
xrange(num_instances + 1)))
# create the pulp variables
variable_matrix = [
[pulp.LpVariable('HI_' + host_key + '_' + instance_num_key,
0, 1, constants.LpInteger)
for instance_num_key in instance_num_keys]
for host_key in host_keys]
# create the 'prob' variable to contain the problem data.
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
# add cost function to pulp solver
cost_variables = [variable_matrix[i][j] for i in xrange(num_hosts)
for j in xrange(num_instances + 1)]
cost_coefficients = [cost_matrix[i][j] for i in xrange(num_hosts)
for j in xrange(num_instances + 1)]
prob += (pulp.lpSum([cost_coefficients[i] * cost_variables[i]
for i in xrange(len(cost_variables))]), "Sum_Costs")
# add constraints to pulp solver
for i in xrange(num_hosts):
for j in xrange(num_instances + 1):
if constraint_matrix[i][j] is False:
prob += (variable_matrix[i][j] == 0,
"Cons_Host_%s" % i + "_NumInst_%s" % j)
# add additional constraints to ensure the problem is valid
# (1) non-trivial solution: number of all instances == that requested
prob += (pulp.lpSum([variable_matrix[i][j] * j for i in
xrange(num_hosts) for j in xrange(num_instances + 1)]) ==
num_instances, "NonTrivialCons")
# (2) valid solution: each host is assigned 1 num-instances value
for i in xrange(num_hosts):
prob += (pulp.lpSum([variable_matrix[i][j] for j in
xrange(num_instances + 1)]) == 1, "ValidCons_Host_%s" % i)
# The problem is solved using PULP's choice of Solver.
prob.solve(pulp_solver_classes.PULP_CBC_CMD(
maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds))
# Create host-instance tuples from the solutions.
if pulp.LpStatus[prob.status] == 'Optimal':
num_insts_on_host = {}
for v in prob.variables():
if v.name.startswith('HI'):
(host_key, instance_num_key) = v.name.lstrip('HI').lstrip(
'_').split('_')
if v.varValue == 1:
num_insts_on_host[host_key] = (
instance_num_key_map[instance_num_key])
instances_iter = iter(instance_uuids)
for host_key in host_keys:
num_insts_on_this_host = num_insts_on_host.get(host_key, 0)
for i in xrange(num_insts_on_this_host):
host_instance_combinations.append(
(host_key_map[host_key], instances_iter.next()))
else:
LOG.warn(_LW("Pulp solver didnot find optimal solution! "
"reason: %s"), pulp.LpStatus[prob.status])
host_instance_combinations = []
return host_instance_combinations
#prob+= t_i_p*i + t_i_o + t_i_l + 1 <= t_j_p*j + t_j_o + bigM*helpBinary, "(%s<%s),k_i=%d,k_j=%d" % (t_i_name,t_j_name,i,j)
#prob+= t_j_p*j + t_j_o + t_j_l + 1 <= t_i_p*j + t_i_o + bigM*(1-helpBinary), "(%s<%s),k_i=%d,k_j=%d" % (t_j_name,t_i_name,i,j)
prob+= t_i_p*i + t_i_o + t_i_l <= t_j_p*j + t_j_o + bigM*(1-helpBinaryForCollisionFree)
prob+= t_j_p*j + t_j_o + t_j_l <= t_i_p*j + t_i_o + bigM*(helpBinaryForCollisionFree)
print "##################"
helpBinaryForSporadicOptimization=LpVariable("help_for_sporadic("+t_i_name+","+t_j_name+")",0,1,LpInteger)
absoluteValueHelpU=LpVariable("U(%s,%s)" % (t_i_name,t_j_name),0,bigM,LpInteger)
prob+=t_i_o-t_j_o+bigM*helpBinaryForSporadicOptimization >= absoluteValueHelpU
prob+=t_j_o-t_i_o+bigM*(1-helpBinaryForSporadicOptimization) >= absoluteValueHelpU
prob+=t_i_o-t_j_o <= absoluteValueHelpU
prob+=t_j_o-t_i_o <= absoluteValueHelpU
# get all U help variables
uObjectives=[ v for k,v in prob.variablesDict().items() if 'U' in k]
prob+=lpSum(uObjectives)
print prob.writeLP("mori.lp")
prob.solve()
print("Status:", LpStatus[prob.status])
for v in prob.variables():
print(v.name, "=", v.varValue)
print "**********************************************************************************************"
prob.solve(GLPK())
for v in prob.variables():
print(v.name, "=", v.varValue)
print("Status:", LpStatus[prob.status])
def host_solve(self, hosts, instance_uuids, request_spec,
filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties.
"""
host_instance_tuples_list = []
print filter_properties['instance_type']['memory_mb']
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
#Setting a unset uuid string for each instance.
instance_uuids = ['unset_uuid' + str(i)
for i in xrange(num_instances)]
num_hosts = len(hosts)
LOG.debug(_("All Hosts: %s") % [h.host for h in hosts])
for host in hosts:
LOG.debug(_("Host state: %s") % host)
# Create dictionaries mapping host/instance IDs to hosts/instances.
host_ids = ['Host' + str(i) for i in range(num_hosts)]
host_id_dict = dict(zip(host_ids, hosts))
instance_ids = ['Instance' + str(i) for i in range(num_instances)]
instance_id_dict = dict(zip(instance_ids, instance_uuids))
# Create the 'prob' variable to contain the problem data.
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
# Create the 'variables' matrix to contain the referenced variables.
variables = [[pulp.LpVariable("IA" + "_Host" + str(i) + "_Instance" +
str(j), 0, 1, constants.LpInteger) for j in
range(num_instances)] for i in range(num_hosts)]
# Get costs and constraints and formulate the linear problem.
self.cost_objects = [cost() for cost in self.cost_classes]
self.constraint_objects = [constraint(variables, hosts,
instance_uuids, request_spec, filter_properties)
for constraint in self.constraint_classes]
costs = [[0 for j in range(num_instances)] for i in range(num_hosts)]
for cost_object in self.cost_objects:
cost = cost_object.get_cost_matrix(hosts, instance_uuids,
request_spec, filter_properties)
cost = cost_object.normalize_cost_matrix(cost, 0.0, 1.0)
weight = float(self.cost_weights[cost_object.__class__.__name__])
costs = [[costs[i][j] + weight * cost[i][j]
for j in range(num_instances)] for i in range(num_hosts)]
prob += (pulp.lpSum([costs[i][j] * variables[i][j]
for i in range(num_hosts) for j in range(num_instances)]),
"Sum_of_Host_Instance_Scheduling_Costs")
for constraint_object in self.constraint_objects:
coefficient_vectors = constraint_object.get_coefficient_vectors(
variables, hosts, instance_uuids,
request_spec, filter_properties)
variable_vectors = constraint_object.get_variable_vectors(
variables, hosts, instance_uuids,
request_spec, filter_properties)
operations = constraint_object.get_operations(
variables, hosts, instance_uuids,
request_spec, filter_properties)
for i in range(len(operations)):
operation = operations[i]
len_vector = len(variable_vectors[i])
prob += (operation(pulp.lpSum([coefficient_vectors[i][j]
* variable_vectors[i][j] for j in range(len_vector)])),
"Costraint_Name_%s" % constraint_object.__class__.__name__
+ "_No._%s" % i)
prob.writeLP('test.lp')
if filter_properties['instance_type']['constraint'] == "soft_affinity":
prob = self.update_with_soft_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances)
elif filter_properties['instance_type']['constraint'] == "strict_affinity":
prob = self.update_with_strict_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances)
elif filter_properties['instance_type']['constraint'] == "strict_antiaffinity":
prob = self.update_with_strict_anti_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances)
elif filter_properties['instance_type']['constraint'] == "soft_antiaffinity":
prob = self.update_with_soft_anti_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances)
else:
temp = filter_properties['instance_type']['constraint']
temp = temp.split("_")
prob = self.update_with_host_count_constraints_and_objective(variables,prob,num_hosts,num_instances,temp[3])
print prob
# The problem is solved using PULP's choice of Solver.
prob.solve()
# Create host-instance tuples from the solutions.
if pulp.LpStatus[prob.status] == 'Optimal':
for v in prob.variables():
if v.name.startswith('IA'):
(host_id, instance_id) = v.name.lstrip('IA').lstrip(
'_').split('_')
if v.varValue == 1.0:
host_instance_tuples_list.append(
(host_id_dict[host_id],
instance_id_dict[instance_id]))
return host_instance_tuples_list
def distribute(self, fairness=True, output=None):
"""
This method is responsible for actually solving the linear programming problem.
It uses the data in the instance variables.
The optional parameter **fairness** indicates if the solution should minimize individual effort. By default the solution will enforce this condition.
An error will be raised in case the data has inconsistencies.
"""
# Validate the problem variables
self._validate()
# State problem
problem = pulp.LpProblem("Fair Distribution Problem", pulp.LpMinimize)
# Prepare variables
targets_objects = {}
for (t, target) in enumerate(self._targets):
for (o, object) in enumerate(self._objects):
variable = pulp.LpVariable(
'x' + str(t) + str(o), lowBound=0, cat='Binary')
position = {'target': t, 'object': o}
targets_objects['x' + str(t) + str(o)] = (variable, position)
# Generate linear expression for self._weights (Summation #1)
weights = [(variable, self._weights[weight_position['target']][weight_position['object']])
for (variable, weight_position) in targets_objects.values()]
weight_expression = pulp.LpAffineExpression(weights)
# Generate linear expression for effort distribution (Summation #2)
weight_diff_vars = []
if fairness:
total_targets = len(self._targets)
for (t, target) in enumerate(self._targets):
weight_diff_aux_variable = pulp.LpVariable(
'weight_diff_'+str(t), lowBound=0)
weight_diff_vars.append(weight_diff_aux_variable)
negative_effort_diff_weights = []
positive_effort_diff_weights = []
negative_factor = -1 * total_targets
positive_factor = 1 * total_targets
for (o, object) in enumerate(self._objects):
id = 'x' + str(t) + str(o)
negative_effort_diff_weights.append(
(targets_objects[id][0], negative_factor * self._weights[t][o]))
positive_effort_diff_weights.append(
(targets_objects[id][0], positive_factor * self._weights[t][o]))
negative_effort_diff = pulp.LpAffineExpression(
negative_effort_diff_weights) + weight_expression
positive_effort_diff = pulp.LpAffineExpression(
positive_effort_diff_weights) - weight_expression
problem += negative_effort_diff <= weight_diff_aux_variable, 'abs negative effort diff ' + \
str(t)
problem += positive_effort_diff <= weight_diff_aux_variable, 'abs positive effort diff ' + \
str(t)
# Constraints - Each task must be done
for (o, object) in enumerate(self._objects):
constraints = []
for (t, target) in enumerate(self._targets):
constraints.append(targets_objects['x' + str(t) + str(o)][0])
problem += pulp.lpSum(constraints) == 1, 'Task ' + \
str(o) + ' must be done'
# Set objective function
problem += weight_expression + \
pulp.lpSum(weight_diff_vars), "obj"
if output:
problem.writeLP(output)
problem.solve()
# Build output
data = {}
for v in filter(lambda x: x.varValue > 0, problem.variables()):
if v.name not in targets_objects:
continue
position = targets_objects[v.name][1]
target = self._targets[position['target']]
object = self._objects[position['object']]
if target not in data:
data[target] = []
data[target].append(object)
return data
def solve(self, hosts, filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties.
"""
host_instance_combinations = []
num_instances = filter_properties['num_instances']
num_hosts = len(hosts)
instance_uuids = filter_properties.get('instance_uuids') or [
'(unknown_uuid)' + str(i) for i in xrange(num_instances)]
LOG.debug(_("All Hosts: %s") % [h.host for h in hosts])
for host in hosts:
LOG.debug(_("Host state: %s") % host)
# Create dictionaries mapping temporary host/instance keys to
# hosts/instance_uuids. These temorary keys are to be used in the
# solving process since we need a convention of lp variable names.
host_keys = ['Host' + str(i) for i in xrange(num_hosts)]
host_key_map = dict(zip(host_keys, hosts))
instance_keys = ['InstanceNum' + str(i) for i in xrange(num_instances)]
instance_key_map = dict(
zip(instance_keys, xrange(1, num_instances + 1)))
# this is currently hard-coded and should match variable names
host_instance_matrix_idx_map = {}
for i in xrange(len(host_keys)):
for j in xrange(len(instance_keys)):
var_name = 'HI_' + host_keys[i] + '_' + instance_keys[j]
host_instance_matrix_idx_map[var_name] = (i, j)
# Create the 'variables' to contain the referenced variables.
self.variables.populate_variables(host_keys, instance_keys)
# Create the 'prob' variable to contain the problem data.
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
# Get costs and constraints and formulate the linear problem.
# Add costs.
cost_objects = [cost() for cost in self.cost_classes]
cost_coeff_matrix = [[0 for j in xrange(num_instances)]
for i in xrange(num_hosts)]
for cost_object in cost_objects:
var_list, coeff_list = cost_object.get_components(
self.variables, hosts, filter_properties)
for i in xrange(len(var_list)):
var = var_list[i]
coeff = coeff_list[i]
hidx, iidx = host_instance_matrix_idx_map[var.name]
cost_coeff_matrix[hidx][iidx] += (
coeff * cost_object.cost_multiplier())
cost_coeff_matrix = self._calculate_host_instance_cost_matrix(
cost_coeff_matrix)
cost_coeff_array = []
for var in var_list:
hidx, iidx = host_instance_matrix_idx_map[var.name]
cost_coeff_array.append(cost_coeff_matrix[hidx][iidx])
cost_variables = var_list
cost_coefficients = cost_coeff_array
if cost_variables:
prob += (pulp.lpSum([cost_coefficients[i] * cost_variables[i]
for i in xrange(len(cost_variables))]), "Sum_Costs")
# Add constraints.
constraint_objects = [constraint()
for constraint in self.constraint_classes]
for constraint_object in constraint_objects:
vars_list, coeffs_list, consts_list, ops_list = (
constraint_object.get_components(self.variables, hosts,
filter_properties))
LOG.debug(_("coeffs of %(name)s is: %(value)s") %
{"name": constraint_object.__class__.__name__,
"value": coeffs_list})
for i in xrange(len(ops_list)):
operation = self._get_operation(ops_list[i])
prob += (
operation(pulp.lpSum([coeffs_list[i][j] *
vars_list[i][j] for j in xrange(len(vars_list[i]))]),
consts_list[i]), "Costraint_Name_%s" %
constraint_object.__class__.__name__ + "_No._%s" % i)
# The problem is solved using PULP's choice of Solver.
prob.solve(pulp_solver_classes.PULP_CBC_CMD(
maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds))
# Create host-instance tuples from the solutions.
if pulp.LpStatus[prob.status] == 'Optimal':
num_insts_on_host = {}
for v in prob.variables():
if v.name.startswith('HI'):
(host_key, instance_key) = v.name.lstrip('HI').lstrip(
'_').split('_')
if v.varValue == 1:
num_insts_on_host[host_key] = (
instance_key_map[instance_key])
instances_iter = iter(instance_uuids)
for host_key in host_keys:
num_insts_on_this_host = num_insts_on_host.get(host_key, 0)
for i in xrange(num_insts_on_this_host):
host_instance_combinations.append(
(host_key_map[host_key], instances_iter.next()))
elif pulp.LpStatus[prob.status] == 'Infeasible':
LOG.warn(_("Pulp solver didnot find optimal solution! reason: %s")
% pulp.LpStatus[prob.status])
host_instance_combinations = []
else:
LOG.warn(_("Pulp solver didnot find optimal solution! reason: %s")
% pulp.LpStatus[prob.status])
raise exception.SolverFailed(reason=pulp.LpStatus[prob.status])
return host_instance_combinations
def createConstraints(problem, variable):
problem += lpSum(choiceVariables) <= 1, "Distribution"
for tower, towerName in towerNames.items():
problem += createLpSum(tower.getChoiceClass(), choiceNames, choiceVariables) >= \
tower.getHeight() + variable * \
tower.getSpeed(), towerName
def solve(self, hosts, filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties.
"""
host_instance_combinations = []
num_instances = filter_properties['num_instances']
num_hosts = len(hosts)
instance_uuids = filter_properties.get('instance_uuids') or [
'(unknown_uuid)' + str(i) for i in xrange(num_instances)
]
filter_properties.setdefault('solver_cache', {})
filter_properties['solver_cache'].update({
'cost_matrix': [],
'constraint_matrix': []
})
cost_matrix = self._get_cost_matrix(hosts, filter_properties)
cost_matrix = self._adjust_cost_matrix(cost_matrix)
constraint_matrix = self._get_constraint_matrix(
hosts, filter_properties)
# Create dictionaries mapping temporary host/instance keys to
# hosts/instance_uuids. These temorary keys are to be used in the
# solving process since we need a convention of lp variable names.
host_keys = ['Host' + str(i) for i in xrange(num_hosts)]
host_key_map = dict(zip(host_keys, hosts))
instance_num_keys = [
'InstanceNum' + str(i) for i in xrange(num_instances + 1)
]
instance_num_key_map = dict(
zip(instance_num_keys, xrange(num_instances + 1)))
# create the pulp variables
variable_matrix = [[
pulp.LpVariable('HI_' + host_key + '_' + instance_num_key, 0, 1,
constants.LpInteger)
for instance_num_key in instance_num_keys
] for host_key in host_keys]
# create the 'prob' variable to contain the problem data.
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
# add cost function to pulp solver
cost_variables = [
variable_matrix[i][j] for i in xrange(num_hosts)
for j in xrange(num_instances + 1)
]
cost_coefficients = [
cost_matrix[i][j] for i in xrange(num_hosts)
for j in xrange(num_instances + 1)
]
prob += (pulp.lpSum([
cost_coefficients[i] * cost_variables[i]
for i in xrange(len(cost_variables))
]), "Sum_Costs")
# add constraints to pulp solver
for i in xrange(num_hosts):
for j in xrange(num_instances + 1):
if constraint_matrix[i][j] is False:
prob += (variable_matrix[i][j] == 0,
"Cons_Host_%s" % i + "_NumInst_%s" % j)
# add additional constraints to ensure the problem is valid
# (1) non-trivial solution: number of all instances == that requested
prob += (pulp.lpSum([
variable_matrix[i][j] * j for i in xrange(num_hosts)
for j in xrange(num_instances + 1)
]) == num_instances, "NonTrivialCons")
# (2) valid solution: each host is assigned 1 num-instances value
for i in xrange(num_hosts):
prob += (pulp.lpSum([
variable_matrix[i][j] for j in xrange(num_instances + 1)
]) == 1, "ValidCons_Host_%s" % i)
# The problem is solved using PULP's choice of Solver.
prob.solve(
pulp_solver_classes.PULP_CBC_CMD(
maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds))
# Create host-instance tuples from the solutions.
if pulp.LpStatus[prob.status] == 'Optimal':
num_insts_on_host = {}
for v in prob.variables():
if v.name.startswith('HI'):
(host_key, instance_num_key
) = v.name.lstrip('HI').lstrip('_').split('_')
if v.varValue == 1:
num_insts_on_host[host_key] = (
instance_num_key_map[instance_num_key])
instances_iter = iter(instance_uuids)
for host_key in host_keys:
num_insts_on_this_host = num_insts_on_host.get(host_key, 0)
for i in xrange(num_insts_on_this_host):
host_instance_combinations.append(
(host_key_map[host_key], instances_iter.next()))
else:
LOG.warn(
_LW("Pulp solver didnot find optimal solution! "
"reason: %s"), pulp.LpStatus[prob.status])
host_instance_combinations = []
return host_instance_combinations
def imposta_vincoli_facoltativi(self, model, dati, str_aux, vincoli):
skd=str_aux.get_schedulazione()
if vincoli["chkSessioneUnica"] == 1:
# Per ogni giorno, ogni corso ed ogni aula il numero di slot massimo consentito è pari alla durata delle sessioni
# ciò evita che ci possano essere due sessioni nello stesso giorno
for c in dati.get_corsi():
for m in dati.get_moduli():
for g in dati.get_giorni():
model += lpSum(skd[(c,m,a,g,s)] for s in dati.get_slot() for a in dati.get_aule()) <= m.get_dur_sessioni()
if vincoli["chkSessioniConsecutive"] == 1:
# Vincoli di consecutività delle sessioni
for c in dati.get_corsi():
for m in dati.get_moduli():
dur_sessione = m.get_dur_sessioni()
if dur_sessione == 2:
for a in dati.get_aule():
for g in dati.get_giorni():
for s in (0,4):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
for s in (1,2,5,6):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
for s in (3,7):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
elif dur_sessione == 3:
for a in dati.get_aule():
for g in dati.get_giorni():
for s in (0,4):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0
for s in (1,5):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0
for s in (2,6):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0
for s in (3,7):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0
elif dur_sessione == 4:
for a in dati.get_aule():
for g in dati.get_giorni():
for s in (0,4):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+3])] <= 0
for s in (1,5):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0
for s in (2,6):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0
for s in (3,7):
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0
model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-3])] <= 0
# Vincolo che fissa il numero massimo di slot per giorno per un anno di corso
if vincoli["chkMaxOre"] == 1:
limsup = vincoli["selMaxOre"]
for c in dati.get_corsi():
for g in dati.get_giorni():
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 1 for s in dati.get_slot() for a in dati.get_aule())<=limsup
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 2 for s in dati.get_slot() for a in dati.get_aule())<=limsup
model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 3 for s in dati.get_slot() for a in dati.get_aule())<=limsup
def createConstraints(problem, variable):
problem += lpSum(choiceVariables) <= 1, "Distribution"
for tower, towerName in towerNames.items():
problem += createLpSum(tower.getChoiceClass(), choiceNames, choiceVariables) >= \
tower.getHeight() + variable * \
tower.getSpeed(), towerName
def distribute_factors(
agents: Dict[str, AgentDef],
cg: ComputationGraph,
footprints: Dict[str, float],
mapping: Dict[str, List[str]],
msg_load: Callable[[str, str], float],
) -> Dict[str, List[str]]:
"""
Optimal distribution of factors on agents.
Parameters
----------
cg: computations graph
agents: dict
a dict {agent_name : AgentDef} containing all available agents
Returns
-------
a dict { agent_name: list of factor names}
"""
pb = LpProblem("ilp_factors", LpMinimize)
# build the inverse mapping var -> agt
inverse_mapping = {} # type: Dict[str, str]
for a in mapping:
inverse_mapping[mapping[a][0]] = a
# One binary variable xij for each (variable, agent) couple
factor_names = [n.name for n in cg.nodes if isinstance(n, FactorComputationNode)]
xs = LpVariable.dict("x", (factor_names, agents), cat=LpBinary)
logger.debug("Binary variables for factor distribution : %s", xs)
# Hard constraints: respect agent's capacity
for a in agents:
# Footprint of the variable this agent is already hosting:
v_footprint = footprints[mapping[a][0]]
pb += (
lpSum([footprints[fn] * xs[fn, a] for fn in factor_names])
<= (agents[a].capacity - v_footprint),
"Agent {} capacity".format(a),
)
# Hard constraints: all computations must be hosted.
for c in factor_names:
pb += lpSum([xs[c, a] for a in agents]) == 1, "Factor {} hosted".format(c)
# 1st objective : minimize communication costs:
comm = LpAffineExpression()
for (fn, an_f) in xs:
for vn in cg.neighbors(fn):
an_v = inverse_mapping[vn] # agt hosting neighbor var vn
comm += agents[an_f].route(an_v) * msg_load(vn, fn) * xs[(fn, an_f)]
# 2st objective : minimize hosting costs
hosting = lpSum([agents[a].hosting_cost(c) * xs[(c, a)] for c, a in xs])
# agregate the two objectives using RATIO_HOST_COMM
pb += lpSum([RATIO_HOST_COMM * comm, (1 - RATIO_HOST_COMM) * hosting])
# solve using GLPK and convert to mapping { agt_name : [factors names]}
status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False, options=["--pcost"]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException(
"No possible optimal distribution for factors"
)
logger.debug("GLPK cost : %s", value(pb.objective))
mapping = {} # type: Dict[str, List[str]]
for k in agents:
agt_computations = [i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1]
# print(k, ' -> ', agt_computations)
mapping[k] = agt_computations
logger.debug("Factors distribution : %s ", mapping)
return mapping
def host_solve(self, hosts, instance_uuids, request_spec,
filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties.
"""
host_instance_tuples_list = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
#Setting a unset uuid string for each instance.
instance_uuids = [
'unset_uuid' + str(i) for i in xrange(num_instances)
]
num_hosts = len(hosts)
LOG.debug(_("All Hosts: %s") % [h.host for h in hosts])
for host in hosts:
LOG.debug(_("Host state: %s") % host)
# Create dictionaries mapping host/instance IDs to hosts/instances.
host_ids = ['Host' + str(i) for i in range(num_hosts)]
host_id_dict = dict(zip(host_ids, hosts))
instance_ids = ['Instance' + str(i) for i in range(num_instances)]
instance_id_dict = dict(zip(instance_ids, instance_uuids))
# Create the 'prob' variable to contain the problem data.
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
# Create the 'variables' matrix to contain the referenced variables.
variables = [[
pulp.LpVariable("IA" + "_Host" + str(i) + "_Instance" + str(j), 0,
1, constants.LpInteger)
for j in range(num_instances)
] for i in range(num_hosts)]
# Get costs and constraints and formulate the linear problem.
self.cost_objects = [cost() for cost in self.cost_classes]
self.constraint_objects = [
constraint(variables, hosts, instance_uuids, request_spec,
filter_properties)
for constraint in self.constraint_classes
]
costs = [[0 for j in range(num_instances)] for i in range(num_hosts)]
for cost_object in self.cost_objects:
cost = cost_object.get_cost_matrix(hosts, instance_uuids,
request_spec, filter_properties)
cost = cost_object.normalize_cost_matrix(cost, 0.0, 1.0)
weight = float(self.cost_weights[cost_object.__class__.__name__])
costs = [[
costs[i][j] + weight * cost[i][j] for j in range(num_instances)
] for i in range(num_hosts)]
prob += (pulp.lpSum([
costs[i][j] * variables[i][j] for i in range(num_hosts)
for j in range(num_instances)
]), "Sum_of_Host_Instance_Scheduling_Costs")
for constraint_object in self.constraint_objects:
coefficient_vectors = constraint_object.get_coefficient_vectors(
variables, hosts, instance_uuids, request_spec,
filter_properties)
variable_vectors = constraint_object.get_variable_vectors(
variables, hosts, instance_uuids, request_spec,
filter_properties)
operations = constraint_object.get_operations(
variables, hosts, instance_uuids, request_spec,
filter_properties)
for i in range(len(operations)):
operation = operations[i]
len_vector = len(variable_vectors[i])
prob += (operation(
pulp.lpSum([
coefficient_vectors[i][j] * variable_vectors[i][j]
for j in range(len_vector)
])), "Costraint_Name_%s" %
constraint_object.__class__.__name__ + "_No._%s" % i)
# The problem is solved using PULP's choice of Solver.
prob.solve()
# Create host-instance tuples from the solutions.
if pulp.LpStatus[prob.status] == 'Optimal':
for v in prob.variables():
if v.name.startswith('IA'):
(host_id,
instance_id) = v.name.lstrip('IA').lstrip('_').split('_')
if v.varValue == 1.0:
host_instance_tuples_list.append(
(host_id_dict[host_id],
instance_id_dict[instance_id]))
return host_instance_tuples_list
def lp_model(cg: ComputationGraph,
agentsdef: Iterable[AgentDef],
footprint: Callable[[str], float],
capacity: Callable[[str], float],
route: Callable[[str, str], float],
msg_load: Callable[[str, str], float],
hosting_cost: Callable[[str, str], float]):
comp_names = [n.name for n in cg.nodes]
agt_names = [a.name for a in agentsdef]
pb = LpProblem('ilp_compref', LpMinimize)
# One binary variable xij for each (variable, agent) couple
xs = LpVariable.dict('x', (comp_names, agt_names), cat=LpBinary)
# One binary variable for computations c1 and c2, and agent a1 and a2
betas = {}
count = 0
for a1, a2 in combinations(agt_names, 2):
# Only create variables for couple c1, c2 if there is an edge in the
# graph between these two computations.
for l in cg.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
count += 2
b = LpVariable('b_{}_{}_{}_{}'.format(c1, a1, c2, a2),
cat=LpBinary)
betas[(c1, a1, c2, a2)] = b
pb += b <= xs[(c1, a1)]
pb += b <= xs[(c2, a2)]
pb += b >= xs[(c2, a2)] + xs[(c1, a1)] - 1
b = LpVariable('b_{}_{}_{}_{}'.format(c1, a2, c2, a1),
cat=LpBinary)
betas[(c1, a2, c2, a1)] = b
pb += b <= xs[(c2, a1)]
pb += b <= xs[(c1, a2)]
pb += b >= xs[(c1, a2)] + xs[(c2, a1)] - 1
# Set objective: communication + hosting_cost
pb += _objective(xs, betas, route, msg_load, hosting_cost), \
'Communication costs and prefs'
# Adding constraints:
# Constraints: Memory capacity for all agents.
for a in agt_names:
pb += lpSum([footprint(i) * xs[i, a] for i in comp_names])\
<= capacity(a), \
'Agent {} capacity'.format(a)
# Constraints: all computations must be hosted.
for c in comp_names:
pb += lpSum([xs[c, a] for a in agt_names]) == 1, \
'Computation {} hosted'.format(c)
# solve using GLPK
status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False,
options=['--pcost']))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
logger.debug('GLPK cost : %s', value(pb.objective))
# print('BETAS:')
# for c1, a1, c2, a2 in betas:
# print(' ', c1, a1, c2, a2, value(betas[(c1, a1, c2, a2)]))
#
# print('XS:')
# for c, a in xs:
# print(' ', c, a, value(xs[(c, a)]))
mapping = {}
for k in agt_names:
agt_computations = [i for i, ka in xs
if ka == k and value(xs[(i, ka)]) == 1]
# print(k, ' -> ', agt_computations)
mapping[k] = agt_computations
return mapping
def createLpSum(choiceClass, choiceNames, choiceVariables):
return lpSum(choiceVariables[choiceNames[choice]] for choice in choiceClass)
def createLpSum(choiceClass, choiceNames, choiceVariables):
return lpSum(choiceVariables[choiceNames[choice]]
for choice in choiceClass)
def host_solve(self, hosts, instance_uuids, request_spec,
filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties
"""
host_instance_tuples_list = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
instance_uuids = [
'unset_uuid%s' % i for i in xrange(num_instances)
]
num_hosts = len(hosts)
host_ids = ['Host%s' % i for i in range(num_hosts)]
LOG.debug(_("All Hosts: %s") % [h.host for h in hosts])
for host in hosts:
LOG.debug(_("Host state: %s") % host)
host_id_dict = dict(zip(host_ids, hosts))
instances = ['Instance%s' % i for i in range(num_instances)]
instance_id_dict = dict(zip(instances, instance_uuids))
# supply is a dictionary for the number of units of
# resource for each Host.
# Currently using only the disk_mb and memory_mb
# as the two resources to satisfy. Need to eventually be able to
# plug-in different resources. An example supply dictionary:
# supply = {"Host1": [1000, 1000],
# "Host2": [4000, 1000]}
supply = dict((host_ids[i], [
self._get_usable_disk_mb(hosts[i]),
self._get_usable_memory_mb(hosts[i]),
]) for i in range(len(host_ids)))
number_of_resource_types_per_host = 2
required_disk_mb = self._get_required_disk_mb(filter_properties)
required_memory_mb = self._get_required_memory_mb(filter_properties)
# demand is a dictionary for the number of
# units of resource required for each Instance.
# An example demand dictionary:
# demand = {"Instance0":[200, 300],
# "Instance1":[900, 100],
# "Instance2":[1800, 200],
# "Instance3":[200, 300],
# "Instance4":[700, 800], }
# However for the current scenario, all instances to be scheduled
# per request have the same requirements. Need to eventually
# to support requests to specify different instance requirements
demand = dict((instances[i], [
required_disk_mb,
required_memory_mb,
]) for i in range(num_instances))
# Creates a list of costs of each Host-Instance assignment
# Currently just like the nova.scheduler.weights.ram.RAMWeigher,
# using host_state.free_ram_mb * ram_weight_multiplier
# as the cost. A negative ram_weight_multiplier means to stack,
# vs spread.
# An example costs list:
# costs = [ # Instances
# # 1 2 3 4 5
# [2, 4, 5, 2, 1], # A Hosts
# [3, 1, 3, 2, 3] # B
# ]
# Multiplying -1 as we want to use the same behavior of
# ram_weight_multiplier as used by ram weigher.
costs = [[
-1 * host.free_ram_mb * CONF.ram_weight_multiplier
for i in range(num_instances)
] for host in hosts]
costs = pulp.makeDict([host_ids, instances], costs, 0)
# The PULP LP problem variable used to add all the problem data
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
all_host_instance_tuples = [(w, b) for w in host_ids
for b in instances]
vars = pulp.LpVariable.dicts("IA", (host_ids, instances), 0, 1,
constants.LpInteger)
# The objective function is added to 'prob' first
prob += (pulp.lpSum([
vars[w][b] * costs[w][b] for (w, b) in all_host_instance_tuples
]), "Sum_of_Host_Instance_Scheduling_Costs")
# The supply maximum constraints are added to
# prob for each supply node (Host)
for w in host_ids:
for i in range(number_of_resource_types_per_host):
prob += (pulp.lpSum(
[vars[w][b] * demand[b][i]
for b in instances]) <= supply[w][i],
"Sum_of_Resource_%s" % i + "_provided_by_Host_%s" % w)
# The number of Hosts required per Instance, in this case it is only 1
for b in instances:
prob += (pulp.lpSum([vars[w][b] for w in host_ids]) == 1,
"Sum_of_Instance_Assignment%s" % b)
# The demand minimum constraints are added to prob for
# each demand node (Instance)
for b in instances:
for j in range(number_of_resource_types_per_host):
prob += (
pulp.lpSum([vars[w][b] * demand[b][j]
for w in host_ids]) >= demand[b][j],
"Sum_of_Resource_%s" % j + "_required_by_Instance_%s" % b)
# The problem is solved using PuLP's choice of Solver
prob.solve()
if pulp.LpStatus[prob.status] == 'Optimal':
for v in prob.variables():
if v.name.startswith('IA'):
(host_id,
instance_id) = v.name.lstrip('IA').lstrip('_').split('_')
if v.varValue == 1.0:
host_instance_tuples_list.append(
(host_id_dict[host_id],
instance_id_dict[instance_id]))
return host_instance_tuples_list
def host_solve(self, hosts, instance_uuids, request_spec,
filter_properties):
"""This method returns a list of tuples - (host, instance_uuid)
that are returned by the solver. Here the assumption is that
all instance_uuids have the same requirement as specified in
filter_properties
"""
host_instance_tuples_list = []
if instance_uuids:
num_instances = len(instance_uuids)
else:
num_instances = request_spec.get('num_instances', 1)
instance_uuids = ['unset_uuid%s' % i
for i in xrange(num_instances)]
num_hosts = len(hosts)
host_ids = ['Host%s' % i for i in range(num_hosts)]
LOG.debug(_("All Hosts: %s") % [h.host for h in hosts])
for host in hosts:
LOG.debug(_("Host state: %s") % host)
host_id_dict = dict(zip(host_ids, hosts))
instances = ['Instance%s' % i for i in range(num_instances)]
instance_id_dict = dict(zip(instances, instance_uuids))
# supply is a dictionary for the number of units of
# resource for each Host.
# Currently using only the disk_mb and memory_mb
# as the two resources to satisfy. Need to eventually be able to
# plug-in different resources. An example supply dictionary:
# supply = {"Host1": [1000, 1000],
# "Host2": [4000, 1000]}
supply = dict((host_ids[i],
[self._get_usable_disk_mb(hosts[i]),
self._get_usable_memory_mb(hosts[i]), ])
for i in range(len(host_ids)))
number_of_resource_types_per_host = 2
required_disk_mb = self._get_required_disk_mb(filter_properties)
required_memory_mb = self._get_required_memory_mb(filter_properties)
# demand is a dictionary for the number of
# units of resource required for each Instance.
# An example demand dictionary:
# demand = {"Instance0":[200, 300],
# "Instance1":[900, 100],
# "Instance2":[1800, 200],
# "Instance3":[200, 300],
# "Instance4":[700, 800], }
# However for the current scenario, all instances to be scheduled
# per request have the same requirements. Need to eventually
# to support requests to specify different instance requirements
demand = dict((instances[i],
[required_disk_mb, required_memory_mb, ])
for i in range(num_instances))
# Creates a list of costs of each Host-Instance assignment
# Currently just like the nova.scheduler.weights.ram.RAMWeigher,
# using host_state.free_ram_mb * ram_weight_multiplier
# as the cost. A negative ram_weight_multiplier means to stack,
# vs spread.
# An example costs list:
# costs = [ # Instances
# # 1 2 3 4 5
# [2, 4, 5, 2, 1], # A Hosts
# [3, 1, 3, 2, 3] # B
# ]
# Multiplying -1 as we want to use the same behavior of
# ram_weight_multiplier as used by ram weigher.
costs = [[-1 * host.free_ram_mb *
CONF.ram_weight_multiplier
for i in range(num_instances)]
for host in hosts]
costs = pulp.makeDict([host_ids, instances], costs, 0)
# The PULP LP problem variable used to add all the problem data
prob = pulp.LpProblem("Host Instance Scheduler Problem",
constants.LpMinimize)
all_host_instance_tuples = [(w, b)
for w in host_ids
for b in instances]
vars = pulp.LpVariable.dicts("IA", (host_ids, instances),
0, 1, constants.LpInteger)
# The objective function is added to 'prob' first
prob += (pulp.lpSum([vars[w][b] * costs[w][b]
for (w, b) in all_host_instance_tuples]),
"Sum_of_Host_Instance_Scheduling_Costs")
# The supply maximum constraints are added to
# prob for each supply node (Host)
for w in host_ids:
for i in range(number_of_resource_types_per_host):
prob += (pulp.lpSum([vars[w][b] * demand[b][i]
for b in instances])
<= supply[w][i],
"Sum_of_Resource_%s" % i + "_provided_by_Host_%s" % w)
# The number of Hosts required per Instance, in this case it is only 1
for b in instances:
prob += (pulp.lpSum([vars[w][b] for w in host_ids])
== 1, "Sum_of_Instance_Assignment%s" % b)
# The demand minimum constraints are added to prob for
# each demand node (Instance)
for b in instances:
for j in range(number_of_resource_types_per_host):
prob += (pulp.lpSum([vars[w][b] * demand[b][j]
for w in host_ids])
>= demand[b][j],
"Sum_of_Resource_%s" % j + "_required_by_Instance_%s" % b)
# The problem is solved using PuLP's choice of Solver
prob.solve()
print prob
if pulp.LpStatus[prob.status] == 'Optimal':
for v in prob.variables():
if v.name.startswith('IA'):
(host_id, instance_id) = v.name.lstrip('IA').lstrip(
'_').split('_')
if v.varValue == 1.0:
host_instance_tuples_list.append(
(host_id_dict[host_id],
instance_id_dict[instance_id]))
return host_instance_tuples_list
def get_optimal_ec2(cpu_usage, mem_usage, optimal=True, debug=False):
"""
Calculates the optimal allocation of resources over a certain time span (with monthly granularity).
Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function.
Returns allocation of reserved instances and a total price for running the allocation on AWS.
"""
assert(len(cpu_usage) == len(mem_usage))
prob = LpProblem("The Simplified EC2 cost optimization", LpMinimize)
# variables
## 1h instances (both on-demand and reserved)
per_h_ondems = []
per_h_reserved = []
for p in range(len(cpu_usage)):
# ondemand
per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()]
# reserved
per_h_reserved += ["p %s reserved %s" %(p, i) for i in vms.keys()]
## nr of 1-year reserved instances
nr_of_1year_reserved = [ "res_1year %s" % i for i in vms.keys()]
nr_of_3year_reserved = [ "res_3year %s" % i for i in vms.keys()]
category = LpInteger if optimal else LpContinuous
vars = LpVariable.dicts("aws", per_h_ondems + per_h_reserved + nr_of_1year_reserved + nr_of_3year_reserved, \
lowBound = 0, upBound = None, cat = category)
# objective function
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_reserved]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[1]][1] for vm in nr_of_1year_reserved]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[1]][2] for vm in nr_of_3year_reserved]) \
, "Total cost of running the infrastructure consuming (CPU/RAM)/h"
# constraints
## demand constraints
for p in range(len(cpu_usage)):
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][4] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CPU demand, period %s" %p
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][5] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand. period %s" %p
## constraints on the reserved instances - cannot use more than we paid for
for i in per_h_reserved:
t = i.split(" ")[3]
prob += vars["res_1year %s" % t] + vars["res_3year %s" % t] >= vars[i], "Nr. of used reserved machines of type %s" %i
prob.solve()
if debug:
print "Status:", LpStatus[prob.status]
print "Total Cost of the solution = ", value(prob.objective)
res_instance = {}
for v in prob.variables():
if v.name.startswith("aws_res_") and v.varValue != 0.0:
res_instance[v.name] = v.varValue
if debug and v.varValue != 0.0:
print v.name, "=", v.varValue
return (res_instance, value(prob.objective))
