def findLottery(vote, classHeights, solverSettings): ''' Returns a Lottery satisfying all constraints specified by the classHeights parameter @type vote: vote.society.Vote @type classHeights: dict(vote.society.ChoiceClass, float) @type solverSettings: vote.solver.settings.SolverSettings @rtype: vote.society.Lottery @raise ValueError: If the constraints are not satisfiable ''' classNames = getUniqueNames(classHeights.keys(), prefix="Class ") choiceNames = getUniqueNames(vote.getChoices(), prefix="Choice ") problem = LpProblem("Lambda", LpMaximize) choiceVariables = LpVariable.dicts("p", choiceNames.values(), lowBound=0) problem += lpSum(choiceVariables) <= 1, "Distribution" for choiceClass, height in classHeights.items(): problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \ height, classNames[choiceClass] + " height" problem.setObjective(lpSum(choiceVariables.values())) checkPulpStatus(problem.solve(solverSettings.getSolver())) choiceValues = dict() for choice, choiceName in choiceNames.items(): choiceValues[choice.getObject()] = choiceVariables[choiceName].value() return Lottery(choiceValues, solverSettings)
def update_with_soft_affinity_constraints_and_objective(self,variables,prob, num_hosts, num_instances): #Adding column sum variables whose value is 1 if there is any instance on that host column_sum_var = [] for i in range(num_hosts): column_sum_var.append(pulp.LpVariable("Normalised_Column_Sum_Host_"+str(i), 0, 1, constants.LpInteger)) #Adding normalisation constraint for i in range(num_hosts): prob += pulp.lpSum([variables[i][j]] for j in range(num_instances)) <= num_instances*column_sum_var[i] prob += column_sum_var[i] <= pulp.lpSum([variables[i][j]] for j in range(num_instances)) z_variables =[] #Adding 'z' variables for i in range(num_hosts): for j in range(num_hosts): if i != j: z_variables.append(pulp.LpVariable("Z_variable_Col_"+str(i)+"Col_"+str(j), 0, 1, constants.LpInteger)) temp = 0 for i in range(num_hosts): for j in range(num_hosts): if i != j: prob += column_sum_var[i] + column_sum_var[j] <= z_variables[temp] + 1 prob += 2 * z_variables[temp]<=column_sum_var[i] + column_sum_var[j] #print str(temp) + " " + str(z_variables[temp]) temp = temp + 1 #Adding the objective prob+=z_variables[0] * 0 + z_variables[1] * 3 + z_variables[2] * 1 + z_variables[4] * 3 + z_variables[5] * 5 + z_variables[8] * 4 return prob
def _objective(xs, betas, route, msg_load, hosting_cost): # We want to minimize communication and hosting costs # Objective is the communication + hosting costs comm = LpAffineExpression() for c1, a1, c2, a2 in betas: comm += route(a1, a2) * msg_load(c1, c2) * betas[(c1, a1, c2, a2)] costs = lpSum([hosting_cost(a, c) * xs[(c, a)] for c, a in xs]) return lpSum([RATIO_HOST_COMM * comm , (1-RATIO_HOST_COMM) * costs])
def update_with_strict_affinity_constraints_and_objective(self,variables,prob, num_hosts, num_instances): #Adding column sum variables whose value is 1 if there is any instance on that host column_sum_var = [] for i in range(num_hosts): column_sum_var.append(pulp.LpVariable("Normalised_Column_Sum_Host_"+str(i), 0, 1, constants.LpInteger)) #Adding normalisation constraint for i in range(num_hosts): prob += pulp.lpSum([variables[i][j]] for j in range(num_instances)) <= num_instances*column_sum_var[i] prob += column_sum_var[i] <= pulp.lpSum([variables[i][j]] for j in range(num_instances)) prob += pulp.lpSum([column_sum_var[i]] for i in range(num_hosts)) == 1 return prob
def calcola_orario(self, model, dati, str_aux): skd = str_aux.get_schedulazione() # Funzione obiettivo model += lpSum(skd[(c,m,a,g,s)] for c in dati.get_corsi() for m in dati.get_moduli() for a in dati.get_aule() for g in dati.get_giorni() for s in dati.get_slot()) model.solve() return pl.LpStatus[model.status]
def createConstraints(problem, variable): problem += lpSum(choiceVariables) <= 1, "Distribution" for choiceClass, height in state.getCurrentClassHeights().items(): problem += createLpSum(choiceClass, choiceNames, choiceVariables) >= \ height, classNames[choiceClass] + " height" for agent in state.getActiveAgents(): problem += createLpSum(state.getCurrentAgentChoiceClass(agent), choiceNames, choiceVariables) >= \ state.getAgentHeight(agent) + variable * state.getAgentSpeed(agent), \ agentNames[agent] + " push"
def update_with_strict_affinity_constraints_and_objective( self, variables, prob, num_hosts, num_instances): #Adding column sum variables whose value is 1 if there is any instance on that host column_sum_var = [] for i in range(num_hosts): column_sum_var.append( pulp.LpVariable("Normalised_Column_Sum_Host_" + str(i), 0, 1, constants.LpInteger)) #Adding normalisation constraint for i in range(num_hosts): prob += pulp.lpSum([variables[i][j]] for j in range( num_instances)) <= num_instances * column_sum_var[i] prob += column_sum_var[i] <= pulp.lpSum( [variables[i][j]] for j in range(num_instances)) prob += pulp.lpSum([column_sum_var[i]] for i in range(num_hosts)) == 1 return prob
def update_with_soft_affinity_constraints_and_objective( self, variables, prob, num_hosts, num_instances): #Adding column sum variables whose value is 1 if there is any instance on that host column_sum_var = [] for i in range(num_hosts): column_sum_var.append( pulp.LpVariable("Normalised_Column_Sum_Host_" + str(i), 0, 1, constants.LpInteger)) #Adding normalisation constraint for i in range(num_hosts): prob += pulp.lpSum([variables[i][j]] for j in range( num_instances)) <= num_instances * column_sum_var[i] prob += column_sum_var[i] <= pulp.lpSum( [variables[i][j]] for j in range(num_instances)) z_variables = [] #Adding 'z' variables for i in range(num_hosts): for j in range(num_hosts): if i != j: z_variables.append( pulp.LpVariable( "Z_variable_Col_" + str(i) + "Col_" + str(j), 0, 1, constants.LpInteger)) temp = 0 for i in range(num_hosts): for j in range(num_hosts): if i != j: prob += column_sum_var[i] + column_sum_var[ j] <= z_variables[temp] + 1 prob += 2 * z_variables[temp] <= column_sum_var[ i] + column_sum_var[j] #print str(temp) + " " + str(z_variables[temp]) temp = temp + 1 #Adding the objective prob += z_variables[0] * 0 + z_variables[1] * 3 + z_variables[ 2] * 1 + z_variables[4] * 3 + z_variables[5] * 5 + z_variables[ 8] * 4 return prob
def imposta_vincoli_obbligatori(self, model, dati, str_aux): skd=str_aux.get_schedulazione() # Vincolo che evita il sovrapporsi di moduli insegnati dagli stessi docenti for d in str_aux.get_docenti(): for s in dati.get_slot(): for g in dati.get_giorni(): model += lpSum(skd[(c,m,a,g,s)]*str_aux.get_titolari_moduli()[(m,d)] for m in dati.get_moduli() for a in dati.get_aule() for c in dati.get_corsi()) <= 1 # Vincolo che fissa il numero di slot da assegnare per un modulo di un'attività didattica in una settimana for m in dati.get_moduli(): model += lpSum(skd[(c,m,a,g,s)] for a in dati.get_aule() for g in dati.get_giorni() for s in dati.get_slot() for c in dati.get_corsi()) == (m.get_num_sessioni()*m.get_dur_sessioni()) # Vincolo che impedisce che ad un aula venga assegnato più di un corso in uno slot di un dato giorno for a in dati.get_aule(): for g in dati.get_giorni(): for s in dati.get_slot(): model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() for c in dati.get_corsi()) <= 1 # Un corso per un dato giorno in un dato slot può essere assegnato ad una sola aula for c in dati.get_corsi(): for m in dati.get_moduli(): for g in dati.get_giorni(): for s in dati.get_slot(): model += lpSum(skd[(c, m, a, g, s)] for a in dati.get_aule()) <= 1 # Vincoli che permettono di assegnare ad un corso solo aule compatibili con la numerosità del corso stesso for c in dati.get_corsi(): for m in dati.get_moduli(): for g in dati.get_giorni(): for s in dati.get_slot(): for a in dati.get_aule(): if str_aux.get_compatibilita_aule()[(m,a)] == 0: model += skd[(c,m,a,g,s)] == 0 # Vincolo che non consente la sovrapposizione di moduli inerenti ad attività didattiche previste allo stesso anno di corso for c in dati.get_corsi(): for g in dati.get_giorni(): for s in dati.get_slot(): model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 1 for a in dati.get_aule()) <= 1 model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 2 for a in dati.get_aule()) <= 1 model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 3 for a in dati.get_aule()) <= 1 # Rende impossibili gli abbinamenti di assegnazioni di moduli di codici corso diversi dal codice del corso in esame for c in dati.get_corsi(): for g in dati.get_giorni(): for s in dati.get_slot(): for m in dati.get_moduli(): for a in dati.get_aule(): if c.get_codice() != m.get_cod_corso(): model += skd[(c,m,a,g,s)] == 0
def get_optimal_rackspace(cpu_usage, mem_usage, optimal=True, debug=False): """ Calculates the price of optimal resources allocation over a certain time span (with monthly granularity). Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function. """ assert(len(cpu_usage) == len(mem_usage)) prob = LpProblem("Rackspace cost optimization", LpMinimize) # variables ## 1h instances per_h_ondems = [] for p in range(len(cpu_usage)): per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()] category = LpInteger if optimal else LpContinuous vars = LpVariable.dicts("rackspace", per_h_ondems, 0, None, cat=category) # objective function prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]), "Total cost of running the infra (wrt to CPU/RAM)" # constraints ## demand constraints for p in range(len(cpu_usage)): prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][2] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CU demand period %s" %p prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand period %s" %p prob.solve() if debug: for v in prob.variables(): if v.varValue != 0.0: print v.name, "=", v.varValue print "Total Cost of the solution = ", value(prob.objective) return value(prob.objective)
def imposta_vincoli_addizionali(self, model, dati, str_aux, vincoli, cod_cds): if vincoli["chkPreferenzeDocenti"] == 1: skd=str_aux.get_schedulazione() for l in dati.get_logistica(): s = l[2] for m in dati.get_moduli(): if (m.get_offerta_id() == l[0] and m.get_id() == l[1]): break for c in dati.get_corsi(): if c.get_id() == m.get_corso_id(): break g = dati.get_giorni() model += lpSum(skd[(c,m,a,g[l[3]-1],dati.get_slot()[s-1])] for a in dati.get_aule()) == 1 if vincoli["posizioniFisse"] == None: None else: skd = str_aux.get_schedulazione() global posizioniFisse if (cod_cds == -1): posizioniFisse = vincoli["posizioniFisse"] else: posizioniFisse = [pf for pf in vincoli["posizioniFisse"] if pf['corso_id']==int(cod_cds)] for p in posizioniFisse: if (p["corso_id"] != -1 and p["modulo_id"] != -1 and p["aula_id"] != -1 and p["giorno_id"] != -1 and p["slot_id"] != -1): for c in dati.get_corsi(): if c.get_id() == p["corso_id"]: break for m in dati.get_moduli(): if m.get_id() == p["modulo_id"]: break for a in dati.get_aule(): if a.get_id() == p["aula_id"]: break for g in dati.get_giorni(): if g.get_id() == p["giorno_id"]: break for s in dati.get_slot(): if s.get_id() == p["slot_id"]: break model += skd[(c,m,a,g,s)] == 1
def solve(self, hosts, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties. """ host_instance_combinations = [] num_instances = filter_properties['num_instances'] num_hosts = len(hosts) instance_uuids = filter_properties.get('instance_uuids') or [ '(unknown_uuid)' + str(i) for i in xrange(num_instances)] filter_properties.setdefault('solver_cache', {}) filter_properties['solver_cache'].update( {'cost_matrix': [], 'constraint_matrix': []}) cost_matrix = self._get_cost_matrix(hosts, filter_properties) cost_matrix = self._adjust_cost_matrix(cost_matrix) constraint_matrix = self._get_constraint_matrix(hosts, filter_properties) # Create dictionaries mapping temporary host/instance keys to # hosts/instance_uuids. These temorary keys are to be used in the # solving process since we need a convention of lp variable names. host_keys = ['Host' + str(i) for i in xrange(num_hosts)] host_key_map = dict(zip(host_keys, hosts)) instance_num_keys = ['InstanceNum' + str(i) for i in xrange(num_instances + 1)] instance_num_key_map = dict(zip(instance_num_keys, xrange(num_instances + 1))) # create the pulp variables variable_matrix = [ [pulp.LpVariable('HI_' + host_key + '_' + instance_num_key, 0, 1, constants.LpInteger) for instance_num_key in instance_num_keys] for host_key in host_keys] # create the 'prob' variable to contain the problem data. prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) # add cost function to pulp solver cost_variables = [variable_matrix[i][j] for i in xrange(num_hosts) for j in xrange(num_instances + 1)] cost_coefficients = [cost_matrix[i][j] for i in xrange(num_hosts) for j in xrange(num_instances + 1)] prob += (pulp.lpSum([cost_coefficients[i] * cost_variables[i] for i in xrange(len(cost_variables))]), "Sum_Costs") # add constraints to pulp solver for i in xrange(num_hosts): for j in xrange(num_instances + 1): if constraint_matrix[i][j] is False: prob += (variable_matrix[i][j] == 0, "Cons_Host_%s" % i + "_NumInst_%s" % j) # add additional constraints to ensure the problem is valid # (1) non-trivial solution: number of all instances == that requested prob += (pulp.lpSum([variable_matrix[i][j] * j for i in xrange(num_hosts) for j in xrange(num_instances + 1)]) == num_instances, "NonTrivialCons") # (2) valid solution: each host is assigned 1 num-instances value for i in xrange(num_hosts): prob += (pulp.lpSum([variable_matrix[i][j] for j in xrange(num_instances + 1)]) == 1, "ValidCons_Host_%s" % i) # The problem is solved using PULP's choice of Solver. prob.solve(pulp_solver_classes.PULP_CBC_CMD( maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds)) # Create host-instance tuples from the solutions. if pulp.LpStatus[prob.status] == 'Optimal': num_insts_on_host = {} for v in prob.variables(): if v.name.startswith('HI'): (host_key, instance_num_key) = v.name.lstrip('HI').lstrip( '_').split('_') if v.varValue == 1: num_insts_on_host[host_key] = ( instance_num_key_map[instance_num_key]) instances_iter = iter(instance_uuids) for host_key in host_keys: num_insts_on_this_host = num_insts_on_host.get(host_key, 0) for i in xrange(num_insts_on_this_host): host_instance_combinations.append( (host_key_map[host_key], instances_iter.next())) else: LOG.warn(_LW("Pulp solver didnot find optimal solution! " "reason: %s"), pulp.LpStatus[prob.status]) host_instance_combinations = [] return host_instance_combinations
#prob+= t_i_p*i + t_i_o + t_i_l + 1 <= t_j_p*j + t_j_o + bigM*helpBinary, "(%s<%s),k_i=%d,k_j=%d" % (t_i_name,t_j_name,i,j) #prob+= t_j_p*j + t_j_o + t_j_l + 1 <= t_i_p*j + t_i_o + bigM*(1-helpBinary), "(%s<%s),k_i=%d,k_j=%d" % (t_j_name,t_i_name,i,j) prob+= t_i_p*i + t_i_o + t_i_l <= t_j_p*j + t_j_o + bigM*(1-helpBinaryForCollisionFree) prob+= t_j_p*j + t_j_o + t_j_l <= t_i_p*j + t_i_o + bigM*(helpBinaryForCollisionFree) print "##################" helpBinaryForSporadicOptimization=LpVariable("help_for_sporadic("+t_i_name+","+t_j_name+")",0,1,LpInteger) absoluteValueHelpU=LpVariable("U(%s,%s)" % (t_i_name,t_j_name),0,bigM,LpInteger) prob+=t_i_o-t_j_o+bigM*helpBinaryForSporadicOptimization >= absoluteValueHelpU prob+=t_j_o-t_i_o+bigM*(1-helpBinaryForSporadicOptimization) >= absoluteValueHelpU prob+=t_i_o-t_j_o <= absoluteValueHelpU prob+=t_j_o-t_i_o <= absoluteValueHelpU # get all U help variables uObjectives=[ v for k,v in prob.variablesDict().items() if 'U' in k] prob+=lpSum(uObjectives) print prob.writeLP("mori.lp") prob.solve() print("Status:", LpStatus[prob.status]) for v in prob.variables(): print(v.name, "=", v.varValue) print "**********************************************************************************************" prob.solve(GLPK()) for v in prob.variables(): print(v.name, "=", v.varValue) print("Status:", LpStatus[prob.status])
def host_solve(self, hosts, instance_uuids, request_spec, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties. """ host_instance_tuples_list = [] print filter_properties['instance_type']['memory_mb'] if instance_uuids: num_instances = len(instance_uuids) else: num_instances = request_spec.get('num_instances', 1) #Setting a unset uuid string for each instance. instance_uuids = ['unset_uuid' + str(i) for i in xrange(num_instances)] num_hosts = len(hosts) LOG.debug(_("All Hosts: %s") % [h.host for h in hosts]) for host in hosts: LOG.debug(_("Host state: %s") % host) # Create dictionaries mapping host/instance IDs to hosts/instances. host_ids = ['Host' + str(i) for i in range(num_hosts)] host_id_dict = dict(zip(host_ids, hosts)) instance_ids = ['Instance' + str(i) for i in range(num_instances)] instance_id_dict = dict(zip(instance_ids, instance_uuids)) # Create the 'prob' variable to contain the problem data. prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) # Create the 'variables' matrix to contain the referenced variables. variables = [[pulp.LpVariable("IA" + "_Host" + str(i) + "_Instance" + str(j), 0, 1, constants.LpInteger) for j in range(num_instances)] for i in range(num_hosts)] # Get costs and constraints and formulate the linear problem. self.cost_objects = [cost() for cost in self.cost_classes] self.constraint_objects = [constraint(variables, hosts, instance_uuids, request_spec, filter_properties) for constraint in self.constraint_classes] costs = [[0 for j in range(num_instances)] for i in range(num_hosts)] for cost_object in self.cost_objects: cost = cost_object.get_cost_matrix(hosts, instance_uuids, request_spec, filter_properties) cost = cost_object.normalize_cost_matrix(cost, 0.0, 1.0) weight = float(self.cost_weights[cost_object.__class__.__name__]) costs = [[costs[i][j] + weight * cost[i][j] for j in range(num_instances)] for i in range(num_hosts)] prob += (pulp.lpSum([costs[i][j] * variables[i][j] for i in range(num_hosts) for j in range(num_instances)]), "Sum_of_Host_Instance_Scheduling_Costs") for constraint_object in self.constraint_objects: coefficient_vectors = constraint_object.get_coefficient_vectors( variables, hosts, instance_uuids, request_spec, filter_properties) variable_vectors = constraint_object.get_variable_vectors( variables, hosts, instance_uuids, request_spec, filter_properties) operations = constraint_object.get_operations( variables, hosts, instance_uuids, request_spec, filter_properties) for i in range(len(operations)): operation = operations[i] len_vector = len(variable_vectors[i]) prob += (operation(pulp.lpSum([coefficient_vectors[i][j] * variable_vectors[i][j] for j in range(len_vector)])), "Costraint_Name_%s" % constraint_object.__class__.__name__ + "_No._%s" % i) prob.writeLP('test.lp') if filter_properties['instance_type']['constraint'] == "soft_affinity": prob = self.update_with_soft_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances) elif filter_properties['instance_type']['constraint'] == "strict_affinity": prob = self.update_with_strict_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances) elif filter_properties['instance_type']['constraint'] == "strict_antiaffinity": prob = self.update_with_strict_anti_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances) elif filter_properties['instance_type']['constraint'] == "soft_antiaffinity": prob = self.update_with_soft_anti_affinity_constraints_and_objective(variables,prob,num_hosts,num_instances) else: temp = filter_properties['instance_type']['constraint'] temp = temp.split("_") prob = self.update_with_host_count_constraints_and_objective(variables,prob,num_hosts,num_instances,temp[3]) print prob # The problem is solved using PULP's choice of Solver. prob.solve() # Create host-instance tuples from the solutions. if pulp.LpStatus[prob.status] == 'Optimal': for v in prob.variables(): if v.name.startswith('IA'): (host_id, instance_id) = v.name.lstrip('IA').lstrip( '_').split('_') if v.varValue == 1.0: host_instance_tuples_list.append( (host_id_dict[host_id], instance_id_dict[instance_id])) return host_instance_tuples_list
def distribute(self, fairness=True, output=None): """ This method is responsible for actually solving the linear programming problem. It uses the data in the instance variables. The optional parameter **fairness** indicates if the solution should minimize individual effort. By default the solution will enforce this condition. An error will be raised in case the data has inconsistencies. """ # Validate the problem variables self._validate() # State problem problem = pulp.LpProblem("Fair Distribution Problem", pulp.LpMinimize) # Prepare variables targets_objects = {} for (t, target) in enumerate(self._targets): for (o, object) in enumerate(self._objects): variable = pulp.LpVariable( 'x' + str(t) + str(o), lowBound=0, cat='Binary') position = {'target': t, 'object': o} targets_objects['x' + str(t) + str(o)] = (variable, position) # Generate linear expression for self._weights (Summation #1) weights = [(variable, self._weights[weight_position['target']][weight_position['object']]) for (variable, weight_position) in targets_objects.values()] weight_expression = pulp.LpAffineExpression(weights) # Generate linear expression for effort distribution (Summation #2) weight_diff_vars = [] if fairness: total_targets = len(self._targets) for (t, target) in enumerate(self._targets): weight_diff_aux_variable = pulp.LpVariable( 'weight_diff_'+str(t), lowBound=0) weight_diff_vars.append(weight_diff_aux_variable) negative_effort_diff_weights = [] positive_effort_diff_weights = [] negative_factor = -1 * total_targets positive_factor = 1 * total_targets for (o, object) in enumerate(self._objects): id = 'x' + str(t) + str(o) negative_effort_diff_weights.append( (targets_objects[id][0], negative_factor * self._weights[t][o])) positive_effort_diff_weights.append( (targets_objects[id][0], positive_factor * self._weights[t][o])) negative_effort_diff = pulp.LpAffineExpression( negative_effort_diff_weights) + weight_expression positive_effort_diff = pulp.LpAffineExpression( positive_effort_diff_weights) - weight_expression problem += negative_effort_diff <= weight_diff_aux_variable, 'abs negative effort diff ' + \ str(t) problem += positive_effort_diff <= weight_diff_aux_variable, 'abs positive effort diff ' + \ str(t) # Constraints - Each task must be done for (o, object) in enumerate(self._objects): constraints = [] for (t, target) in enumerate(self._targets): constraints.append(targets_objects['x' + str(t) + str(o)][0]) problem += pulp.lpSum(constraints) == 1, 'Task ' + \ str(o) + ' must be done' # Set objective function problem += weight_expression + \ pulp.lpSum(weight_diff_vars), "obj" if output: problem.writeLP(output) problem.solve() # Build output data = {} for v in filter(lambda x: x.varValue > 0, problem.variables()): if v.name not in targets_objects: continue position = targets_objects[v.name][1] target = self._targets[position['target']] object = self._objects[position['object']] if target not in data: data[target] = [] data[target].append(object) return data
def solve(self, hosts, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties. """ host_instance_combinations = [] num_instances = filter_properties['num_instances'] num_hosts = len(hosts) instance_uuids = filter_properties.get('instance_uuids') or [ '(unknown_uuid)' + str(i) for i in xrange(num_instances)] LOG.debug(_("All Hosts: %s") % [h.host for h in hosts]) for host in hosts: LOG.debug(_("Host state: %s") % host) # Create dictionaries mapping temporary host/instance keys to # hosts/instance_uuids. These temorary keys are to be used in the # solving process since we need a convention of lp variable names. host_keys = ['Host' + str(i) for i in xrange(num_hosts)] host_key_map = dict(zip(host_keys, hosts)) instance_keys = ['InstanceNum' + str(i) for i in xrange(num_instances)] instance_key_map = dict( zip(instance_keys, xrange(1, num_instances + 1))) # this is currently hard-coded and should match variable names host_instance_matrix_idx_map = {} for i in xrange(len(host_keys)): for j in xrange(len(instance_keys)): var_name = 'HI_' + host_keys[i] + '_' + instance_keys[j] host_instance_matrix_idx_map[var_name] = (i, j) # Create the 'variables' to contain the referenced variables. self.variables.populate_variables(host_keys, instance_keys) # Create the 'prob' variable to contain the problem data. prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) # Get costs and constraints and formulate the linear problem. # Add costs. cost_objects = [cost() for cost in self.cost_classes] cost_coeff_matrix = [[0 for j in xrange(num_instances)] for i in xrange(num_hosts)] for cost_object in cost_objects: var_list, coeff_list = cost_object.get_components( self.variables, hosts, filter_properties) for i in xrange(len(var_list)): var = var_list[i] coeff = coeff_list[i] hidx, iidx = host_instance_matrix_idx_map[var.name] cost_coeff_matrix[hidx][iidx] += ( coeff * cost_object.cost_multiplier()) cost_coeff_matrix = self._calculate_host_instance_cost_matrix( cost_coeff_matrix) cost_coeff_array = [] for var in var_list: hidx, iidx = host_instance_matrix_idx_map[var.name] cost_coeff_array.append(cost_coeff_matrix[hidx][iidx]) cost_variables = var_list cost_coefficients = cost_coeff_array if cost_variables: prob += (pulp.lpSum([cost_coefficients[i] * cost_variables[i] for i in xrange(len(cost_variables))]), "Sum_Costs") # Add constraints. constraint_objects = [constraint() for constraint in self.constraint_classes] for constraint_object in constraint_objects: vars_list, coeffs_list, consts_list, ops_list = ( constraint_object.get_components(self.variables, hosts, filter_properties)) LOG.debug(_("coeffs of %(name)s is: %(value)s") % {"name": constraint_object.__class__.__name__, "value": coeffs_list}) for i in xrange(len(ops_list)): operation = self._get_operation(ops_list[i]) prob += ( operation(pulp.lpSum([coeffs_list[i][j] * vars_list[i][j] for j in xrange(len(vars_list[i]))]), consts_list[i]), "Costraint_Name_%s" % constraint_object.__class__.__name__ + "_No._%s" % i) # The problem is solved using PULP's choice of Solver. prob.solve(pulp_solver_classes.PULP_CBC_CMD( maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds)) # Create host-instance tuples from the solutions. if pulp.LpStatus[prob.status] == 'Optimal': num_insts_on_host = {} for v in prob.variables(): if v.name.startswith('HI'): (host_key, instance_key) = v.name.lstrip('HI').lstrip( '_').split('_') if v.varValue == 1: num_insts_on_host[host_key] = ( instance_key_map[instance_key]) instances_iter = iter(instance_uuids) for host_key in host_keys: num_insts_on_this_host = num_insts_on_host.get(host_key, 0) for i in xrange(num_insts_on_this_host): host_instance_combinations.append( (host_key_map[host_key], instances_iter.next())) elif pulp.LpStatus[prob.status] == 'Infeasible': LOG.warn(_("Pulp solver didnot find optimal solution! reason: %s") % pulp.LpStatus[prob.status]) host_instance_combinations = [] else: LOG.warn(_("Pulp solver didnot find optimal solution! reason: %s") % pulp.LpStatus[prob.status]) raise exception.SolverFailed(reason=pulp.LpStatus[prob.status]) return host_instance_combinations
def createConstraints(problem, variable): problem += lpSum(choiceVariables) <= 1, "Distribution" for tower, towerName in towerNames.items(): problem += createLpSum(tower.getChoiceClass(), choiceNames, choiceVariables) >= \ tower.getHeight() + variable * \ tower.getSpeed(), towerName
def solve(self, hosts, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties. """ host_instance_combinations = [] num_instances = filter_properties['num_instances'] num_hosts = len(hosts) instance_uuids = filter_properties.get('instance_uuids') or [ '(unknown_uuid)' + str(i) for i in xrange(num_instances) ] filter_properties.setdefault('solver_cache', {}) filter_properties['solver_cache'].update({ 'cost_matrix': [], 'constraint_matrix': [] }) cost_matrix = self._get_cost_matrix(hosts, filter_properties) cost_matrix = self._adjust_cost_matrix(cost_matrix) constraint_matrix = self._get_constraint_matrix( hosts, filter_properties) # Create dictionaries mapping temporary host/instance keys to # hosts/instance_uuids. These temorary keys are to be used in the # solving process since we need a convention of lp variable names. host_keys = ['Host' + str(i) for i in xrange(num_hosts)] host_key_map = dict(zip(host_keys, hosts)) instance_num_keys = [ 'InstanceNum' + str(i) for i in xrange(num_instances + 1) ] instance_num_key_map = dict( zip(instance_num_keys, xrange(num_instances + 1))) # create the pulp variables variable_matrix = [[ pulp.LpVariable('HI_' + host_key + '_' + instance_num_key, 0, 1, constants.LpInteger) for instance_num_key in instance_num_keys ] for host_key in host_keys] # create the 'prob' variable to contain the problem data. prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) # add cost function to pulp solver cost_variables = [ variable_matrix[i][j] for i in xrange(num_hosts) for j in xrange(num_instances + 1) ] cost_coefficients = [ cost_matrix[i][j] for i in xrange(num_hosts) for j in xrange(num_instances + 1) ] prob += (pulp.lpSum([ cost_coefficients[i] * cost_variables[i] for i in xrange(len(cost_variables)) ]), "Sum_Costs") # add constraints to pulp solver for i in xrange(num_hosts): for j in xrange(num_instances + 1): if constraint_matrix[i][j] is False: prob += (variable_matrix[i][j] == 0, "Cons_Host_%s" % i + "_NumInst_%s" % j) # add additional constraints to ensure the problem is valid # (1) non-trivial solution: number of all instances == that requested prob += (pulp.lpSum([ variable_matrix[i][j] * j for i in xrange(num_hosts) for j in xrange(num_instances + 1) ]) == num_instances, "NonTrivialCons") # (2) valid solution: each host is assigned 1 num-instances value for i in xrange(num_hosts): prob += (pulp.lpSum([ variable_matrix[i][j] for j in xrange(num_instances + 1) ]) == 1, "ValidCons_Host_%s" % i) # The problem is solved using PULP's choice of Solver. prob.solve( pulp_solver_classes.PULP_CBC_CMD( maxSeconds=CONF.solver_scheduler.pulp_solver_timeout_seconds)) # Create host-instance tuples from the solutions. if pulp.LpStatus[prob.status] == 'Optimal': num_insts_on_host = {} for v in prob.variables(): if v.name.startswith('HI'): (host_key, instance_num_key ) = v.name.lstrip('HI').lstrip('_').split('_') if v.varValue == 1: num_insts_on_host[host_key] = ( instance_num_key_map[instance_num_key]) instances_iter = iter(instance_uuids) for host_key in host_keys: num_insts_on_this_host = num_insts_on_host.get(host_key, 0) for i in xrange(num_insts_on_this_host): host_instance_combinations.append( (host_key_map[host_key], instances_iter.next())) else: LOG.warn( _LW("Pulp solver didnot find optimal solution! " "reason: %s"), pulp.LpStatus[prob.status]) host_instance_combinations = [] return host_instance_combinations
def imposta_vincoli_facoltativi(self, model, dati, str_aux, vincoli): skd=str_aux.get_schedulazione() if vincoli["chkSessioneUnica"] == 1: # Per ogni giorno, ogni corso ed ogni aula il numero di slot massimo consentito è pari alla durata delle sessioni # ciò evita che ci possano essere due sessioni nello stesso giorno for c in dati.get_corsi(): for m in dati.get_moduli(): for g in dati.get_giorni(): model += lpSum(skd[(c,m,a,g,s)] for s in dati.get_slot() for a in dati.get_aule()) <= m.get_dur_sessioni() if vincoli["chkSessioniConsecutive"] == 1: # Vincoli di consecutività delle sessioni for c in dati.get_corsi(): for m in dati.get_moduli(): dur_sessione = m.get_dur_sessioni() if dur_sessione == 2: for a in dati.get_aule(): for g in dati.get_giorni(): for s in (0,4): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 for s in (1,2,5,6): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 for s in (3,7): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 elif dur_sessione == 3: for a in dati.get_aule(): for g in dati.get_giorni(): for s in (0,4): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0 for s in (1,5): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0 for s in (2,6): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0 for s in (3,7): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0 elif dur_sessione == 4: for a in dati.get_aule(): for g in dati.get_giorni(): for s in (0,4): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+3])] <= 0 for s in (1,5): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])]-skd[(c,m,a,g,dati.get_slot()[s+2])] <= 0 for s in (2,6): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s+1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0 for s in (3,7): model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-1])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-2])] <= 0 model += skd[(c,m,a,g,dati.get_slot()[s])]-skd[(c,m,a,g,dati.get_slot()[s-3])] <= 0 # Vincolo che fissa il numero massimo di slot per giorno per un anno di corso if vincoli["chkMaxOre"] == 1: limsup = vincoli["selMaxOre"] for c in dati.get_corsi(): for g in dati.get_giorni(): model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 1 for s in dati.get_slot() for a in dati.get_aule())<=limsup model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 2 for s in dati.get_slot() for a in dati.get_aule())<=limsup model += lpSum(skd[(c,m,a,g,s)] for m in dati.get_moduli() if m.get_anno_corso() == 3 for s in dati.get_slot() for a in dati.get_aule())<=limsup
def distribute_factors( agents: Dict[str, AgentDef], cg: ComputationGraph, footprints: Dict[str, float], mapping: Dict[str, List[str]], msg_load: Callable[[str, str], float], ) -> Dict[str, List[str]]: """ Optimal distribution of factors on agents. Parameters ---------- cg: computations graph agents: dict a dict {agent_name : AgentDef} containing all available agents Returns ------- a dict { agent_name: list of factor names} """ pb = LpProblem("ilp_factors", LpMinimize) # build the inverse mapping var -> agt inverse_mapping = {} # type: Dict[str, str] for a in mapping: inverse_mapping[mapping[a][0]] = a # One binary variable xij for each (variable, agent) couple factor_names = [n.name for n in cg.nodes if isinstance(n, FactorComputationNode)] xs = LpVariable.dict("x", (factor_names, agents), cat=LpBinary) logger.debug("Binary variables for factor distribution : %s", xs) # Hard constraints: respect agent's capacity for a in agents: # Footprint of the variable this agent is already hosting: v_footprint = footprints[mapping[a][0]] pb += ( lpSum([footprints[fn] * xs[fn, a] for fn in factor_names]) <= (agents[a].capacity - v_footprint), "Agent {} capacity".format(a), ) # Hard constraints: all computations must be hosted. for c in factor_names: pb += lpSum([xs[c, a] for a in agents]) == 1, "Factor {} hosted".format(c) # 1st objective : minimize communication costs: comm = LpAffineExpression() for (fn, an_f) in xs: for vn in cg.neighbors(fn): an_v = inverse_mapping[vn] # agt hosting neighbor var vn comm += agents[an_f].route(an_v) * msg_load(vn, fn) * xs[(fn, an_f)] # 2st objective : minimize hosting costs hosting = lpSum([agents[a].hosting_cost(c) * xs[(c, a)] for c, a in xs]) # agregate the two objectives using RATIO_HOST_COMM pb += lpSum([RATIO_HOST_COMM * comm, (1 - RATIO_HOST_COMM) * hosting]) # solve using GLPK and convert to mapping { agt_name : [factors names]} status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False, options=["--pcost"])) if status != LpStatusOptimal: raise ImpossibleDistributionException( "No possible optimal distribution for factors" ) logger.debug("GLPK cost : %s", value(pb.objective)) mapping = {} # type: Dict[str, List[str]] for k in agents: agt_computations = [i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1] # print(k, ' -> ', agt_computations) mapping[k] = agt_computations logger.debug("Factors distribution : %s ", mapping) return mapping
def host_solve(self, hosts, instance_uuids, request_spec, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties. """ host_instance_tuples_list = [] if instance_uuids: num_instances = len(instance_uuids) else: num_instances = request_spec.get('num_instances', 1) #Setting a unset uuid string for each instance. instance_uuids = [ 'unset_uuid' + str(i) for i in xrange(num_instances) ] num_hosts = len(hosts) LOG.debug(_("All Hosts: %s") % [h.host for h in hosts]) for host in hosts: LOG.debug(_("Host state: %s") % host) # Create dictionaries mapping host/instance IDs to hosts/instances. host_ids = ['Host' + str(i) for i in range(num_hosts)] host_id_dict = dict(zip(host_ids, hosts)) instance_ids = ['Instance' + str(i) for i in range(num_instances)] instance_id_dict = dict(zip(instance_ids, instance_uuids)) # Create the 'prob' variable to contain the problem data. prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) # Create the 'variables' matrix to contain the referenced variables. variables = [[ pulp.LpVariable("IA" + "_Host" + str(i) + "_Instance" + str(j), 0, 1, constants.LpInteger) for j in range(num_instances) ] for i in range(num_hosts)] # Get costs and constraints and formulate the linear problem. self.cost_objects = [cost() for cost in self.cost_classes] self.constraint_objects = [ constraint(variables, hosts, instance_uuids, request_spec, filter_properties) for constraint in self.constraint_classes ] costs = [[0 for j in range(num_instances)] for i in range(num_hosts)] for cost_object in self.cost_objects: cost = cost_object.get_cost_matrix(hosts, instance_uuids, request_spec, filter_properties) cost = cost_object.normalize_cost_matrix(cost, 0.0, 1.0) weight = float(self.cost_weights[cost_object.__class__.__name__]) costs = [[ costs[i][j] + weight * cost[i][j] for j in range(num_instances) ] for i in range(num_hosts)] prob += (pulp.lpSum([ costs[i][j] * variables[i][j] for i in range(num_hosts) for j in range(num_instances) ]), "Sum_of_Host_Instance_Scheduling_Costs") for constraint_object in self.constraint_objects: coefficient_vectors = constraint_object.get_coefficient_vectors( variables, hosts, instance_uuids, request_spec, filter_properties) variable_vectors = constraint_object.get_variable_vectors( variables, hosts, instance_uuids, request_spec, filter_properties) operations = constraint_object.get_operations( variables, hosts, instance_uuids, request_spec, filter_properties) for i in range(len(operations)): operation = operations[i] len_vector = len(variable_vectors[i]) prob += (operation( pulp.lpSum([ coefficient_vectors[i][j] * variable_vectors[i][j] for j in range(len_vector) ])), "Costraint_Name_%s" % constraint_object.__class__.__name__ + "_No._%s" % i) # The problem is solved using PULP's choice of Solver. prob.solve() # Create host-instance tuples from the solutions. if pulp.LpStatus[prob.status] == 'Optimal': for v in prob.variables(): if v.name.startswith('IA'): (host_id, instance_id) = v.name.lstrip('IA').lstrip('_').split('_') if v.varValue == 1.0: host_instance_tuples_list.append( (host_id_dict[host_id], instance_id_dict[instance_id])) return host_instance_tuples_list
def lp_model(cg: ComputationGraph, agentsdef: Iterable[AgentDef], footprint: Callable[[str], float], capacity: Callable[[str], float], route: Callable[[str, str], float], msg_load: Callable[[str, str], float], hosting_cost: Callable[[str, str], float]): comp_names = [n.name for n in cg.nodes] agt_names = [a.name for a in agentsdef] pb = LpProblem('ilp_compref', LpMinimize) # One binary variable xij for each (variable, agent) couple xs = LpVariable.dict('x', (comp_names, agt_names), cat=LpBinary) # One binary variable for computations c1 and c2, and agent a1 and a2 betas = {} count = 0 for a1, a2 in combinations(agt_names, 2): # Only create variables for couple c1, c2 if there is an edge in the # graph between these two computations. for l in cg.links: # As we support hypergraph, we may have more than 2 ends to a link for c1, c2 in combinations(l.nodes, 2): count += 2 b = LpVariable('b_{}_{}_{}_{}'.format(c1, a1, c2, a2), cat=LpBinary) betas[(c1, a1, c2, a2)] = b pb += b <= xs[(c1, a1)] pb += b <= xs[(c2, a2)] pb += b >= xs[(c2, a2)] + xs[(c1, a1)] - 1 b = LpVariable('b_{}_{}_{}_{}'.format(c1, a2, c2, a1), cat=LpBinary) betas[(c1, a2, c2, a1)] = b pb += b <= xs[(c2, a1)] pb += b <= xs[(c1, a2)] pb += b >= xs[(c1, a2)] + xs[(c2, a1)] - 1 # Set objective: communication + hosting_cost pb += _objective(xs, betas, route, msg_load, hosting_cost), \ 'Communication costs and prefs' # Adding constraints: # Constraints: Memory capacity for all agents. for a in agt_names: pb += lpSum([footprint(i) * xs[i, a] for i in comp_names])\ <= capacity(a), \ 'Agent {} capacity'.format(a) # Constraints: all computations must be hosted. for c in comp_names: pb += lpSum([xs[c, a] for a in agt_names]) == 1, \ 'Computation {} hosted'.format(c) # solve using GLPK status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False, options=['--pcost'])) if status != LpStatusOptimal: raise ImpossibleDistributionException("No possible optimal" " distribution ") logger.debug('GLPK cost : %s', value(pb.objective)) # print('BETAS:') # for c1, a1, c2, a2 in betas: # print(' ', c1, a1, c2, a2, value(betas[(c1, a1, c2, a2)])) # # print('XS:') # for c, a in xs: # print(' ', c, a, value(xs[(c, a)])) mapping = {} for k in agt_names: agt_computations = [i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1] # print(k, ' -> ', agt_computations) mapping[k] = agt_computations return mapping
def createLpSum(choiceClass, choiceNames, choiceVariables): return lpSum(choiceVariables[choiceNames[choice]] for choice in choiceClass)
def host_solve(self, hosts, instance_uuids, request_spec, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties """ host_instance_tuples_list = [] if instance_uuids: num_instances = len(instance_uuids) else: num_instances = request_spec.get('num_instances', 1) instance_uuids = [ 'unset_uuid%s' % i for i in xrange(num_instances) ] num_hosts = len(hosts) host_ids = ['Host%s' % i for i in range(num_hosts)] LOG.debug(_("All Hosts: %s") % [h.host for h in hosts]) for host in hosts: LOG.debug(_("Host state: %s") % host) host_id_dict = dict(zip(host_ids, hosts)) instances = ['Instance%s' % i for i in range(num_instances)] instance_id_dict = dict(zip(instances, instance_uuids)) # supply is a dictionary for the number of units of # resource for each Host. # Currently using only the disk_mb and memory_mb # as the two resources to satisfy. Need to eventually be able to # plug-in different resources. An example supply dictionary: # supply = {"Host1": [1000, 1000], # "Host2": [4000, 1000]} supply = dict((host_ids[i], [ self._get_usable_disk_mb(hosts[i]), self._get_usable_memory_mb(hosts[i]), ]) for i in range(len(host_ids))) number_of_resource_types_per_host = 2 required_disk_mb = self._get_required_disk_mb(filter_properties) required_memory_mb = self._get_required_memory_mb(filter_properties) # demand is a dictionary for the number of # units of resource required for each Instance. # An example demand dictionary: # demand = {"Instance0":[200, 300], # "Instance1":[900, 100], # "Instance2":[1800, 200], # "Instance3":[200, 300], # "Instance4":[700, 800], } # However for the current scenario, all instances to be scheduled # per request have the same requirements. Need to eventually # to support requests to specify different instance requirements demand = dict((instances[i], [ required_disk_mb, required_memory_mb, ]) for i in range(num_instances)) # Creates a list of costs of each Host-Instance assignment # Currently just like the nova.scheduler.weights.ram.RAMWeigher, # using host_state.free_ram_mb * ram_weight_multiplier # as the cost. A negative ram_weight_multiplier means to stack, # vs spread. # An example costs list: # costs = [ # Instances # # 1 2 3 4 5 # [2, 4, 5, 2, 1], # A Hosts # [3, 1, 3, 2, 3] # B # ] # Multiplying -1 as we want to use the same behavior of # ram_weight_multiplier as used by ram weigher. costs = [[ -1 * host.free_ram_mb * CONF.ram_weight_multiplier for i in range(num_instances) ] for host in hosts] costs = pulp.makeDict([host_ids, instances], costs, 0) # The PULP LP problem variable used to add all the problem data prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) all_host_instance_tuples = [(w, b) for w in host_ids for b in instances] vars = pulp.LpVariable.dicts("IA", (host_ids, instances), 0, 1, constants.LpInteger) # The objective function is added to 'prob' first prob += (pulp.lpSum([ vars[w][b] * costs[w][b] for (w, b) in all_host_instance_tuples ]), "Sum_of_Host_Instance_Scheduling_Costs") # The supply maximum constraints are added to # prob for each supply node (Host) for w in host_ids: for i in range(number_of_resource_types_per_host): prob += (pulp.lpSum( [vars[w][b] * demand[b][i] for b in instances]) <= supply[w][i], "Sum_of_Resource_%s" % i + "_provided_by_Host_%s" % w) # The number of Hosts required per Instance, in this case it is only 1 for b in instances: prob += (pulp.lpSum([vars[w][b] for w in host_ids]) == 1, "Sum_of_Instance_Assignment%s" % b) # The demand minimum constraints are added to prob for # each demand node (Instance) for b in instances: for j in range(number_of_resource_types_per_host): prob += ( pulp.lpSum([vars[w][b] * demand[b][j] for w in host_ids]) >= demand[b][j], "Sum_of_Resource_%s" % j + "_required_by_Instance_%s" % b) # The problem is solved using PuLP's choice of Solver prob.solve() if pulp.LpStatus[prob.status] == 'Optimal': for v in prob.variables(): if v.name.startswith('IA'): (host_id, instance_id) = v.name.lstrip('IA').lstrip('_').split('_') if v.varValue == 1.0: host_instance_tuples_list.append( (host_id_dict[host_id], instance_id_dict[instance_id])) return host_instance_tuples_list
def host_solve(self, hosts, instance_uuids, request_spec, filter_properties): """This method returns a list of tuples - (host, instance_uuid) that are returned by the solver. Here the assumption is that all instance_uuids have the same requirement as specified in filter_properties """ host_instance_tuples_list = [] if instance_uuids: num_instances = len(instance_uuids) else: num_instances = request_spec.get('num_instances', 1) instance_uuids = ['unset_uuid%s' % i for i in xrange(num_instances)] num_hosts = len(hosts) host_ids = ['Host%s' % i for i in range(num_hosts)] LOG.debug(_("All Hosts: %s") % [h.host for h in hosts]) for host in hosts: LOG.debug(_("Host state: %s") % host) host_id_dict = dict(zip(host_ids, hosts)) instances = ['Instance%s' % i for i in range(num_instances)] instance_id_dict = dict(zip(instances, instance_uuids)) # supply is a dictionary for the number of units of # resource for each Host. # Currently using only the disk_mb and memory_mb # as the two resources to satisfy. Need to eventually be able to # plug-in different resources. An example supply dictionary: # supply = {"Host1": [1000, 1000], # "Host2": [4000, 1000]} supply = dict((host_ids[i], [self._get_usable_disk_mb(hosts[i]), self._get_usable_memory_mb(hosts[i]), ]) for i in range(len(host_ids))) number_of_resource_types_per_host = 2 required_disk_mb = self._get_required_disk_mb(filter_properties) required_memory_mb = self._get_required_memory_mb(filter_properties) # demand is a dictionary for the number of # units of resource required for each Instance. # An example demand dictionary: # demand = {"Instance0":[200, 300], # "Instance1":[900, 100], # "Instance2":[1800, 200], # "Instance3":[200, 300], # "Instance4":[700, 800], } # However for the current scenario, all instances to be scheduled # per request have the same requirements. Need to eventually # to support requests to specify different instance requirements demand = dict((instances[i], [required_disk_mb, required_memory_mb, ]) for i in range(num_instances)) # Creates a list of costs of each Host-Instance assignment # Currently just like the nova.scheduler.weights.ram.RAMWeigher, # using host_state.free_ram_mb * ram_weight_multiplier # as the cost. A negative ram_weight_multiplier means to stack, # vs spread. # An example costs list: # costs = [ # Instances # # 1 2 3 4 5 # [2, 4, 5, 2, 1], # A Hosts # [3, 1, 3, 2, 3] # B # ] # Multiplying -1 as we want to use the same behavior of # ram_weight_multiplier as used by ram weigher. costs = [[-1 * host.free_ram_mb * CONF.ram_weight_multiplier for i in range(num_instances)] for host in hosts] costs = pulp.makeDict([host_ids, instances], costs, 0) # The PULP LP problem variable used to add all the problem data prob = pulp.LpProblem("Host Instance Scheduler Problem", constants.LpMinimize) all_host_instance_tuples = [(w, b) for w in host_ids for b in instances] vars = pulp.LpVariable.dicts("IA", (host_ids, instances), 0, 1, constants.LpInteger) # The objective function is added to 'prob' first prob += (pulp.lpSum([vars[w][b] * costs[w][b] for (w, b) in all_host_instance_tuples]), "Sum_of_Host_Instance_Scheduling_Costs") # The supply maximum constraints are added to # prob for each supply node (Host) for w in host_ids: for i in range(number_of_resource_types_per_host): prob += (pulp.lpSum([vars[w][b] * demand[b][i] for b in instances]) <= supply[w][i], "Sum_of_Resource_%s" % i + "_provided_by_Host_%s" % w) # The number of Hosts required per Instance, in this case it is only 1 for b in instances: prob += (pulp.lpSum([vars[w][b] for w in host_ids]) == 1, "Sum_of_Instance_Assignment%s" % b) # The demand minimum constraints are added to prob for # each demand node (Instance) for b in instances: for j in range(number_of_resource_types_per_host): prob += (pulp.lpSum([vars[w][b] * demand[b][j] for w in host_ids]) >= demand[b][j], "Sum_of_Resource_%s" % j + "_required_by_Instance_%s" % b) # The problem is solved using PuLP's choice of Solver prob.solve() print prob if pulp.LpStatus[prob.status] == 'Optimal': for v in prob.variables(): if v.name.startswith('IA'): (host_id, instance_id) = v.name.lstrip('IA').lstrip( '_').split('_') if v.varValue == 1.0: host_instance_tuples_list.append( (host_id_dict[host_id], instance_id_dict[instance_id])) return host_instance_tuples_list
def get_optimal_ec2(cpu_usage, mem_usage, optimal=True, debug=False): """ Calculates the optimal allocation of resources over a certain time span (with monthly granularity). Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function. Returns allocation of reserved instances and a total price for running the allocation on AWS. """ assert(len(cpu_usage) == len(mem_usage)) prob = LpProblem("The Simplified EC2 cost optimization", LpMinimize) # variables ## 1h instances (both on-demand and reserved) per_h_ondems = [] per_h_reserved = [] for p in range(len(cpu_usage)): # ondemand per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()] # reserved per_h_reserved += ["p %s reserved %s" %(p, i) for i in vms.keys()] ## nr of 1-year reserved instances nr_of_1year_reserved = [ "res_1year %s" % i for i in vms.keys()] nr_of_3year_reserved = [ "res_3year %s" % i for i in vms.keys()] category = LpInteger if optimal else LpContinuous vars = LpVariable.dicts("aws", per_h_ondems + per_h_reserved + nr_of_1year_reserved + nr_of_3year_reserved, \ lowBound = 0, upBound = None, cat = category) # objective function prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]) \ + lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_reserved]) \ + lpSum([vars[vm] * vms[vm.split(" ")[1]][1] for vm in nr_of_1year_reserved]) \ + lpSum([vars[vm] * vms[vm.split(" ")[1]][2] for vm in nr_of_3year_reserved]) \ , "Total cost of running the infrastructure consuming (CPU/RAM)/h" # constraints ## demand constraints for p in range(len(cpu_usage)): prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][4] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CPU demand, period %s" %p prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][5] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand. period %s" %p ## constraints on the reserved instances - cannot use more than we paid for for i in per_h_reserved: t = i.split(" ")[3] prob += vars["res_1year %s" % t] + vars["res_3year %s" % t] >= vars[i], "Nr. of used reserved machines of type %s" %i prob.solve() if debug: print "Status:", LpStatus[prob.status] print "Total Cost of the solution = ", value(prob.objective) res_instance = {} for v in prob.variables(): if v.name.startswith("aws_res_") and v.varValue != 0.0: res_instance[v.name] = v.varValue if debug and v.varValue != 0.0: print v.name, "=", v.varValue return (res_instance, value(prob.objective))