def get_optimal_rackspace(cpu_usage, mem_usage, optimal=True, debug=False):
"""
Calculates the price of optimal resources allocation over a certain time span (with monthly granularity).
Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function.
"""
assert(len(cpu_usage) == len(mem_usage))
prob = LpProblem("Rackspace cost optimization", LpMinimize)
# variables
## 1h instances
per_h_ondems = []
for p in range(len(cpu_usage)):
per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()]
category = LpInteger if optimal else LpContinuous
vars = LpVariable.dicts("rackspace", per_h_ondems, 0, None, cat=category)
# objective function
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]), "Total cost of running the infra (wrt to CPU/RAM)"
# constraints
## demand constraints
for p in range(len(cpu_usage)):
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][2] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CU demand period %s" %p
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_ondems if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand period %s" %p
prob.solve()
if debug:
for v in prob.variables():
if v.varValue != 0.0:
print v.name, "=", v.varValue
print "Total Cost of the solution = ", value(prob.objective)
return value(prob.objective)
def fg_secp_ilp(
cg: ComputationsFactorGraph,
agents: List[AgentDef],
already_assigned: Distribution,
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
) -> Distribution:
variables = [n for n in cg.nodes if n.type == "VariableComputation"]
factors = [n for n in cg.nodes if n.type == "FactorComputation"]
agents = list(agents)
agents_names = [a.name for a in agents]
# Only keep computations for which we actually need to find an agent.
vars_to_host = [
v.name for v in variables
if not already_assigned.has_computation(v.name)
]
facs_to_host = [
f.name for f in factors if not already_assigned.has_computation(f.name)
]
# x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
xs = _build_xs_binvar(vars_to_host, agents_names)
# f_j^k : binary variable indicating if factor f_j is hosted on agent a_k.
fs = _build_fs_binvar(facs_to_host, agents_names)
# alpha_ijk : binary variable indicating if x_i and f_j are both on a_k.
alphas = _build_alphaijk_binvars(cg, agents_names)
logger.debug(f"alpha_ijk {alphas}")
# LP problem with objective function (total communication cost).
pb = LpProblem("distribution", LpMinimize)
pb += (
secp_dist_objective_function(cg, communication_load, alphas,
agents_names),
"Communication costs",
)
# Constraints.
# All variable computations must be hosted:
for i in vars_to_host:
pb += (
lpSum([xs[(i, k)] for k in agents_names]) == 1,
"var {} is hosted".format(i),
)
# All factor computations must be hosted:
for j in facs_to_host:
pb += (
lpSum([fs[(j, k)] for k in agents_names]) == 1,
"factor {} is hosted".format(j),
)
# Each agent must host at least one computation:
# We only need this constraints for agents that do not already host a
# computation:
empty_agents = [
a for a in agents_names if not already_assigned.computations_hosted(a)
]
for k in empty_agents:
pb += (
lpSum([xs[(i, k)] for i in vars_to_host]) +
lpSum([fs[(j, k)] for j in facs_to_host]) >= 1,
"atleastone {}".format(k),
)
# Memory capacity constraint for agents
for a in agents:
# Decrease capacity for already hosted computations
capacity = a.capacity - sum([
secp_computation_memory_in_cg(c, cg, computation_memory)
for c in already_assigned.computations_hosted(a.name)
])
pb += (
lpSum([
secp_computation_memory_in_cg(i, cg, computation_memory) * xs[
(i, a.name)] for i in vars_to_host
]) + lpSum([
secp_computation_memory_in_cg(j, cg, computation_memory) * fs[
(j, a.name)] for j in facs_to_host
]) <= capacity,
"memory {}".format(a.name),
)
# Linearization constraints for alpha_ijk.
for link in cg.links:
i, j = link.variable_node, link.factor_node
for k in agents_names:
if i in vars_to_host and j in facs_to_host:
pb += alphas[((i, j), k)] <= xs[(i, k)], "lin1 {}{}{}".format(
i, j, k)
pb += alphas[((i, j), k)] <= fs[(j, k)], "lin2 {}{}{}".format(
i, j, k)
pb += (
alphas[((i, j), k)] >= xs[(i, k)] + fs[(j, k)] - 1,
"lin3 {}{}{}".format(i, j, k),
)
elif i in vars_to_host and j not in facs_to_host:
# Var is free, factor is already hosted
if already_assigned.agent_for(j) == k:
pb += alphas[((i, j), k)] == xs[(i, k)]
else:
pb += alphas[((i, j), k)] == 0
elif i not in vars_to_host and j in facs_to_host:
# if i is not in vars_vars_to_host, it means that it's a
# computation that is already hosted (from hints)
if already_assigned.agent_for(i) == k:
pb += alphas[((i, j), k)] == fs[(j, k)]
else:
pb += alphas[((i, j), k)] == 0
else:
# i and j are both alredy hosted
if (already_assigned.agent_for(i) == k
and already_assigned.agent_for(j) == k):
pb += alphas[((i, j), k)] == 1
else:
pb += alphas[((i, j), k)] == 0
# Now solve our LP
# status = pb.solve(GLPK_CMD())
# status = pb.solve(GLPK_CMD(mip=1))
# status = pb.solve(GLPK_CMD(mip=0, keepFiles=1,
# options=['--simplex', '--interior']))
status = pb.solve(GLPK_CMD(keepFiles=0, msg=False, options=["--pcost"]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
else:
logger.debug("GLPK cost : %s", pulp.value(pb.objective))
comp_dist = already_assigned
for k in agents_names:
agt_vars = [
i for i, ka in xs if ka == k and pulp.value(xs[(i, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_vars)
agt_rels = [
j for j, ka in fs if ka == k and pulp.value(fs[(j, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_rels)
return comp_dist
def lp_model(cg: ComputationGraph,
agentsdef: Iterable[AgentDef],
footprint: Callable[[str], float],
capacity: Callable[[str], float],
route: Callable[[str, str], float],
msg_load: Callable[[str, str], float],
hosting_cost: Callable[[str, str], float]):
comp_names = [n.name for n in cg.nodes]
agt_names = [a.name for a in agentsdef]
pb = LpProblem('ilp_compref', LpMinimize)
# One binary variable xij for each (variable, agent) couple
xs = LpVariable.dict('x', (comp_names, agt_names), cat=LpBinary)
# One binary variable for computations c1 and c2, and agent a1 and a2
betas = {}
count = 0
for a1, a2 in combinations(agt_names, 2):
# Only create variables for couple c1, c2 if there is an edge in the
# graph between these two computations.
for l in cg.links:
# As we support hypergraph, we may have more than 2 ends to a link
for c1, c2 in combinations(l.nodes, 2):
count += 2
b = LpVariable('b_{}_{}_{}_{}'.format(c1, a1, c2, a2),
cat=LpBinary)
betas[(c1, a1, c2, a2)] = b
pb += b <= xs[(c1, a1)]
pb += b <= xs[(c2, a2)]
pb += b >= xs[(c2, a2)] + xs[(c1, a1)] - 1
b = LpVariable('b_{}_{}_{}_{}'.format(c1, a2, c2, a1),
cat=LpBinary)
betas[(c1, a2, c2, a1)] = b
pb += b <= xs[(c2, a1)]
pb += b <= xs[(c1, a2)]
pb += b >= xs[(c1, a2)] + xs[(c2, a1)] - 1
# Set objective: communication + hosting_cost
pb += _objective(xs, betas, route, msg_load, hosting_cost), \
'Communication costs and prefs'
# Adding constraints:
# Constraints: Memory capacity for all agents.
for a in agt_names:
pb += lpSum([footprint(i) * xs[i, a] for i in comp_names])\
<= capacity(a), \
'Agent {} capacity'.format(a)
# Constraints: all computations must be hosted.
for c in comp_names:
pb += lpSum([xs[c, a] for a in agt_names]) == 1, \
'Computation {} hosted'.format(c)
# solve using GLPK
status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False,
options=['--pcost']))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
logger.debug('GLPK cost : %s', value(pb.objective))
# print('BETAS:')
# for c1, a1, c2, a2 in betas:
# print(' ', c1, a1, c2, a2, value(betas[(c1, a1, c2, a2)]))
#
# print('XS:')
# for c, a in xs:
# print(' ', c, a, value(xs[(c, a)]))
mapping = {}
for k in agt_names:
agt_computations = [i for i, ka in xs
if ka == k and value(xs[(i, ka)]) == 1]
# print(k, ' -> ', agt_computations)
mapping[k] = agt_computations
return mapping
def distribute_factors(
agents: Dict[str, AgentDef],
cg: ComputationGraph,
footprints: Dict[str, float],
mapping: Dict[str, List[str]],
msg_load: Callable[[str, str], float],
) -> Dict[str, List[str]]:
"""
Optimal distribution of factors on agents.
Parameters
----------
cg: computations graph
agents: dict
a dict {agent_name : AgentDef} containing all available agents
Returns
-------
a dict { agent_name: list of factor names}
"""
pb = LpProblem("ilp_factors", LpMinimize)
# build the inverse mapping var -> agt
inverse_mapping = {} # type: Dict[str, str]
for a in mapping:
inverse_mapping[mapping[a][0]] = a
# One binary variable xij for each (variable, agent) couple
factor_names = [n.name for n in cg.nodes if isinstance(n, FactorComputationNode)]
xs = LpVariable.dict("x", (factor_names, agents), cat=LpBinary)
logger.debug("Binary variables for factor distribution : %s", xs)
# Hard constraints: respect agent's capacity
for a in agents:
# Footprint of the variable this agent is already hosting:
v_footprint = footprints[mapping[a][0]]
pb += (
lpSum([footprints[fn] * xs[fn, a] for fn in factor_names])
<= (agents[a].capacity - v_footprint),
"Agent {} capacity".format(a),
)
# Hard constraints: all computations must be hosted.
for c in factor_names:
pb += lpSum([xs[c, a] for a in agents]) == 1, "Factor {} hosted".format(c)
# 1st objective : minimize communication costs:
comm = LpAffineExpression()
for (fn, an_f) in xs:
for vn in cg.neighbors(fn):
an_v = inverse_mapping[vn] # agt hosting neighbor var vn
comm += agents[an_f].route(an_v) * msg_load(vn, fn) * xs[(fn, an_f)]
# 2st objective : minimize hosting costs
hosting = lpSum([agents[a].hosting_cost(c) * xs[(c, a)] for c, a in xs])
# agregate the two objectives using RATIO_HOST_COMM
pb += lpSum([RATIO_HOST_COMM * comm, (1 - RATIO_HOST_COMM) * hosting])
# solve using GLPK and convert to mapping { agt_name : [factors names]}
status = pb.solve(solver=GLPK_CMD(keepFiles=1, msg=False, options=["--pcost"]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException(
"No possible optimal distribution for factors"
)
logger.debug("GLPK cost : %s", value(pb.objective))
mapping = {} # type: Dict[str, List[str]]
for k in agents:
agt_computations = [i for i, ka in xs if ka == k and value(xs[(i, ka)]) == 1]
# print(k, ' -> ', agt_computations)
mapping[k] = agt_computations
logger.debug("Factors distribution : %s ", mapping)
return mapping
def cg_secp_ilp(
cg: ComputationConstraintsHyperGraph,
agents: List[AgentDef],
already_assigned: Distribution,
computation_memory: Callable[[ComputationNode], float],
communication_load: Callable[[ComputationNode, str], float],
timeout=600, # Max 10 min
) -> Distribution:
start_t = time.time()
agents = list(agents)
agents_names = [a.name for a in agents]
# Only keep computations for which we actually need to find an agent.
comps_to_host = [
c for c in cg.node_names() if not already_assigned.has_computation(c)
]
# x_i^k : binary variable indicating if var x_i is hosted on agent a_k.
xs = _build_cs_binvar(comps_to_host, agents_names)
# alpha_ijk : binary variable indicating if x_i and f_j are both on a_k.
alphas = _build_alphaijk_binvars(cg, agents_names)
logger.debug(f"alpha_ijk {alphas}")
# LP problem with objective function (total communication cost).
pb = LpProblem("distribution", LpMinimize)
pb += (
_objective_function(cg, communication_load, alphas, agents_names),
"Communication costs",
)
# Constraints.
# All variable computations must be hosted:
for i in comps_to_host:
pb += (
lpSum([xs[(i, k)] for k in agents_names]) == 1,
"var {} is hosted".format(i),
)
# Each agent must host at least one computation:
# We only need this constraints for agents that do not already host a
# computation:
empty_agents = [
a for a in agents_names if not already_assigned.computations_hosted(a)
]
for k in empty_agents:
pb += (
lpSum([xs[(i, k)] for i in comps_to_host]) >= 1,
"atleastone {}".format(k),
)
# Memory capacity constraint for agents
for a in agents:
# Decrease capacity for already hosted computations
capacity = a.capacity - sum([
secp_computation_memory_in_cg(c, cg, computation_memory)
for c in already_assigned.computations_hosted(a.name)
])
pb += (
lpSum([
secp_computation_memory_in_cg(i, cg, computation_memory) *
xs[(i, a.name)] for i in comps_to_host
]) <= capacity,
"memory {}".format(a.name),
)
# Linearization constraints for alpha_ijk.
for (i, j), k in alphas:
if i in comps_to_host and j in comps_to_host:
pb += alphas[((i, j), k)] <= xs[(i, k)], "lin1 {}{}{}".format(
i, j, k)
pb += alphas[((i, j), k)] <= xs[(j, k)], "lin2 {}{}{}".format(
i, j, k)
pb += (
alphas[((i, j), k)] >= xs[(i, k)] + xs[(j, k)] - 1,
"lin3 {}{}{}".format(i, j, k),
)
elif i in comps_to_host and j not in comps_to_host:
# Var is free, factor is already hosted
if already_assigned.agent_for(j) == k:
pb += alphas[((i, j), k)] == xs[(i, k)]
else:
pb += alphas[((i, j), k)] == 0
elif i not in comps_to_host and j in comps_to_host:
# if i is not in vars_vars_to_host, it means that it's a
# computation that is already hosted (from hints)
if already_assigned.agent_for(i) == k:
pb += alphas[((i, j), k)] == xs[(j, k)]
else:
pb += alphas[((i, j), k)] == 0
else:
# i and j are both alredy hosted
if (already_assigned.agent_for(i) == k
and already_assigned.agent_for(j) == k):
pb += alphas[((i, j), k)] == 1
else:
pb += alphas[((i, j), k)] == 0
# the timeout for the solver must be monierd by the time spent to build the pb:
remaining_time = round(timeout - (time.time() - start_t)) - 2
# Now solve our LP
status = pb.solve(
GLPK_CMD(keepFiles=0,
msg=False,
options=["--pcost", "--tmlim",
str(remaining_time)]))
if status != LpStatusOptimal:
raise ImpossibleDistributionException("No possible optimal"
" distribution ")
else:
logger.debug("GLPK cost : %s", pulp.value(pb.objective))
comp_dist = already_assigned
for k in agents_names:
agt_vars = [
i for i, ka in xs if ka == k and pulp.value(xs[(i, ka)]) == 1
]
comp_dist.host_on_agent(k, agt_vars)
return comp_dist
def get_optimal_ec2(cpu_usage, mem_usage, optimal=True, debug=False):
"""
Calculates the optimal allocation of resources over a certain time span (with monthly granularity).
Formulates the problem of satisfying user demand (in CPU and RAM) as an LP problem with a monetary objective function.
Returns allocation of reserved instances and a total price for running the allocation on AWS.
"""
assert(len(cpu_usage) == len(mem_usage))
prob = LpProblem("The Simplified EC2 cost optimization", LpMinimize)
# variables
## 1h instances (both on-demand and reserved)
per_h_ondems = []
per_h_reserved = []
for p in range(len(cpu_usage)):
# ondemand
per_h_ondems += ["p %s ondem %s" %(p, i) for i in vms.keys()]
# reserved
per_h_reserved += ["p %s reserved %s" %(p, i) for i in vms.keys()]
## nr of 1-year reserved instances
nr_of_1year_reserved = [ "res_1year %s" % i for i in vms.keys()]
nr_of_3year_reserved = [ "res_3year %s" % i for i in vms.keys()]
category = LpInteger if optimal else LpContinuous
vars = LpVariable.dicts("aws", per_h_ondems + per_h_reserved + nr_of_1year_reserved + nr_of_3year_reserved, \
lowBound = 0, upBound = None, cat = category)
# objective function
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][0] for vm in per_h_ondems]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[3]][3] for vm in per_h_reserved]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[1]][1] for vm in nr_of_1year_reserved]) \
+ lpSum([vars[vm] * vms[vm.split(" ")[1]][2] for vm in nr_of_3year_reserved]) \
, "Total cost of running the infrastructure consuming (CPU/RAM)/h"
# constraints
## demand constraints
for p in range(len(cpu_usage)):
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][4] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= cpu_usage[p], "CPU demand, period %s" %p
prob += lpSum([vars[vm] * vms[vm.split(" ")[3]][5] for vm in (per_h_ondems + per_h_reserved) if int(vm.split(" ")[1]) == p]) >= mem_usage[p], "RAM demand. period %s" %p
## constraints on the reserved instances - cannot use more than we paid for
for i in per_h_reserved:
t = i.split(" ")[3]
prob += vars["res_1year %s" % t] + vars["res_3year %s" % t] >= vars[i], "Nr. of used reserved machines of type %s" %i
prob.solve()
if debug:
print "Status:", LpStatus[prob.status]
print "Total Cost of the solution = ", value(prob.objective)
res_instance = {}
for v in prob.variables():
if v.name.startswith("aws_res_") and v.varValue != 0.0:
res_instance[v.name] = v.varValue
if debug and v.varValue != 0.0:
print v.name, "=", v.varValue
return (res_instance, value(prob.objective))