def opt(e_init, e_target, hrv_hist, hrv_en):
# verbosity is how much log is reported back from CPlex. 3 is the most verbose
verbosity = 3
m = CPlexModel(verbosity)
b = m.new((epochs_per_day, nodes, mod_levels),
vtype=int,
lb=0,
ub=1,
name='b')
l = m.new((epochs_per_day, nodes, bin_num),
vtype=float,
lb=-1,
ub=battery_cap,
name='l')
fixed_prob = np.linespace(0, 1, num=bin_num, endpoint=True, dtype=float)
e_init_hist = np.zeros(e_init.shape, dtype=float)
e_init_hist[:, 0] = 1
hist_rv = np.zeros((epochs_per_day, nodes, bin_num), dtype=float)
hist_rv[0] = e_init_hist
# prepare the energy vector here
en_rv = np.zeros((epochs_per_day, nodes, bin_num), dtype=float)
en_rv[0] = e_init
for i in xrange(1, epochs_per_day):
en_rv[i], hist_rv[i] = next_battery_level(en_rv[i - 1], hist_rv[i - 1],\
hrv_en[i, :, :] - (np.vectorize(energy))(b[i, :]), hrv_hist[i, :, :])
m.constrain(en_rv[i] >= 0)
m.constrain(sum(np.vectorize(time)(b[i, :])) <= D)
m.maximize(objective_function(en_rv[-1], hist_rv[-1]))
return m
def decomposePiLP(S, A, T, s0, terminal, rawX, x, gamma=1):
"""
DEPRECATED.
This tries to decouple a policy into the optimal policy (following no constraints) and another policy \pi'.
\pi' may be a dominating policy.
Described in Eq. 2 on Aug.29, 2017.
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
Sr = range(len(S))
Ar = range(len(A))
y = m.new((len(S), len(A)), lb=0, name='y')
sigma = m.new(lb=0, ub=1, name='sigma')
for s in Sr:
for a in Ar:
# note that x and rawX use S x A as domains
m.constrain(sigma * rawX[S[s], A[a]] + y[s, a] == x[S[s], A[a]])
# make sure y is a valid occupancy
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
# and make sure there is no flow back from the terminal states
if not terminal(S[sp]):
m.constrain(sum(y[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[s]))) for s in Sr for a in Ar) == (1 - sigma) * (S[sp] == s0))
else:
m.constrain(sum(y[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[sp]))) for s in Sr for a in Ar) == (1 - sigma) * (S[sp] == s0))
obj = m.maximize(sigma)
# return sigma and the value of y
return obj, {(S[s], A[a]): m[y][s, a] for s in Sr for a in Ar}
def decomposePiLP(S, A, T, s0, terminal, rawX, x, gamma=1):
"""
This tries to decouple a policy into the optimal policy (following no constraints) and another policy \pi'.
\pi' may be a dominating policy.
Described in Eq. 2 on Aug.29, 2017.
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
Sr = range(len(S))
Ar = range(len(A))
y = m.new((len(S), len(A)), lb=0, name='y')
sigma = m.new(lb=0, ub=1, name='sigma')
for s in Sr:
for a in Ar:
# note that x and rawX use S x A as domains
m.constrain(sigma * rawX[S[s], A[a]] + y[s, a] == x[S[s], A[a]])
# make sure y is a valid occupancy
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
# and make sure there is no flow back from the terminal states
if not terminal(S[sp]):
m.constrain(sum(y[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[s]))) for s in Sr for a in Ar) == (1 - sigma) * (S[sp] == s0))
else:
m.constrain(sum(y[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[sp]))) for s in Sr for a in Ar) == (1 - sigma) * (S[sp] == s0))
obj = m.maximize(sigma)
# return sigma and the value of y
return obj, {(S[s], A[a]): m[y][s, a] for s in Sr for a in Ar}
def lpDualCPLEX(mdp,
zeroConstraints=(),
positiveConstraints=(),
positiveConstraintsOcc=1):
"""
DEPRECATED since we moved to gurobi. but leave the function here for sanity check
Solve the dual problem of lp, maybe with some constraints
Same arguments
Note that this is a lower level function that does not consider feature extraction.
r should be a reward function, not a reward parameter.
"""
S = mdp.S
A = mdp.A
T = mdp.T
r = mdp.r
gamma = mdp.gamma
alpha = mdp.alpha
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
Sr = range(len(S))
Ar = range(len(A))
x = m.new((len(S), len(A)), lb=0, name='x')
# make sure x is a valid occupancy
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
m.constrain(
sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]))
for s in Sr for a in Ar) == alpha(S[sp]))
# == constraints
if len(zeroConstraints) > 0:
m.constrain(
sum(x[S.index(s), A.index(a)] for s, a in zeroConstraints) == 0)
# >= constraints
if len(positiveConstraints) > 0:
m.constrain(
sum(x[S.index(s), A.index(a)]
for s, a in positiveConstraints) >= positiveConstraintsOcc)
# obj
try:
obj = m.maximize(sum([x[s, a] * r(S[s], A[a]) for s in Sr
for a in Ar]))
except CPlexException as err:
print 'Exception', err
# we return obj value as None and occ measure as {}. this should be handled correctly
return {'feasible': False}
return {
'feasible': True,
'obj': obj,
'pi': {(S[s], A[a]): m[x][s, a]
for s in Sr for a in Ar}
}
def lwaLP(graph_object):
capabilites = graph_object.capabilities
no_vertices = graph_object.n
attacker_strategy = list()
m = CPlexModel()
cv = m.new(no_vertices, vtype=float, ub=1, lb=0)
U = m.new(vtype=float)
diag = np.diag(capabilites)
m.constrain(U <= -diag * (1 - cv))
m.constrain(sum(cv) <= graph_object.R)
m.maximize(U)
for i in xrange(m[cv]):
if m[cv][i] > 0:
attacker_strategy.append(i)
start = findStrategySet(m[cv], graph_object.R)
return start, attacker_strategy
def domPiMilp(S, A, r, T, s0, terminal, domPis, consIdx, gamma=1):
"""
Finding dominating policies by representing constraints as possible negative rewards.
Described in the report on aug.19, 2017.
"""
rmax = 10000
M = 0.001
consLen = len(consIdx)
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# state range
Sr = range(len(S))
# action range
Ar = range(len(A))
# decision variables
x = m.new((len(S), len(A)), lb=0, name='x')
z = m.new(consLen, vtype=bool, name='z')
#z = [0, 1, 0] # test for office nav domain
t = m.new(name='t')
# flow conservation
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
# and make sure there is no flow back from the terminal states
if not terminal(S[sp]):
m.constrain(sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[s]))) for s in Sr for a in Ar) == (S[sp] == s0))
#print S[sp], [(S[s], A[a]) for s in Sr for a in Ar if T(S[s], A[a], S[sp]) > 0]
else:
m.constrain(sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[sp]))) for s in Sr for a in Ar) == (S[sp] == s0))
# t is the lower bound of the difference between x and y
# note: i don't think expressions in constraints can call other functions
for y in domPis:
# note that y is indexed by elements in S x A, not numbered indices
m.constrain(sum(x[s, a] * r(S[s], A[a]) for s in Sr for a in Ar) -\
sum(y[S[s], A[a]] *
(r(S[s], A[a]) + sum(- rmax * (S[s][consIdx[i]] != s0[consIdx[i]]) * z[i] for i in range(consLen)))\
for s in Sr for a in Ar)\
>= t)
for s in Sr:
for i in range(consLen):
if S[s][consIdx[i]] != s0[consIdx[i]]:
for a in Ar:
m.constrain(z[i] + M * x[s, a] <= 1)
# obj
obj = m.maximize(t)
print m[z]
return obj, {(S[s], A[a]): m[x][s, a] for s in Sr for a in Ar}
def milp(S, A, R, T, s0, psi, maxV):
"""
Solve the MILP problem in greedy construction of policy query
Args:
S: state set
A: action set
R: reward candidate set
T: transition function
s0: init state
psi: prior belief on rewards
maxV: maxV[i] = max_{\pi \in q} V_{r_i}^\pi
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
rLen = len(R)
M = 10000 # a large number
Sr = range(len(S))
Ar = range(len(A))
# decision variables
# FIXME i removed upper bound of x. it shoundn't have such bound without transient-state assumption, right?
x = m.new((len(S), len(A)), lb=0, name='x')
z = m.new(rLen, vtype=bool, name='z')
y = m.new(rLen, name='y')
# constraints on y
m.constrain([y[i] <= sum([x[s, a] * R[i](S[s], A[a]) for s in Sr for a in Ar]) - maxV[i] + (1 - z[i]) * M for i in xrange(rLen)])
m.constrain([y[i] <= z[i] * M for i in xrange(rLen)])
# constraints on x (valid occupancy)
for sp in Sr:
if S[sp] == s0:
m.constrain(sum([x[sp, ap] for ap in Ar]) == 1)
else:
m.constrain(sum([x[sp, ap] for ap in Ar]) == sum([x[s, a] * T(S[s], A[a], S[sp]) for s in Sr for a in Ar]))
# obj
obj = m.maximize(sum([psi[i] * y[i] for i in xrange(rLen)]))
if config.VERBOSE:
print 'obj', obj
print 'x', m[x]
print 'y', m[y]
print 'z', m[z]
# build occupancy as S x A -> x[.,.]
# z[i] == 1 then this policy is better than maxV on the i-th reward candidate
res = util.Counter()
for s in Sr:
for a in Ar:
res[S[s], A[a]] = m[x][s, a]
return res
def domPiMilp(S, A, r, T, s0, terminal, domPis, consIdx, gamma=1):
"""
Finding dominating policies by representing constraints as possible negative rewards.
Described in the report on aug.19, 2017.
"""
rmax = 10000
M = 0.001
consLen = len(consIdx)
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# state range
Sr = range(len(S))
# action range
Ar = range(len(A))
# decision variables
x = m.new((len(S), len(A)), lb=0, name='x')
z = m.new(consLen, vtype=bool, name='z')
#z = [0, 1, 0] # test for office nav domain
t = m.new(name='t')
# flow conservation
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
# and make sure there is no flow back from the terminal states
if not terminal(S[sp]):
m.constrain(sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[s]))) for s in Sr for a in Ar) == (S[sp] == s0))
#print S[sp], [(S[s], A[a]) for s in Sr for a in Ar if T(S[s], A[a], S[sp]) > 0]
else:
m.constrain(sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp]) * (not terminal(S[sp]))) for s in Sr for a in Ar) == (S[sp] == s0))
# t is the lower bound of the difference between x and y
# note: i don't think expressions in constraints can call other functions
for y in domPis:
# note that y is indexed by elements in S x A, not numbered indices
m.constrain(sum(x[s, a] * r(S[s], A[a]) for s in Sr for a in Ar) -\
sum(y[S[s], A[a]] *
(r(S[s], A[a]) + sum(- rmax * (S[s][consIdx[i]] != s0[consIdx[i]]) * z[i] for i in range(consLen)))\
for s in Sr for a in Ar)\
>= t)
for s in Sr:
for i in range(consLen):
if S[s][consIdx[i]] != s0[consIdx[i]]:
for a in Ar:
m.constrain(z[i] + M * x[s, a] <= 1)
# obj
obj = m.maximize(t)
print m[z]
return obj, {(S[s], A[a]): m[x][s, a] for s in Sr for a in Ar}
def milp(S, A, R, T, s0, psi, maxV):
"""
Solve the MILP problem in greedy construction of policy query
Args:
S: state set
A: action set
R: reward candidate set
T: transition function
s0: init state
psi: prior belief on rewards
maxV: maxV[i] = max_{\pi \in q} V_{r_i}^\pi
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
rLen = len(R)
M = 10000 # a large number
Sr = range(len(S))
Ar = range(len(A))
# decision variables
# FIXME i removed upper bound of x. it shoundn't have such bound without transient-state assumption, right?
x = m.new((len(S), len(A)), lb=0, name='x')
z = m.new(rLen, vtype=bool, name='z')
y = m.new(rLen, name='y')
# constraints on y
m.constrain([y[i] <= sum([x[s, a] * R[i](S[s], A[a]) for s in Sr for a in Ar]) - maxV[i] + (1 - z[i]) * M for i in xrange(rLen)])
m.constrain([y[i] <= z[i] * M for i in xrange(rLen)])
# constraints on x (valid occupancy)
for sp in Sr:
if S[sp] == s0:
m.constrain(sum([x[sp, ap] for ap in Ar]) == 1)
else:
m.constrain(sum([x[sp, ap] for ap in Ar]) == sum([x[s, a] * T(S[s], A[a], S[sp]) for s in Sr for a in Ar]))
# obj
obj = m.maximize(sum([psi[i] * y[i] for i in xrange(rLen)]))
if config.VERBOSE:
print 'obj', obj
print 'x', m[x]
print 'y', m[y]
print 'z', m[z]
# build occupancy as S x A -> x[.,.]
# z[i] == 1 then this policy is better than maxV on the i-th reward candidate
res = util.Counter()
for s in Sr:
for a in Ar:
res[S[s], A[a]] = m[x][s, a]
return res
def findUndominatedReward(mdpH, mdpR, newPi, humanPi, localDifferentPis,
domPis):
"""
Implementation of the linear programming problem (Eq.2) in report 12.5
Returns the objective value and a reward function (which is only useful when the obj value is > 0)
newPi is \hat{\pi} in the linear programming problem in the report.
The robot tries to see if there exists a reward function where newPi is better than the best policy in domPis.
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
S = mdpH.S
robotA = mdpR.A
humanA = mdpH.A
# index of states and actions
Sr = range(len(S))
robotAr = range(len(robotA))
humanAr = range(len(humanA))
r = m.new(len(S), lb=0, ub=1, name='r')
z = m.new(
name='z'
) # when the optimal value is attained, z = \max_{domPi \in domPis} V^{domPi}_r
for domPi in domPis:
m.constrain(z >= sum(
[domPi[S[s], robotA[a]] * r[s] for s in Sr for a in robotAr]))
# make sure r is consistent with humanPi
for s in S:
for a in humanA:
# humanPi is better than a locally different policy which takes action a in state a
m.constrain(sum(sum((humanPi[S[sp], humanA[ap]] - localDifferentPis[s, a][S[sp], humanA[ap]]) for ap in humanAr)\
* r[sp] for sp in Sr) >= 0)
# maxi_r { V^{newPi}_r - \max_{domPi \in domPis} V^{domPi}_r }
cplexObj = m.maximize(
sum(newPi[S[s], robotA[a]] * r[s] for s in Sr for a in robotAr) - z)
obj = sum([newPi[S[s], robotA[a]] * m[r][s] for s in Sr
for a in robotAr]) - m[z]
# the reward function has the same values for same states, but need to convert back to the S x A space
rFunc = lambda s, a: m[r][Sr.index(s)]
print 'cplexobj', cplexObj
print 'obj', obj
print 'newPi'
printPi(newPi)
print 'z', m[z], 'r', m[r]
return obj, rFunc
def findUndominatedReward(mdpH, mdpR, newPi, humanPi, localDifferentPis, domPis):
"""
Implementation of the linear programming problem (Eq.2) in report 12.5
Returns the objective value and a reward function (which is only useful when the obj value is > 0)
newPi is \hat{\pi} in the linear programming problem in the report.
The robot tries to see if there exists a reward function where newPi is better than the best policy in domPis.
"""
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
S = mdpH.S
robotA = mdpR.A
humanA = mdpH.A
# index of states and actions
Sr = range(len(S))
robotAr = range(len(robotA))
humanAr = range(len(humanA))
r = m.new(len(S), lb=0, ub=1, name='r')
z = m.new(name='z') # when the optimal value is attained, z = \max_{domPi \in domPis} V^{domPi}_r
for domPi in domPis:
m.constrain(z >= sum([domPi[S[s], robotA[a]] * r[s] for s in Sr for a in robotAr]))
# make sure r is consistent with humanPi
for s in S:
for a in humanA:
# humanPi is better than a locally different policy which takes action a in state a
m.constrain(sum(sum((humanPi[S[sp], humanA[ap]] - localDifferentPis[s, a][S[sp], humanA[ap]]) for ap in humanAr)\
* r[sp] for sp in Sr) >= 0)
# maxi_r { V^{newPi}_r - \max_{domPi \in domPis} V^{domPi}_r }
cplexObj = m.maximize(sum(newPi[S[s], robotA[a]] * r[s] for s in Sr for a in robotAr) - z)
obj = sum([newPi[S[s], robotA[a]] * m[r][s] for s in Sr for a in robotAr]) - m[z]
# the reward function has the same values for same states, but need to convert back to the S x A space
rFunc = lambda s, a: m[r][Sr.index(s)]
print 'cplexobj', cplexObj
print 'obj', obj
print 'newPi'
printPi(newPi)
print 'z', m[z], 'r', m[r]
return obj, rFunc
def lpDualCPLEX(mdp, zeroConstraints=[], positiveConstraints=[], positiveConstraintsOcc=1):
"""
Solve the dual problem of lp, maybe with some constraints
Same arguments
Note that this is a lower level function that does not consider feature extraction.
r should be a reward function, not a reward parameter.
"""
S = mdp.S
A = mdp.A
T = mdp.T
r = mdp.r
gamma = mdp.gamma
alpha = mdp.alpha
m = CPlexModel()
if not config.VERBOSE: m.setVerbosity(0)
# useful constants
Sr = range(len(S))
Ar = range(len(A))
x = m.new((len(S), len(A)), lb=0, name='x')
# make sure x is a valid occupancy
for sp in Sr:
# x (x(s) - \gamma * T) = \sigma
m.constrain(sum(x[s, a] * ((s == sp) - gamma * T(S[s], A[a], S[sp])) for s in Sr for a in Ar) == alpha(S[sp]))
# == constraints
if len(zeroConstraints) > 0:
m.constrain(sum(x[S.index(s), A.index(a)] for s, a in zeroConstraints) == 0)
# >= constraints
if len(positiveConstraints) > 0:
m.constrain(sum(x[S.index(s), A.index(a)] for s, a in positiveConstraints) >= positiveConstraintsOcc)
# obj
try:
obj = m.maximize(sum([x[s, a] * r(S[s], A[a]) for s in Sr for a in Ar]))
except CPlexException as err:
print 'Exception', err
# we return obj value as None and occ measure as {}. this should be handled correctly
return {'feasible': False}
return {'feasible': True, 'obj': obj, 'pi': {(S[s], A[a]): m[x][s, a] for s in Sr for a in Ar}}