def KellyCriterion(symbols, riskyRet, money=1e6, solver="cplex"):
'''
@riskyRet, shape: M*T
maximize W*R - 1/2W^T \simga W
'''
t = time.time()
mu = riskyRet.mean(axis=1)
print "mu:", mu
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
#decision variables
model.W = Var(model.symbols, within=NonNegativeReals)
#constraint
def CapitalConstraint_rule(model):
allocation = sum(model.W[idx] for idx in model.symbols)
return allocation == money
model.CapitalConstraint = Constraint()
#objective
def KellyObjective_rule(model):
profit = sum(model.W[idx]*mu[idx] for idx in model.symbols)
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
risk += model.W[idx] * model.W[jdx] * mu[idx]* mu[jdx]
return profit - 1./2 * risk
model.KellyObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "Kelly elapsed %.3f secs"%(time.time()-t)
def MeanVariance(symbols, riskyRet, money=1e6, risk_weight=1, solver="cplex"):
'''
@riskyRet, shape: M*T
minimize risk_weight * risk - (1-risk_weight) * mean
'''
t = time.time()
sigma = np.cov(riskyRet)
mu = riskyRet.mean(axis=1)
print "mu:", mu
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
#decision variables
model.W = Var(model.symbols, within=NonNegativeReals)
#constraint
def CapitalConstraint_rule(model):
allocation = sum(model.W[idx] for idx in model.symbols)
return allocation == money
model.CapitalConstraint = Constraint()
#objective
def minRiskObjective_rule(model):
profit = sum(model.W[idx] * mu[idx] for idx in model.symbols)
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
risk += model.W[idx] * model.W[jdx] * sigma[idx, jdx]
return 1. / 2 * risk_weight * risk - (1. - risk_weight) * profit
model.minRiskObjective = Objective(sense=minimize)
# Create a solver
opt = SolverFactory(solver)
if solver == "cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "MeanVariance elapsed %.3f secs" % (time.time() - t)
def KellyCriterion(symbols, riskyRet, money=1e6, solver="cplex"):
'''
@riskyRet, shape: M*T
maximize W*R - 1/2W^T \simga W
'''
t = time.time()
mu = riskyRet.mean(axis=1)
print "mu:", mu
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
#decision variables
model.W = Var(model.symbols, within=NonNegativeReals)
#constraint
def CapitalConstraint_rule(model):
allocation = sum(model.W[idx] for idx in model.symbols)
return allocation == money
model.CapitalConstraint = Constraint()
#objective
def KellyObjective_rule(model):
profit = sum(model.W[idx] * mu[idx] for idx in model.symbols)
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
risk += model.W[idx] * model.W[jdx] * mu[idx] * mu[jdx]
return profit - 1. / 2 * risk
model.KellyObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver == "cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "Kelly elapsed %.3f secs" % (time.time() - t)
def MeanVariance(symbols, risk_ret, money=1e6, risk_weight=1, solver="cplex"):
'''
@riskyRet, shape: M*T
minimize risk_weight * risk - (1-risk_weight) * mean
'''
t = time.time()
sigma = np.cov(risk_ret)
mu = risk_ret.mean(axis=1)
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
#decision variables
model.W = Var(model.symbols, within=NonNegativeReals)
#constraint
def CapitalConstraint_rule(model):
allocation = sum(model.W[idx] for idx in model.symbols)
return allocation == money
model.CapitalConstraint = Constraint()
#objective
def minRiskObjective_rule(model):
profit = sum(model.W[idx]*mu[idx] for idx in model.symbols)
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
risk += model.W[idx] * model.W[jdx] * sigma[idx, jdx]
return 1./2 * risk_weight * risk - (1. - risk_weight) * profit
model.minRiskObjective = Objective(sense=minimize)
# Create a solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "MeanVariance elapsed %.3f secs"%(time.time()-t)
def _main(argv):
print 'Arguments:', argv
if len(argv) < 5:
print 'Error: missing command line arguments'
print 'Specify: files_directory_path core_file time_file stoch_file'
return
dir_path = argv[1]
if not dir_path.endswith('/'):
dir_path += '/'
CORE_FILE_PATH = dir_path + argv[2]
TIME_FILE_PATH = dir_path + argv[3]
STOCH_FILE_PATH = dir_path + argv[4]
two_stage_builder = SmpsToTwoStageBuilder(CORE_FILE_PATH, TIME_FILE_PATH,
STOCH_FILE_PATH)
two_stage_problem = two_stage_builder.build_two_stage_instance()
det_equiv = DeterministicEquivalent(two_stage_problem)
model = det_equiv.create_model()
#solver = SolverFactory('cplex', solver_io='python')
solver = SolverFactory('cplex')
#model.pprint()
#det_equiv.solve('cbc')
det_equiv.solve(solver)
def IP():
t1 = time.time()
model = ConcreteModel()
#decision variable
model.x = Var(within=NonNegativeIntegers)
model.y = Var(within=NonNegativeIntegers)
#constraint
def rule1(model):
return (-model.x + model.y) <=1
model.c1 = Constraint(rule=rule1)
def rule2(model):
return (3 * model.x +2* model.y) <=12
model.c2 = Constraint(rule=rule2)
def rule3(model):
return (2*model.x + 3*model.y) <=12
model.c3 = Constraint(rule=rule3)
#objective
def obj_rule(model):
return model.y
model.obj = Objective(sense=maximize)
#optimizer
opt = SolverFactory('glpk')
instance = model.create()
results = opt.solve(instance)
print type(results)
print results
instance.load(results)
display(instance)
# maxy = results.Solution.Objective.__default_objective__['value']
# print "maxy:", maxy
print "elapsed %.3f secs"%(time.time()-t1)
def do_pyomo_work(profit_rate_windows):
Products = ['Doors', 'Windows']
ProfitRate = {'Doors': 300, 'Windows': profit_rate_windows}
Plants = ['Door Fab', 'Window Fab', 'Assembly']
HoursAvailable = {'Door Fab': 4, 'Window Fab': 12, 'Assembly': 18}
HoursPerUnit = {
('Doors', 'Door Fab'): 1,
('Windows', 'Window Fab'): 2,
('Doors', 'Assembly'): 3,
('Windows', 'Assembly'): 2,
('Windows', 'Door Fab'): 0,
('Doors', 'Window Fab'): 0
}
#Concrete Model
model = ConcreteModel()
#Decision Variables
model.WeeklyProd = Var(Products, within=NonNegativeReals)
#Objective
model.obj = Objective(expr=sum(ProfitRate[i] * model.WeeklyProd[i]
for i in Products),
sense=maximize)
def CapacityRule(model, p):
"""User Defined Capacity Rule
Accepts a pyomo Concrete Model as the first positional argument,
and a list of Plants as a second positional argument
"""
return sum(HoursPerUnit[i, p] * model.WeeklyProd[i]
for i in Products) <= HoursAvailable[p]
model.Capacity = Constraint(Plants, rule=CapacityRule)
opt = SolverFactory("glpk")
instance = model.create()
results = opt.solve(instance)
#results.write()
return results.Solution()
def WorstCVaRPortfolioSP(symbols,
riskyRet,
riskFreeRet,
allocatedWealth,
depositWealth,
buyTransFee,
sellTransFee,
alpha,
predictRiskyRet,
predictRiskFreeRet,
n_scenario,
probs=None,
solver="cplex"):
'''
two-stage stochastic programming
variable:
M: num. of symbols,
S: num. of scenarios
symbols, list of string,
riskRet, numpy.array, size: M
riskFreeRet, float
allocatedWealth, numpy.array, size: M
depositWealth, float,
buyTransFee, numpy.array, size: M
sellTransFee, numpy.array, size: M
alpha, float,
predictRiskRet, numpy.array, size: L * M * S, L個不確定的dist.
predictRiskFreeRet, float
probs, numpy.array, size: S
solver, string in {glpk or cplex}
'''
t = time.time()
assert len(predictRiskyRet.shape) == 3
n_dist = predictRiskyRet.shape[0]
if not probs:
probs = np.ones(n_scenario, dtype=np.float) / n_scenario
# Model
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
model.scenarios = range(n_scenario)
model.distributions = range(n_dist)
#decision variables
model.buys = Var(model.symbols, within=NonNegativeReals) #stage 1
model.sells = Var(model.symbols, within=NonNegativeReals) #stage 1
model.riskyWealth = Var(model.symbols, within=NonNegativeReals) #stage 2
model.riskFreeWealth = Var(within=NonNegativeReals) #stage 2
#aux variable, variable in definition of CVaR, equals to VaR at opt. sol.
model.Z = Var()
#aux variable, portfolio wealth less than than VaR (Z)
model.Ys = Var(model.distributions,
model.scenarios,
within=NonNegativeReals)
#constraint
def riskyWeathConstraint_rule(model, m):
'''
riskyWealth is a decision variable depending on both buys and sells.
(it means riskyWealth depending on scenario).
therefore
buys and sells are fist stage variable,
riskywealth is second stage variable
'''
return (
model.riskyWealth[m] == (1. + riskyRet[m]) * allocatedWealth[m] +
model.buys[m] - model.sells[m])
model.riskyWeathConstraint = Constraint(model.symbols)
def riskFreeWealthConstraint_rule(model):
'''
riskFreeWealth is decision variable depending on both buys and sells.
therefore
buys and sells are fist stage variable,
riskFreewealth is second stage variable
'''
totalSell = sum(
(1 - sellTransFee[m]) * model.sells[m] for m in model.symbols)
totalBuy = sum(
(1 + buyTransFee[m]) * model.buys[m] for m in model.symbols)
return (model.riskFreeWealth == (1. + riskFreeRet) * depositWealth +
totalSell - totalBuy)
model.riskFreeWealthConstraint = Constraint()
def WCVaRConstraint_rule(model, d, s):
'''auxiliary variable Y depends on scenario. CVaR <= VaR'''
wealth = sum((1. + predictRiskyRet[d, m, s]) * model.riskyWealth[m]
for m in model.symbols)
return model.Ys[d, s] >= (model.Z - wealth)
model.WCVaRConstraint = Constraint(model.distributions, model.scenarios)
#objective
def WCVaRObjective_rule(model):
val = 0
for d in model.distributions:
for s in model.scenarios:
val += model.Ys[d, s] * probs[s]
val *= 1 / (1 - alpha)
val = model.Z - val
return val
model.WCVaRObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver == "cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
WCVaR = results.Solution.Objective.__default_objective__['value']
# display(instance)
M = len(symbols)
results = {"WCVaR": WCVaR}
for v in instance.active_components(Var):
# print "Variable",v
varobject = getattr(instance, v)
if v == "buys":
results[v] = np.fromiter(
(varobject[index].value for index in varobject), np.float)
elif v == "sells":
results[v] = np.fromiter(
(varobject[index].value for index in varobject), np.float)
elif v == "Z":
results["VaR"] = varobject.value
# print results
print "WCVaR:", WCVaR
print "WorstCVaRPortfolioSP elapsed %.3f secs" % (time.time() - t)
return results
# Constraints - they have to obey the ODEs
def ode_1(model):
return model.sv*c_af - model.sv*model.ca - k1*model.ca - 2.0*k3*model.ca**2.0 == 0
def ode_2(model):
return -model.sv*model.cb - k1*model.ca - k2*model.cb == 0
def ode_3(model):
return -model.sv*model.cc + k2*model.cb == 0
def ode_4(model):
return -model.sv*model.cd + k3*model.ca**2.0 == 0
model.ca_bal = Constraint(rule=ode_1)
model.cb_bal = Constraint(rule=ode_2)
model.cc_bal = Constraint(rule=ode_3)
model.cd_bal = Constraint(rule=ode_4)
#This is an optional code path that allows the script to be run outside of
#pyomo command-line. For example: python cstr.py
if __name__ == '__main__':
#This replicates what the pyomo command-line tools does
from coopr.opt import SolverFactory
opt = SolverFactory("glpk")
instance = model.create()
results = opt.solve(instance)
#sends results to stdout
results.write()
def abstractPharmacy():
t1 = time.time()
#build model
model = AbstractModel()
#set
model.drug = Set()
model.material = Set()
model.budget = Set()
#parameter
model.sellPrice = Param(model.drug)
model.grams = Param(model.drug)
model.HRhours = Param(model.drug)
model.EQhours = Param(model.drug)
model.EQCost = Param(model.drug)
model.materialCost = Param(model.material, mutable=True)
model.materialContent = Param(model.material)
model.budgeValue = Param(model.budget)
#decision variables
model.D = Var(model.drug, domain=NonNegativeReals)
model.R = Var(model.material, domain=NonNegativeReals)
#objective
def objective_rule(model):
return summation(model.sellPrice, model.D) - \
summation(model.EQCost, model.D) - \
summation(model.materialCost, model.R)
model.Objective = Objective(rule=objective_rule, sense=maximize)
#constraint
def balance_constraint_rule(model):
return summation(model.materialContent, model.R) - \
summation(model.grams, model.D) >= 0
def storage_constraint_rule(model):
return summation(model.R) <= model.budgeValue['storage']
def HR_constraint_rule(model):
return summation(model.HRhours, model.D) <= model.budgeValue['HR']
def EQ_constraint_rule(model):
return summation(model.EQhours, model.D) <= model.budgeValue['hour']
def money_constraint_rule(model):
return (summation(model.EQCost, model.D) +
summation(model.materialCost, model.R)) <=model.budgeValue['money']
model.balanceConstraint = Constraint(rule=balance_constraint_rule)
model.storageConstraint = Constraint(rule=storage_constraint_rule)
model.HRConstraint = Constraint(rule=HR_constraint_rule)
model.EQConstraint = Constraint(rule=EQ_constraint_rule)
model.moneyConstraint = Constraint(rule=money_constraint_rule)
# Create a model instance and optimize
# Create a solver
opt = SolverFactory('cplex')
data = DataPortal()
data.load(filename='pharmacy.dat')
instance = model.create(data)
instance.pprint()
results = opt.solve(instance)
print results
print "original elapsed %.3f secs"%(time.time()-t1)
t2 = time.time()
#change parameter and resolve
getattr(instance, "materialCost")[2] = 199
instance.preprocess()
results = opt.solve(instance)
print results
print "resolve, elapsed %.3f secs"%(time.time()-t2)
model.types=Param(model.tasks)
model.nums=Param(model.providers)
print model.v, model.types
#Decision Variables
model.x=Var(model.providers,domain=NonNegativeIntegers)
#Objective
def obj_expression(model):
return summation(model.v, model.x)
model.obj = Objective(rule=obj_expression, sense = maximize)
#Constraints
def ax_constraint_rule(model,i):
return sum(model.v[i][j] * model.x[j] for j in model.providers) > model.types[i]
model.AxbConstraint = Constraint(model.tasks, rule=ax_constraint_rule)
#This is an optional code path that allows the script to be run outside of
#pyomo command-line. For example: python wyndor.py
if __name__ == '__main__':
#This replicates what the pyomo command-line tools does
from coopr.opt import SolverFactory
opt = SolverFactory("glpk")
instance = model.create()
res_gne = opt.solve(instance)
#sends results to stdout
res_gne.write()
def __init__(self, cov, occ, groups_4digit, allele_table, beta, t_max_allele=2, solver="glpk", threads=1,
verbosity=0):
"""
Constructor
"""
self.__allele_table = allele_table
self.__beta = float(beta)
self.__t_max_allele = t_max_allele
self.__solver = SolverFactory(solver)
self.__solver.options["threads"] = threads
self.__verbosity = True if verbosity > 0 else False
self.__changed = True
self.__ks = 1
self.__groups_4digit = groups_4digit
loci_alleles = defaultdict(list)
for type_4digit, group_alleles in groups_4digit.iteritems():
#print type_4digit, group_alleles
loci_alleles[type_4digit.split('*')[0]].extend(group_alleles)
loci = loci_alleles
self.__allele_to_4digit = {allele: type_4digit for type_4digit, group in groups_4digit.iteritems() for allele in
group}
'''
generates the basic ILP model
'''
model = ConcreteModel()
#init Sets
model.LociNames = Set(initialize=loci.keys())
model.Loci = Set(model.LociNames, initialize=lambda m, l: loci[l])
L = list(itertools.chain(*loci.values()))
reconst = {allele_id: 0.01 for allele_id in L if '_' in allele_id}
R = set([r for (r, _) in cov.keys()])
model.L = Set(initialize=L)
model.R = Set(initialize=R)
#init Params
model.cov = Param(model.R, model.L, initialize=lambda model, r, a: cov.get((r, a), 0))
model.reconst = Param(model.L, initialize=lambda model, a: reconst.get(a, 0))
model.occ = Param(model.R, initialize=occ)
model.t_allele = Param(initialize=self.__t_max_allele, mutable=True)
model.beta = Param(initialize=self.__beta,
validate=lambda val, model: 0.0 <= float(self.__beta) <= 0.999,
mutable=True)
model.nof_loci = Param(initialize=len(loci))
#init variables
model.x = Var(model.L, domain=Binary)
model.y = Var(model.R, domain=Binary)
model.re = Var(model.R, bounds=(0.0, None))
model.hetero = Var(bounds=(0.0, model.nof_loci))
#init objective
model.read_cov = Objective(
rule=lambda model: sum(model.occ[r] * (model.y[r] - model.beta * (model.re[r])) for r in model.R) - sum(
model.reconst[a] * model.x[a] for a in model.L), sense=maximize)
#init Constraints
model.max_allel_selection = Constraint(model.LociNames, rule=lambda model, l: sum(
model.x[a] for a in model.Loci[l]) <= model.t_allele)
model.min_allel_selection = Constraint(model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]) >= 1)
model.is_read_cov = Constraint(model.R,
rule=lambda model, r: sum(model.cov[r, a] * model.x[a] for a in model.L) >=
model.y[r])
model.heterozygot_count = Constraint(
rule=lambda model: model.hetero >= sum(model.x[a] for a in model.L) - model.nof_loci)
#regularization constraints
model.reg1 = Constraint(model.R, rule=lambda model, r: model.re[r] <= model.nof_loci * model.y[r])
model.reg2 = Constraint(model.R, rule=lambda model, r: model.re[r] <= model.hetero)
model.reg3 = Constraint(model.R,
rule=lambda model, r: model.re[r] >= model.hetero - model.nof_loci * (1 - model.y[r]))
#generate constraint list for solution enumeration
model.c = ConstraintList()
#generate instance
self.__instance = model.create()
def maxRetPortfolio(riskyRetMtx, riskFreeRetVec,
buyTransFeeMtx, sellTransFeeMtx,
allocatedWealth):
'''
-假設有T期的資料, 投資在M個risky assets, 以及1個riskfree asset
-在(T+1)期初結算
goal: 最大化T+1期期初財產
@param riskyRetMtx, numpy.array, size: M * T+1
@param riskFreeRetVec, numpy.array, size: T+1
@param buyTransFeeMtx, numpy.array, size: M * T
@param sellTransFeeMtx, numpy.array, size: M * T
@param allocatedWealth, numpy.array, size: (M+1) (最後一個為cash)
@return (buyMtx, sellMtx), numpy.array, each size: M*T
'''
assert buyTransFeeMtx.shape == sellTransFeeMtx.shape
assert riskyRetMtx.shape[1] == riskFreeRetVec.size
M, T = buyTransFeeMtx.shape
t1 = time.time()
#create model
model = ConcreteModel()
#number of asset and number of periods
model.symbols = range(M)
model.T = range(T)
model.Tp1 = range(T+1)
model.Mp1 = range(M+1)
#decision variables
model.buys = Var(model.symbols, model.T, within=NonNegativeReals)
model.sells = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskyWealth = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskFreeWealth = Var(model.T, within=NonNegativeReals)
#objective
def objective_rule(model):
wealth =sum( (1. + riskyRetMtx[m, T]) * model.riskyWealth[m, T-1]
for m in xrange(M))
wealth += (1.+ riskFreeRetVec[T]) * model.riskFreeWealth[T-1]
return wealth
model.objective = Objective(rule=objective_rule, sense=maximize)
#constraint
def riskyWealth_constraint_rule(model, m, t):
if t>=1:
preWealth = model.riskyWealth[m, t-1]
else:
preWealth = allocatedWealth[m]
return (model.riskyWealth[m, t] ==
(1. + riskyRetMtx[m,t])*preWealth +
model.buys[m,t] - model.sells[m,t])
def riskyFreeWealth_constraint_rule(model, t):
totalSell = sum((1-sellTransFeeMtx[mdx, t])*model.sells[mdx, t]
for mdx in xrange(M))
totalBuy = sum((1+buyTransFeeMtx[mdx, t])*model.buys[mdx, t]
for mdx in xrange(M))
if t >=1:
preWealth = model.riskFreeWealth[t-1]
else:
preWealth = allocatedWealth[-1]
return( model.riskFreeWealth[t] ==
(1. + riskFreeRetVec[t])*preWealth +
totalSell - totalBuy)
model.riskyWeathConstraint = Constraint(model.symbols, model.T,
rule=riskyWealth_constraint_rule)
model.riskFreeWealthConstraint = Constraint(model.T,
rule=riskyFreeWealth_constraint_rule)
#optimizer
opt = SolverFactory('cplex')
# opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
print results
# instance.load(results)
# display(instance)
# instance.load(results)
# for var in instance.active_components(Var):
# varobj = getattr(instance, var)
# for idx in varobj:
# print varobj[idx], varobj[idx].value
#
print "elapsed %.3f secs"%(time.time()-t1)
def parse_args():
import argparse
from coopr.opt import SolverFactory as SF
from pyutilib.component.core import PluginGlobals
logger = PluginGlobals.env().log
logger.disabled = True # no need for warnings: it's what we're testing!
available_solvers = set(solver # name of solver; a string
for solver in filter(lambda x: '_' != x[0], SF.services())
if SF(solver).available(False)
)
logger.disabled = False
if available_solvers:
if 'cplex' in available_solvers:
default_solver = 'cplex'
elif 'gurobi' in available_solvers:
default_solver = 'gurobi'
elif 'cbc' in available_solvers:
default_solver = 'cbc'
elif 'glpk' in available_solvers:
default_solver = 'glpk'
else:
default_solver = available_solvers[0]
else:
default_solver = 'NONE'
SE.write('\nNOTICE: Coopr did not find any suitable solvers. Temoa will '
'not be able to solve any models. If you need help, ask on the '
'Temoa Project forum: http://temoaproject.org/\n\n')
parser = argparse.ArgumentParser()
graphviz = parser.add_argument_group('Graphviz Options')
solver = parser.add_argument_group('Solver Options')
parser.add_argument('dot_dat',
type=str,
nargs='+',
help='AMPL-format data file(s) with which to create a model instance. '
'e.g. "data.dat"'
)
graphviz.add_argument('--graph_format',
help='Create a system-wide visual depiction of the model. The '
'available options are the formats available to Graphviz. To get '
'a list of available formats, use the "dot" command: dot -Txxx. '
'[Default: None]',
action='store',
dest='graph_format',
default=None)
graphviz.add_argument('--show_capacity',
help='Choose whether or not the capacity shows up in the subgraphs. '
'[Default: not shown]',
action='store_true',
dest='show_capacity',
default=False)
graphviz.add_argument('--graph_type',
help='Choose the type of subgraph depiction desired. [Default: '
'separate_vintages]',
action='store',
dest='graph_type',
choices=('explicit_vintages', 'separate_vintages'),
default='separate_vintages')
graphviz.add_argument('--use_splines',
help='Choose whether the subgraph edges needs to be straight or curved.'
' [Default: use straight lines, not splines]',
action='store_true',
dest='splinevar',
default=False)
solver.add_argument('--solver',
help="Which backend solver to use. See 'pyomo --help-solvers' for a list "
'of solvers with which Coopr can interface. The list shown here is '
'what Coopr can currently find on this system. [Default: {}]'
.format(default_solver),
action='store',
choices=sorted(available_solvers),
dest='solver',
default=default_solver)
solver.add_argument('--symbolic_solver_labels',
help='When interfacing with the solver, use model-derived symbol names. '
'For example, "V_Capacity(coal_plant,2000)" instead of "x(47)". '
'Mainly used for debugging purposes. [Default: use x(47) style]',
action='store_true',
dest='useSymbolLabels',
default=False)
solver.add_argument('--generate_solver_lp_file',
help='Request that solver create an LP representation of the optimization '
'problem. Mainly used for model debugging purposes. The file name '
'will have the same base name as the first dot_dat file specified. '
'[Default: do not create solver LP file]',
action='store_true',
dest='generateSolverLP',
default=False)
solver.add_argument('--keep_coopr_lp_file',
help='Save the LP file as written by Pyomo. This is distinct from the '
"solver's generated LP file, but /should/ represent the same model. "
'Mainly used for debugging purposes. The file name will have the '
'same base name as the first dot_dat file specified. '
'[Default: remove Pyomo LP]',
action='store_true',
dest='keepPyomoLP',
default=False)
options = parser.parse_args()
return options
def KellySP(symbols,
riskyRet,
riskFreeRet,
allocatedWealth,
depositWealth,
buyTransFee,
sellTransFee,
predictRiskyRet,
predictRiskFreeRet,
n_scenario,
probs=None,
solver="cplex"):
'''
M: n_symbol, S: n_scenario
@symbols, list, size: M
@riskRet: np.array, size: M
@riskFreeRet: float
@allocatedWealth, np.array, size:M
@depositWealth, float
@buyTransFee, np.array, size:M
@sellTransFee, np.array, size:M
@riskyRet, shape: M*T
@predictRiskyRet, np.array, size: M*S
@predictRiskFreeRet, float
@n_scneario, int
@probs: np.array, size: S
maximize E(W*R - 1/2W^T \simga W)
'''
t = time.time()
if not probs:
probs = np.ones(n_scenario, dtype=np.float) / n_scenario
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
model.scenarios = range(n_scenario)
#decision variables
#stage 1
model.buys = Var(model.symbols, within=NonNegativeReals)
model.sells = Var(model.symbols, within=NonNegativeReals)
#stage 2
model.riskyWealth = Var(model.symbols, within=NonNegativeReals)
model.riskFreeWealth = Var(within=NonNegativeReals)
#constraint
def riskyWeathConstraint_rule(model, m):
'''
riskyWealth is a decision variable depending on both buys and sells.
(it means riskyWealth depending on scenario).
therefore
buys and sells are fist stage variable,
riskywealth is second stage variable
'''
return (
model.riskyWealth[m] == (1. + riskyRet[m]) * allocatedWealth[m] +
model.buys[m] - model.sells[m])
model.riskyWeathConstraint = Constraint(model.symbols)
def riskFreeWealthConstraint_rule(model):
'''
riskFreeWealth is decision variable depending on both buys and sells.
therefore
buys and sells are fist stage variable,
riskFreewealth is second stage variable
'''
totalSell = sum(
(1 - sellTransFee[m]) * model.sells[m] for m in model.symbols)
totalBuy = sum(
(1 + buyTransFee[m]) * model.buys[m] for m in model.symbols)
return (model.riskFreeWealth == (1. + riskFreeRet) * depositWealth +
totalSell - totalBuy)
model.riskFreeWealthConstraint = Constraint()
#objective
def TotalCostObjective_rule(model):
'''' E(W*R - 1/2W^T \simga W) '''
profit = sum(probs[s] * model.riskyWealth[symbol] *
predictRiskyRet[symbol, s] for symbol in model.symbols
for s in xrange(n_scenario))
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
for s in xrange(n_scenario):
risk += (model.riskyWealth[idx] * model.riskyWealth[jdx] *
predictRiskyRet[idx, s] * predictRiskyRet[jdx, s])
return profit - 1. / 2 * risk
model.TotalCostObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver == "cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "KellySP elapsed %.3f secs" % (time.time() - t)
def temoa_solve(model_data):
from sys import argv, version_info
if version_info < (2, 7):
msg = ("Temoa requires Python v2.7 to run.\n\nIf you've "
"installed Coopr with Python 2.6 or less, you'll need to reinstall "
'Coopr, taking care to install with a Python 2.7 (or greater) '
'executable.')
raise SystemExit(msg)
from time import clock
from coopr.opt import SolverFactory, SolverManagerFactory
from coopr.pyomo import ModelData
from utils import results_writer
from pformat_results import pformat_results
tee = False
solver_manager = SolverManagerFactory('serial')
options = parse_args()
dot_dats = options.dot_dat
opt = SolverFactory(options.solver)
if opt:
opt.keepFiles = options.keepPyomoLP
opt.generateSymbolicLabels = options.useSymbolLabels
if options.generateSolverLP:
opt.options.wlp = path.basename(options.dot_dat[0])[:-4] + '.lp'
SE.write('\nSolver will write file: {}\n\n'.format(opt.options.wlp))
elif options.solver != 'NONE':
SE.write("\nWarning: Unable to initialize solver interface for '{}'\n\n"
.format(options.solver))
SE.write('[ ] Reading data files.')
SE.flush()
# Recreate the pyomo command's ability to specify multiple "dot dat" files
# on the command line
begin = clock()
duration = lambda: clock() - begin
mdata = ModelData()
for f in dot_dats:
if f[-4:] != '.dat':
msg = "\n\nExpecting a dot dat (e.g., data.dat) file, found '{}'\n"
raise SystemExit(msg.format(f))
mdata.add(f)
mdata.read(model_data.model)
SE.write('\r[%8.2f\n' % duration())
SE.write('[ ] Creating Temoa model instance.')
SE.flush()
# Now do the solve and ...
model_data.instance = model_data.model.create(mdata)
SE.write('\r[%8.2f\n' % duration())
SE.write('[ ] Solving.')
SE.flush()
if opt:
model_data.result = solver_manager.solve(model_data.instance, opt=opt, tee=tee,
suffixes=['dual', 'rc'])
# result = opt.solve(instance)
SE.write('\r[%8.2f\n' % duration())
else:
SE.write('\r---------- Not solving: no available solver\n')
raise SystemExit
SE.write('[ ] Formatting results.')
SE.flush()
# ... print the easier-to-read/parse format
results_writer(model_data.result, model_data.instance)
# updated_results = instance.update_results(result)
# formatted_results = pformat_results(instance, updated_results)
SE.write('\r[%8.2f\n' % duration())
# SO.write(formatted_results)
if options.graph_format:
SE.write('[ ] Creating Temoa model diagrams.')
SE.flush()
instance.load(result)
CreateModelDiagrams(instance, options)
SE.write('\r[%8.2f\n' % duration())
if not (SO.isatty() or SE.isatty()):
SO.write("\n\nNotice: You are not receiving 'standard error' messages."
" Temoa uses the 'standard error' file to send meta information "
"on the progress of the solve. If you aren't intentionally "
"ignoring standard error messages, you may correct the issue by "
"updating coopr/src/coopr.misc/coopr/misc/scripts.py as per this "
"coopr changeset: "
"https://software.sandia.gov/trac/coopr/changeset/5363\n")
def optimal2DCopulaSampling(data, n_scenario = 20, solver="cplex"):
'''
the optimal samples close to the empirical copula functions
it can only to deal with bivariate samples
@data, numpy.array, size: n_rv * 2
'''
assert data.shape[1] == 2
n_rv = data.shape[0]
tgt_copula = buildEmpiricalCopula(data)
# Model
model = ConcreteModel()
#Set, dimension 1 and 2
model.x = range(n_scenario)
model.y = range(n_scenario)
#decision variables
model.X = Var(model.x, model.y, within=Binary)
model.yp = Var(model.x, model.y, within=NonNegativeReals)
model.yn = Var(model.x, model.y, within=NonNegativeReals)
#constraint
def rowConstraint_rule(model, x):
'''to ensure that each rank is used only once in each row'''
val = sum( model.X[x, j] for j in model.y)
return val == 1
model.rowConstraint = Constraint(model.x)
def columnConstraint_rule(model, y):
'''to ensure that each rank is used only once in each column'''
val = sum( model.X[i, y] for i in model.x)
return val == 1
model.columnConstraint = Constraint(model.y)
def copulaConstraint_rule(model, i, j):
'''bias constraint '''
val = 0
for kdx in xrange(i):
for ldx in xrange(j):
val += model.X[kdx, ldx]
val = val - model.yp[i, j] + model.yn[i, j]
point = [(i+1.)/n_scenario, (j+1.)/n_scenario]
copula_val = getCopula(tgt_copula, point)
print "point %s copula:%s, S*copula:%s"%(point, copula_val, n_scenario * copula_val )
return val == n_scenario * copula_val
model.copulaConstraint = Constraint(model.x, model.y)
#objective
def minBias_rule(model):
'''minimize the bias between the sampling and given CDF'''
val = 0
for idx in model.x:
for jdx in model.y:
val += (model.yp[idx, jdx] + model.yn[idx, jdx])
return val
model.minBias = Objective(sense=minimize)
# Create a solver
solver= solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
display(instance)
results = {}
for v in instance.active_components(Var):
varobject = getattr(instance, v)
if v == "X":
results[v] = np.fromiter((varobject[idx, jdx].value
for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
parsed = []
for row in xrange(n_scenario):
for col in xrange(n_scenario):
if results[v][row][col] >=0.9:
parsed.append((row, col))
parsed = (np.asarray(parsed) + 1)/n_scenario
results["parsed"] = parsed
elif v == "yp":
results[v] = np.fromiter((varobject[idx, jdx].value
for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
elif v == "yn":
results[v] = np.fromiter((varobject[idx, jdx].value
for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
# print results
return results
# on this system. This example was tested using Ipopt # version 3.10.2 from coopr.pyomo import * from coopr.opt import SolverFactory import coopr.environ import scipy import numpy import matplotlib ### Create the ipopt solver plugin using the ASL interface solver = 'ipopt' solver_io = 'nl' stream_solver = False # True prints solver output to screen keepfiles = False # True prints intermediate file names (.nl,.sol,...) opt = SolverFactory(solver,solver_io=solver_io) ### ### Create the example model model = ConcreteModel() model.x1 = Var(bounds=(1,5),initialize=1.0) model.x2 = Var(bounds=(1,5),initialize=5.0) model.x3 = Var(bounds=(1,5),initialize=5.0) model.x4 = Var(bounds=(1,5),initialize=1.0) model.obj = Objective(expr=model.x1*model.x4*(model.x1+model.x2+model.x3) + model.x3) model.inequality = Constraint(expr=model.x1*model.x2*model.x3*model.x4 >= 25.0) model.equality = Constraint(expr=model.x1**2 + model.x2**2 + model.x3**2 + model.x4**2 == 40.0) ### ### Declare all suffixes # Incoming Ipopt bound multipliers (obtained from solution)
def Run_FSNAC(problem_data, fixed_parameters,output_directory):
opt = SolverFactory("cplex")
options = Options()
opt.options.mip_tolerances_mipgap = .0001
opt.options.mip_tolerances_absmipgap = .0001
####################################################################
### Generate NACs
####################################################################
phiij = {}
OC = {}
prod = problem_data.product
SS = problem_data.SS
pb = {}
outcome = {}
for s in SS:
pb[s] = problem_data.List_of_Scenarios[s].probability
outcome[s] = problem_data.List_of_Scenarios[s].outcome
from Progressive_NAC import Progressive_NAC
phiij = Progressive_NAC(fixed_parameters, problem_data.product, problem_data.stage_gate,problem_data.SS, outcome)
###phi,phii,phij = Progressive_NAC(fixed_parameters, problem_data.product, problem_data.stage_gate,problem_data.SS, outcome)
####################################################################
### Generate Model
####################################################################
model = SAA_LP(problem_data.product, problem_data.stage_gate, problem_data.time_step, problem_data.resource_type,problem_data.SS,problem_data.resource_max,problem_data.gammaL,problem_data.gammaD,problem_data.duration,problem_data.trial_cost,problem_data.resource_required, problem_data.revenue_max,pb, problem_data.success,problem_data.Last_Time_Step, problem_data.last_trial, problem_data.running_revenue, problem_data.open_revenue, problem_data.discounting_factor, phiij, outcome)
###model = deLR(problem_data.product, problem_data.stage_gate, problem_data.time_step, problem_data.resource_type,problem_data.SS,problem_data.resource_max,problem_data.gammaL,problem_data.gammaD,problem_data.duration,problem_data.trial_cost,problem_data.resource_required, problem_data.revenue_max,pb, problem_data.success,problem_data.Last_Time_Step, problem_data.last_trial, problem_data.running_revenue, problem_data.open_revenue, problem_data.discounting_factor, phi, phii,phiij, outcome)
####################################################################
### Creating Instance
####################################################################
instance = model.create()
####################################################################
### Fix Parameters Based on Scenario Outcomes
####################################################################
list_o_fixes = {}
for s in SS:
for itms in fixed_parameters:
for jtms in fixed_parameters[itms]:
if len(jtms) == 0:
instance.Decision_X[itms[0],itms[1],itms[2] + 1,s].value = itms[3]
instance.Decision_X[itms[0],itms[1],itms[2] + 1,s].fixed = True
list_o_fixes[(itms[0],itms[1],itms[2] + 1, s)] = itms[3]
else:
###pdb.set_trace()
cntr = 0
for ktms in jtms:
if ktms[2] == 0:
if outcome[s][ktms[0]] == ktms[1]:
cntr += 1
else:
if outcome[s][ktms[0]] > ktms[1]:
cntr += 1
if cntr == len(jtms):
instance.Decision_X[itms[0],itms[1],itms[2] +1,s].value = itms[3]
instance.Decision_X[itms[0],itms[1],itms[2]+1,s].fixed = True
list_o_fixes[(itms[0],itms[1],itms[2] + 1, s)] = itms[3]
####################################################################
### Preprocess to Fix Decisions
####################################################################
del model
instance.preprocess()
####################################################################
### Solve
####################################################################
results= opt.solve(instance)
instance.load(results)
####################################################################
### Write Output
####################################################################
save_file = 'FSNAC Solution Details'
save_file2 = 'phiij'
save_file3 = 'model_file'
### Open save file
if not os.path.exists(output_directory):
os.makedirs(output_directory)
f = open(os.path.join(output_directory, save_file), "w")
transformed_results = instance.update_results(results)
tr = str(transformed_results)
f.write(tr + '\n')
f.close()
f = open(os.path.join(output_directory, save_file2), "w")
phiij = str(phiij)
f.write(phiij + '\n')
f.close()
del instance
del transformed_results
if results.solver.status == SolverStatus.ok and results.solver.termination_condition == TerminationCondition.optimal:
return results.solution.objective['__default_objective__']['Value']
else:
pdb.set_trace()
def __init__(self, model_data, kda_results, output_directory, mipgap=.001):
### Process model_data for deterministic solve
from MTSSP import PRDP_Data_Processing
self.problem_data = PRDP_Data_Processing.MTSSP_PRDP_Data_Processing(
model_data._data)
self.results = kda_results.output['results']
self.sp_realizations = kda_results.output['sub_problem_realizations']
opt = SolverFactory("cplex")
options = Options()
opt.options.mip_tolerances_mipgap = mipgap
### Generate Non-Anticipativity Constraints
self.PRDP_NAC_Generation()
self.Scen_Spec_Parms()
### Generate Model
from MSSP.defunction import de
wrm_model_start_time = time.clock()
model = de(
self.problem_data.product, self.problem_data.stage_gate,
self.problem_data.time_step, self.problem_data.resource_type,
self.problem_data.SS, self.problem_data.resource_max,
self.problem_data.gammaL, self.problem_data.gammaD,
self.problem_data.duration, self.problem_data.trial_cost,
self.problem_data.resource_required, self.problem_data.revenue_max,
self.pb, self.problem_data.success,
self.problem_data.Last_Time_Step, self.problem_data.last_trial,
self.problem_data.running_revenue, self.problem_data.open_revenue,
self.problem_data.discounting_factor, self.phi, self.phii,
self.phij, self.outcome)
### Create Instance
wrm_instnce_strt_time = time.clock()
instance = model.create()
del model
gc.collect()
for s in self.problem_data.SS:
### Calculate X
xbox = self.Calculate_X(self.problem_data.List_of_Scenarios[s])
### Fix X Values in Instance
for i in self.problem_data.product:
for j in self.problem_data.stage_gate:
for t in self.problem_data.stage_gate:
idx = self.problem_data.product.index(i)
jdx = self.problem_data.stage_gate.index(j)
tdx = self.problem_data.stage_gate.index(t)
if xbox[idx][jdx][tdx]:
instance.Decision_X[i, j, t, s].value == 1
### Solve Model
wrm_strt_time = time.clock()
output = opt.solve(instance, warmstart=True)
wrm_fnsh_time = time.clock()
WS_Solve_Time = wrm_fnsh_time - wrm_strt_time
WS_InstanceGen_Time = wrm_strt_time - wrm_instnce_strt_time
WS_ModelCreate_Time = wrm_instnce_strt_time - wrm_model_start_time
Warmstart_Total_Time = wrm_fnsh_time - wrm_model_start_time
### Clear Solution Variables
del instance
del output
model_start_time = time.clock()
### Recreate Model and Solve for Deterministic Time
model = de(
self.problem_data.product, self.problem_data.stage_gate,
self.problem_data.time_step, self.problem_data.resource_type,
self.problem_data.SS, self.problem_data.resource_max,
self.problem_data.gammaL, self.problem_data.gammaD,
self.problem_data.duration, self.problem_data.trial_cost,
self.problem_data.resource_required, self.problem_data.revenue_max,
self.pb, self.problem_data.success,
self.problem_data.Last_Time_Step, self.problem_data.last_trial,
self.problem_data.running_revenue, self.problem_data.open_revenue,
self.problem_data.discounting_factor, self.phi, self.phii,
self.phij, self.outcome)
instnce_strt_time = time.clock()
### Create Instance
instance = model.create()
### time and solve DE_Version
strt_time = time.clock()
de_results = opt.solve(instance)
fnsh_time = time.clock()
NWS_Solve_Time = fnsh_time - strt_time
NWS_InstanceGen_Time = strt_time - instnce_strt_time
NWS_ModelCreate_Time = instnce_strt_time - model_start_time
Total_Time = fnsh_time - model_start_time
### Load Results
instance.load(de_results)
### Transform Results
transformed_results = instance.update_results(de_results)
### Write File
self.Output_Write(transformed_results, Warmstart_Total_Time,
WS_Solve_Time, WS_InstanceGen_Time,
WS_ModelCreate_Time, Total_Time, NWS_Solve_Time,
NWS_InstanceGen_Time, NWS_ModelCreate_Time,
output_directory)
model.LimitAmountSold = Constraint(model.CROPS)
def EnforceQuotas_rule(model, i):
return (0.0, model.QuantitySubQuotaSold[i], model.PriceQuota[i])
model.EnforceQuotas = Constraint(model.CROPS)
# minimize: sum of StageCostVariables
# A active scenario objective equivalent to that generated by PySP is
# included here for informational purposes.
def total_cost_rule(model):
v1 = summation(model.PlantingCostPerAcre, model.DevotedAcreage )
v2 = summation(model.SubQuotaSellingPrice, model.QuantitySubQuotaSold)
v3 = summation(model.SuperQuotaSellingPrice, model.QuantitySuperQuotaSold)
v4 = summation(model.PurchasePrice, model.QuantityPurchased )
return v1 - v2 -v3 + v4
model.Total_Cost_Objective = Objective(rule=total_cost_rule, sense=minimize)
opt = SolverFactory('cplex')
# data = DataPortal()
# data.load(filename='farmer.dat')
instance = model.create('farmer.dat')
instance.pprint()
results = opt.solve(instance)
print results
def KellySP(symbols, riskyRet, riskFreeRet, allocatedWealth,
depositWealth, buyTransFee, sellTransFee,
predictRiskyRet, predictRiskFreeRet, n_scenario,
probs=None, solver="cplex"):
'''
M: n_symbol, S: n_scenario
@symbols, list, size: M
@riskRet: np.array, size: M
@riskFreeRet: float
@allocatedWealth, np.array, size:M
@depositWealth, float
@buyTransFee, np.array, size:M
@sellTransFee, np.array, size:M
@riskyRet, shape: M*T
@predictRiskyRet, np.array, size: M*S
@predictRiskFreeRet, float
@n_scneario, int
@probs: np.array, size: S
maximize E(W*R - 1/2W^T \simga W)
'''
t = time.time()
if not probs:
probs = np.ones(n_scenario, dtype=np.float)/n_scenario
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
model.scenarios = range(n_scenario)
#decision variables
#stage 1
model.buys = Var(model.symbols, within=NonNegativeReals)
model.sells = Var(model.symbols, within=NonNegativeReals)
#stage 2
model.riskyWealth = Var(model.symbols, within=NonNegativeReals)
model.riskFreeWealth = Var(within=NonNegativeReals)
#constraint
def riskyWeathConstraint_rule(model, m):
'''
riskyWealth is a decision variable depending on both buys and sells.
(it means riskyWealth depending on scenario).
therefore
buys and sells are fist stage variable,
riskywealth is second stage variable
'''
return (model.riskyWealth[m] ==
(1. + riskyRet[m]) * allocatedWealth[m] +
model.buys[m] - model.sells[m])
model.riskyWeathConstraint = Constraint(model.symbols)
def riskFreeWealthConstraint_rule(model):
'''
riskFreeWealth is decision variable depending on both buys and sells.
therefore
buys and sells are fist stage variable,
riskFreewealth is second stage variable
'''
totalSell = sum((1 - sellTransFee[m]) * model.sells[m]
for m in model.symbols)
totalBuy = sum((1 + buyTransFee[m]) * model.buys[m]
for m in model.symbols)
return (model.riskFreeWealth ==
(1. + riskFreeRet)* depositWealth +
totalSell - totalBuy)
model.riskFreeWealthConstraint = Constraint()
#objective
def TotalCostObjective_rule(model):
'''' E(W*R - 1/2W^T \simga W) '''
profit = sum(probs[s]* model.riskyWealth[symbol]* predictRiskyRet[symbol, s]
for symbol in model.symbols
for s in xrange(n_scenario))
risk = 0
for idx in model.symbols:
for jdx in model.symbols:
for s in xrange(n_scenario):
risk += (model.riskyWealth[idx] * model.riskyWealth[jdx] *
predictRiskyRet[idx, s]*predictRiskyRet[jdx, s])
return profit - 1./2*risk
model.TotalCostObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
obj = results.Solution.Objective.__default_objective__['value']
display(instance)
print "KellySP elapsed %.3f secs"%(time.time()-t)
index=(model.BARRAS, model.ESCENARIOS))
data.load(filename=path_datos+'data_zone.csv',
param=(model.zonal_rup, model.zonal_rdn),
index=model.ZONAS)
data.load(filename=path_datos+'data_config.csv',
param=model.config_value,
index=model.CONFIG)
data.load(filename=path_datos+'data_scenarios.csv',
set=model.ESCENARIOS)
print ("--- Creando Modelo.... ---")
instance = model.create(data)
opt = SolverFactory("cplex")
#### - - - - - - RESOLVIENDO LA OPTIMIZACION - - - - - - #######
print ('\n--------Resolviendo la optimizacion "%s"--------' % instance.config_value['scuc'])
results = opt.solve(instance, tee=True)
# results.write()
instance.load(results)
#### - - - - - - IMPRIMIENDO SI ES NECESARIO - - - - - - #######
if instance.config_value['debugging']:
print ("--- Exportando LP ---")
stdout_ = sys.stdout # Keep track of the previous value.
stream = cStringIO.StringIO()
sys.stdout = stream
print instance.pprint() # Here you can do whatever you want, import module1, call test
class OptiType(object):
"""
classdocs
"""
def __init__(self,
cov,
occ,
groups_4digit,
allele_table,
beta,
t_max_allele=2,
solver="glpk",
threads=1,
verbosity=0):
"""
Constructor
"""
self.__allele_table = allele_table
self.__beta = float(beta)
self.__t_max_allele = t_max_allele
self.__solver = SolverFactory(solver)
self.__threads = threads
self.__verbosity = True if verbosity > 0 else False
self.__changed = True
self.__ks = 1
self.__groups_4digit = groups_4digit
loci_alleles = defaultdict(list)
for type_4digit, group_alleles in groups_4digit.iteritems():
#print type_4digit, group_alleles
loci_alleles[type_4digit.split('*')[0]].extend(group_alleles)
loci = loci_alleles
self.__allele_to_4digit = {
allele: type_4digit
for type_4digit, group in groups_4digit.iteritems()
for allele in group
}
'''
generates the basic ILP model
'''
model = ConcreteModel()
#init Sets
model.LociNames = Set(initialize=loci.keys())
model.Loci = Set(model.LociNames, initialize=lambda m, l: loci[l])
L = list(itertools.chain(*loci.values()))
reconst = {allele_id: 0.01 for allele_id in L if '_' in allele_id}
R = set([r for (r, _) in cov.keys()])
model.L = Set(initialize=L)
model.R = Set(initialize=R)
#init Params
model.cov = Param(model.R,
model.L,
initialize=lambda model, r, a: cov.get((r, a), 0))
model.reconst = Param(model.L,
initialize=lambda model, a: reconst.get(a, 0))
model.occ = Param(model.R, initialize=occ)
model.t_allele = Param(initialize=self.__t_max_allele, mutable=True)
model.beta = Param(
initialize=self.__beta,
validate=lambda val, model: 0.0 <= float(self.__beta) <= 0.999,
mutable=True)
model.nof_loci = Param(initialize=len(loci))
#init variables
model.x = Var(model.L, domain=Binary)
model.y = Var(model.R, domain=Binary)
model.re = Var(model.R, bounds=(0.0, None))
model.hetero = Var(bounds=(0.0, model.nof_loci))
#init objective
model.read_cov = Objective(rule=lambda model: sum(
model.occ[r] * (model.y[r] - model.beta * (model.re[r]))
for r in model.R) - sum(model.reconst[a] * model.x[a]
for a in model.L),
sense=maximize)
#init Constraints
model.max_allel_selection = Constraint(
model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]
) <= model.t_allele)
model.min_allel_selection = Constraint(
model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]) >= 1)
model.is_read_cov = Constraint(
model.R,
rule=lambda model, r: sum(model.cov[r, a] * model.x[a]
for a in model.L) >= model.y[r])
model.heterozygot_count = Constraint(
rule=lambda model: model.hetero >= sum(model.x[a] for a in model.L
) - model.nof_loci)
#regularization constraints
model.reg1 = Constraint(
model.R,
rule=lambda model, r: model.re[r] <= model.nof_loci * model.y[r])
model.reg2 = Constraint(
model.R, rule=lambda model, r: model.re[r] <= model.hetero)
model.reg3 = Constraint(model.R,
rule=lambda model, r: model.re[r] >= model.
hetero - model.nof_loci * (1 - model.y[r]))
#generate constraint list for solution enumeration
model.c = ConstraintList()
#generate instance
self.__instance = model.create()
def set_beta(self, beta):
"""
Sets the parameter beta
"""
self.__changed = True
getattr(self.__instance,
str(self.__instance.beta)).set_value(float(beta))
def set_t_max_allele(self, t_max_allele):
"""
Sets the upper bound of alleles selected per loci
"""
self.__changed = True
getattr(self.__instance,
str(self.__instance.t_allele)).set_value(t_max_allele)
def solve(self, ks):
"""
solves the problem k times and discards the found solutions in the next run.
"""
d = defaultdict(
list
) #in there we store the typing +objective and generate afterwards a DatarFrame with it
if self.__changed or self.__ks != ks:
self.__ks = ks
for k in xrange(ks):
expr = 0
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
try:
if self.__threads > 1:
res = self.__solver.solve(self.__instance,
options="threads=" +
str(self.__threads),
tee=self.__verbosity)
else:
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
except:
print "WARNING: Solver does not support multi-threading. Please change the config " \
"file accordingly! Fall back to single-threading."
del self.__solver.options["threads"]
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
self.__instance.load(res)
#if self.__verbosity > 0:
# res.write(num=1)
if str(res.Solution.status) != 'optimal':
break
selected = []
indices = []
encountered_4digit = []
for j in self.__instance.x:
if self.__allele_to_4digit[j][0] in 'HJG':
if 0.99 <= self.__instance.x[j].value <= 1.01:
selected.append(j)
indices.append(j)
continue
if 0.99 <= self.__instance.x[j].value <= 1.01:
selected.append(j)
exp_i = 0
exp_i += self.__instance.x[j]
if self.__allele_to_4digit[j] in encountered_4digit:
continue
encountered_4digit.append(self.__allele_to_4digit[j])
for i_allele in self.__groups_4digit[
self.__allele_to_4digit[j]]:
if self.__instance.x[i_allele].value <= 0:
exp_i += self.__instance.x[i_allele]
indices.append(i_allele)
expr += (1.0 - exp_i)
zero_indices = set([j for j in self.__instance.x
]).difference(set(indices))
for j in zero_indices:
expr += self.__instance.x[j]
self.__instance.c.add(expr >= 1)
#if self.__verbosity > 0:
# print selected
# self.__instance.c.pprint()
aas = [
self.__allele_to_4digit[x].split('*')[0] for x in selected
]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(selected[i])
d[aas[i] + "2"].append(selected[i])
else:
d[aas[i] + str(c[aas[i]])].append(selected[i])
c[aas[i]] += 1
nof_reads = sum(
(self.__instance.occ[j] * self.__instance.y[j].value
for j in self.__instance.y))
#if self.__verbosity > 0:
# print "Obj", res.Solution.Objective.__default_objective__.Value
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
self.__instance.c.clear()
self.__changed = False
self.__enumeration = pd.DataFrame(d)
#self.__rank()
return self.__enumeration
else:
return self.__enumeration
def solve_for_k_alleles(self, k, ks=1):
"""
EXPERIMENTAL!
generates a solution without the regularization term and only k selected alleles
"""
if k < int(self.__instance.nof_loci.value) or k > int(
self.__instance.nof_loci.value * self.__t_max_allele):
raise Warning("k " + str(k) + " is out of range [" +
str(self.__instance.nof_loci.value) + "," +
str(self.__instance.nof_loci * self.__t_max_allele) +
"]")
#copy the instance
inst = self.__instance.clone()
#set beta = 0 because we do homozygosity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#generate constraint which allows only k alleles to be selected
expr1 = 0
for j in inst.x:
expr1 += inst.x[j]
inst.c.add(expr1 == k)
d = defaultdict(list)
for _ in xrange(ks):
#try:
inst.x.reset()
inst.y.reset()
inst.preprocess()
try:
if self.__threads > 1:
res = self.__solver.solve(self.__instance,
options="threads=" +
str(self.__threads),
tee=self.__verbosity)
else:
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
except:
del self.__solver.options["threads"]
print "WARNING: Solver does not support multi-threading. Please change the config file accordingly! " \
"Fall back to single-threading."
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
inst.load(res)
if self.__verbosity > 0:
res.write(num=1)
if str(res.Solution.status) != 'optimal':
break
selected = []
expr = 0
indices = []
encountered_4digit = []
for j in inst.x:
if 0.99 <= inst.x[j].value <= 1.01:
exp_i = 0
selected.append(j)
exp_i += inst.x[j]
if self.__allele_to_4digit[j] in encountered_4digit:
continue
encountered_4digit.append(self.__allele_to_4digit[j])
for i_allele in self.__groups_4digit[
self.__allele_to_4digit[j]]:
if inst.x[i_allele].value <= 0:
exp_i += inst.x[i_allele]
indices.append(i_allele)
expr += (1 - exp_i)
zero_indices = set([j for j in self.__instance.x
]).difference(set(indices))
for j in zero_indices:
expr += inst.x[j]
inst.c.add(expr >= 1)
if self.__verbosity > 0:
print selected
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(selected[i])
d[aas[i] + "2"].append(selected[i])
else:
d[aas[i] + str(c[aas[i]])].append(selected[i])
c[aas[i]] += 1
#print "Obj", res.Solution.Objective.__default_objective__.Value
nof_reads = sum((inst.occ[j] * inst.y[j].value for j in inst.y))
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
def solve_fixed_typing(self, fixed_alleles):
"""
EXPERIMENTAL!
forces the allele to pic a 4-digit of the provided alleles
"""
k = len(set(fixed_alleles))
if k < int(self.__instance.nof_loci.value) or k > int(
self.__instance.nof_loci.value * self.__t_max_allele):
raise Warning("k " + str(k) + " is out of range [" +
str(self.__instance.nof_loci.value) + "," +
str(self.__instance.nof_loci * self.__t_max_allele) +
"]")
#copy the instance
inst = self.__instance.clone()
#set beta = 0 because we do homozygocity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#generate constraint which allows only k alleles to be selected
expr1 = 0
for j in inst.x:
expr1 += inst.x[j]
inst.c.add(expr1 == k)
#generate for each of the provided alleles the fixation constraint:
for a in set(fixed_alleles):
expr_f = 0
print self.__groups_4digit
for ids in self.__groups_4digit[a]:
print ids
expr_f += inst.x[ids]
inst.c.add(expr_f == 1)
d = defaultdict(list)
inst.x.reset()
inst.y.reset()
inst.preprocess()
try:
if self.__threads > 1:
res = self.__solver.solve(self.__instance,
options="threads=" +
str(self.__threads),
tee=self.__verbosity)
else:
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
except:
del self.__solver.options["threads"]
print "WARNING: Solver does not support multi-threading. Please change the config file accordingly! " \
"Fall back to single-threading."
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
inst.load(res)
opt_ids = [j for j in inst.x if 0.99 <= inst.x[j].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in opt_ids]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(opt_ids[i])
d[aas[i] + "2"].append(opt_ids[i])
else:
d[aas[i] + str(c[aas[i]])].append(opt_ids[i])
c[aas[i]] += 1
nof_reads = sum((inst.occ[j] * inst.y[j].value for j in inst.y))
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
def enumerate_allele_wise(self):
"""
EXPERIMENTAL!
fixes all but one allele and solves it again to investigate the influence of this
particular allele on the objective value.
"""
d = defaultdict(list)
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
try:
if self.__threads > 1:
res = self.__solver.solve(self.__instance,
options="threads=" +
str(self.__threads),
tee=self.__verbosity)
else:
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
except:
del self.__solver.options["threads"]
print "WARNING: Solver does not support multi-threading. Please change the config file accordingly! " \
"Fall back to single-threading."
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
self.__instance.load(res)
opt_ids = [
j for j in self.__instance.x
if 0.99 <= self.__instance.x[j].value <= 1.01
]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in opt_ids]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(opt_ids[i])
d[aas[i] + "2"].append(opt_ids[i])
else:
d[aas[i] + str(c[aas[i]])].append(opt_ids[i])
c[aas[i]] += 1
nof_reads = sum((self.__instance.occ[j] * self.__instance.y[j].value
for j in self.__instance.y))
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
d['discarded'].append(0)
for j in opt_ids:
if self.__verbosity > 0:
self.__instance.c.pprint()
self.__instance.c.clear()
#fix all but j'th variable
fix = 0
for i in opt_ids:
if i != j:
fix += (1 - self.__instance.x[i])
self.__instance.c.add(fix == 0.0)
#discard j'th allele and all its 4digit equivalent alleles form the next solution
discard = 0
for k in self.__groups_4digit[self.__allele_to_4digit[j]]:
discard += self.__instance.x[k]
self.__instance.c.add(discard == 0.0)
#solve with new constraints
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
try:
res = self.__solver.solve(
self.__instance,
tee=self.__verbosity) #,tee=True) verbose solvinf
self.__instance.load(res)
except:
print Warning("There is no replacement for allele " +
self.__allele_to_4digit[j])
continue
selected = [
al for al in self.__instance.x
if 0.99 <= self.__instance.x[al].value <= 1.01
]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for q in xrange(len(aas)):
if aas.count(aas[q]) < 2:
d[aas[q] + "1"].append(selected[q])
d[aas[q] + "2"].append(selected[q])
else:
d[aas[q] + str(c[aas[q]])].append(selected[q])
c[aas[q]] += 1
nof_reads = sum(
(self.__instance.occ[h] * self.__instance.y[h].value
for h in self.__instance.y))
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
d['discarded'].append(j)
return pd.DataFrame(d)
def solve_enforced_zygosity(self, gosity_dict):
"""
EXPERIMENTAL!
solves the ilp without regularization but enforced h**o/heterozygosity for each locus
@param gosity_dict: a dictionary with all loci as keys and value = number of alleles per locus (default is 2)
"""
inst = self.__instance.clone()
#set beta = 0 because we do homozygocity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#now delete max_allele_constraint and reconstruct it again
inst.del_component("max_allel_selection")
for locus in inst.LociNames:
cons = 0
for a in inst.Loci[locus]:
cons += inst.x[a]
inst.c.add(cons <= gosity_dict.get(locus, 2))
d = defaultdict(list)
inst.x.reset()
inst.y.reset()
inst.preprocess()
try:
if self.__threads > 1:
res = self.__solver.solve(self.__instance,
options="threads=" +
str(self.__threads),
tee=self.__verbosity)
else:
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
except:
del self.__solver.options["threads"]
print "WARNING: Solver does not support multi-threading. Please change the config file accordingly! " \
"Fall back to single-threading."
res = self.__solver.solve(self.__instance,
options="",
tee=self.__verbosity)
inst.load(res)
selected = [al for al in inst.x if 0.99 <= inst.x[al].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for q in xrange(len(aas)):
if aas.count(aas[q]) < 2:
d[aas[q] + "1"].append(selected[q])
d[aas[q] + "2"].append(selected[q])
else:
d[aas[q] + str(c[aas[q]])].append(selected[q])
c[aas[q]] += 1
nof_reads = sum((inst.occ[h] * inst.y[h].value for h in inst.y))
d['obj'].append(self.__instance.read_cov())
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
class OptiType(object):
"""
classdocs
"""
def __init__(self, cov, occ, groups_4digit, allele_table, beta, t_max_allele=2, solver="glpk", threads=1,
verbosity=0):
"""
Constructor
"""
self.__allele_table = allele_table
self.__beta = float(beta)
self.__t_max_allele = t_max_allele
self.__solver = SolverFactory(solver)
self.__solver.options["threads"] = threads
self.__verbosity = True if verbosity > 0 else False
self.__changed = True
self.__ks = 1
self.__groups_4digit = groups_4digit
loci_alleles = defaultdict(list)
for type_4digit, group_alleles in groups_4digit.iteritems():
#print type_4digit, group_alleles
loci_alleles[type_4digit.split('*')[0]].extend(group_alleles)
loci = loci_alleles
self.__allele_to_4digit = {allele: type_4digit for type_4digit, group in groups_4digit.iteritems() for allele in
group}
'''
generates the basic ILP model
'''
model = ConcreteModel()
#init Sets
model.LociNames = Set(initialize=loci.keys())
model.Loci = Set(model.LociNames, initialize=lambda m, l: loci[l])
L = list(itertools.chain(*loci.values()))
reconst = {allele_id: 0.01 for allele_id in L if '_' in allele_id}
R = set([r for (r, _) in cov.keys()])
model.L = Set(initialize=L)
model.R = Set(initialize=R)
#init Params
model.cov = Param(model.R, model.L, initialize=lambda model, r, a: cov.get((r, a), 0))
model.reconst = Param(model.L, initialize=lambda model, a: reconst.get(a, 0))
model.occ = Param(model.R, initialize=occ)
model.t_allele = Param(initialize=self.__t_max_allele, mutable=True)
model.beta = Param(initialize=self.__beta,
validate=lambda val, model: 0.0 <= float(self.__beta) <= 0.999,
mutable=True)
model.nof_loci = Param(initialize=len(loci))
#init variables
model.x = Var(model.L, domain=Binary)
model.y = Var(model.R, domain=Binary)
model.re = Var(model.R, bounds=(0.0, None))
model.hetero = Var(bounds=(0.0, model.nof_loci))
#init objective
model.read_cov = Objective(
rule=lambda model: sum(model.occ[r] * (model.y[r] - model.beta * (model.re[r])) for r in model.R) - sum(
model.reconst[a] * model.x[a] for a in model.L), sense=maximize)
#init Constraints
model.max_allel_selection = Constraint(model.LociNames, rule=lambda model, l: sum(
model.x[a] for a in model.Loci[l]) <= model.t_allele)
model.min_allel_selection = Constraint(model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]) >= 1)
model.is_read_cov = Constraint(model.R,
rule=lambda model, r: sum(model.cov[r, a] * model.x[a] for a in model.L) >=
model.y[r])
model.heterozygot_count = Constraint(
rule=lambda model: model.hetero >= sum(model.x[a] for a in model.L) - model.nof_loci)
#regularization constraints
model.reg1 = Constraint(model.R, rule=lambda model, r: model.re[r] <= model.nof_loci * model.y[r])
model.reg2 = Constraint(model.R, rule=lambda model, r: model.re[r] <= model.hetero)
model.reg3 = Constraint(model.R,
rule=lambda model, r: model.re[r] >= model.hetero - model.nof_loci * (1 - model.y[r]))
#generate constraint list for solution enumeration
model.c = ConstraintList()
#generate instance
self.__instance = model.create()
def set_beta(self, beta):
"""
Sets the parameter beta
"""
self.__changed = True
getattr(self.__instance, str(self.__instance.beta)).set_value(float(beta))
def set_t_max_allele(self, t_max_allele):
"""
Sets the upper bound of alleles selected per loci
"""
self.__changed = True
getattr(self.__instance, str(self.__instance.t_allele)).set_value(t_max_allele)
def solve(self, ks):
"""
solves the problem k times and discards the found solutions in the next run.
"""
d = defaultdict(list) #in there we store the typing +objective and generate afterwards a DatarFrame with it
if self.__changed or self.__ks != ks:
self.__ks = ks
for k in xrange(ks):
expr = 0
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
res = self.__solver.solve(self.__instance, tee=self.__verbosity)
self.__instance.load(res)
#if self.__verbosity > 0:
# res.write(num=1)
if str(res.Solution.status) != 'optimal':
break
selected = []
indices = []
encountered_4digit = []
for j in self.__instance.x:
if self.__allele_to_4digit[j][0] in 'HJG':
if 0.99 <= self.__instance.x[j].value <= 1.01:
selected.append(j)
indices.append(j)
continue
if 0.99 <= self.__instance.x[j].value <= 1.01:
selected.append(j)
exp_i = 0
exp_i += self.__instance.x[j]
if self.__allele_to_4digit[j] in encountered_4digit:
continue
encountered_4digit.append(self.__allele_to_4digit[j])
for i_allele in self.__groups_4digit[self.__allele_to_4digit[j]]:
if self.__instance.x[i_allele].value <= 0:
exp_i += self.__instance.x[i_allele]
indices.append(i_allele)
expr += (1.0 - exp_i)
zero_indices = set([j for j in self.__instance.x]).difference(set(indices))
for j in zero_indices:
expr += self.__instance.x[j]
self.__instance.c.add(expr >= 1)
#if self.__verbosity > 0:
# print selected
# self.__instance.c.pprint()
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(selected[i])
d[aas[i] + "2"].append(selected[i])
else:
d[aas[i] + str(c[aas[i]])].append(selected[i])
c[aas[i]] += 1
nof_reads = sum((self.__instance.occ[j] * self.__instance.y[j].value for j in self.__instance.y))
#if self.__verbosity > 0:
# print "Obj", res.Solution.Objective.__default_objective__.Value
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
self.__instance.c.clear()
self.__changed = False
self.__enumeration = pd.DataFrame(d)
#self.__rank()
return self.__enumeration
else:
return self.__enumeration
def solve_for_k_alleles(self, k, ks=1):
"""
EXPERIMENTAL!
generates a solution without the regularization term and only k selected alleles
"""
if k < int(self.__instance.nof_loci.value) or k > int(self.__instance.nof_loci.value * self.__t_max_allele):
raise Warning("k " + str(k) + " is out of range [" + str(self.__instance.nof_loci.value) + "," + str(
self.__instance.nof_loci * self.__t_max_allele) + "]")
#copy the instance
inst = self.__instance.clone()
#set beta = 0 because we do homozygosity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#generate constraint which allows only k alleles to be selected
expr1 = 0
for j in inst.x:
expr1 += inst.x[j]
inst.c.add(expr1 == k)
d = defaultdict(list)
for _ in xrange(ks):
#try:
inst.x.reset()
inst.y.reset()
inst.preprocess()
res = self.__solver.solve(inst, tee=self.__verbosity)
inst.load(res)
if self.__verbosity > 0:
res.write(num=1)
if str(res.Solution.status) != 'optimal':
break
selected = []
expr = 0
indices = []
encountered_4digit = []
for j in inst.x:
if 0.99 <= inst.x[j].value <= 1.01:
exp_i = 0
selected.append(j)
exp_i += inst.x[j]
if self.__allele_to_4digit[j] in encountered_4digit:
continue
encountered_4digit.append(self.__allele_to_4digit[j])
for i_allele in self.__groups_4digit[self.__allele_to_4digit[j]]:
if inst.x[i_allele].value <= 0:
exp_i += inst.x[i_allele]
indices.append(i_allele)
expr += (1 - exp_i)
zero_indices = set([j for j in self.__instance.x]).difference(set(indices))
for j in zero_indices:
expr += inst.x[j]
inst.c.add(expr >= 1)
if self.__verbosity > 0:
print selected
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(selected[i])
d[aas[i] + "2"].append(selected[i])
else:
d[aas[i] + str(c[aas[i]])].append(selected[i])
c[aas[i]] += 1
#print "Obj", res.Solution.Objective.__default_objective__.Value
nof_reads = sum((inst.occ[j] * inst.y[j].value for j in inst.y))
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
def solve_fixed_typing(self, fixed_alleles):
"""
EXPERIMENTAL!
forces the allele to pic a 4-digit of the provided alleles
"""
k = len(set(fixed_alleles))
if k < int(self.__instance.nof_loci.value) or k > int(self.__instance.nof_loci.value * self.__t_max_allele):
raise Warning("k " + str(k) + " is out of range [" + str(self.__instance.nof_loci.value) + "," + str(
self.__instance.nof_loci * self.__t_max_allele) + "]")
#copy the instance
inst = self.__instance.clone()
#set beta = 0 because we do homozygocity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#generate constraint which allows only k alleles to be selected
expr1 = 0
for j in inst.x:
expr1 += inst.x[j]
inst.c.add(expr1 == k)
#generate for each of the provided alleles the fixation constraint:
for a in set(fixed_alleles):
expr_f = 0
print self.__groups_4digit
for ids in self.__groups_4digit[a]:
print ids
expr_f += inst.x[ids]
inst.c.add(expr_f == 1)
d = defaultdict(list)
inst.x.reset()
inst.y.reset()
inst.preprocess()
res = self.__solver.solve(inst, tee=self.__verbosity) #,tee=True) verbose solvinf
inst.load(res)
opt_ids = [j for j in inst.x if 0.99 <= inst.x[j].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in opt_ids]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(opt_ids[i])
d[aas[i] + "2"].append(opt_ids[i])
else:
d[aas[i] + str(c[aas[i]])].append(opt_ids[i])
c[aas[i]] += 1
nof_reads = sum((inst.occ[j] * inst.y[j].value for j in inst.y))
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
def enumerate_allele_wise(self):
"""
EXPERIMENTAL!
fixes all but one allele and solves it again to investigate the influence of this
particular allele on the objective value.
"""
d = defaultdict(list)
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
res = self.__solver.solve(self.__instance, tee=self.__verbosity) #,tee=True) verbose solvinf
self.__instance.load(res)
opt_ids = [j for j in self.__instance.x if 0.99 <= self.__instance.x[j].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in opt_ids]
c = dict.fromkeys(aas, 1)
for i in xrange(len(aas)):
if aas.count(aas[i]) < 2:
d[aas[i] + "1"].append(opt_ids[i])
d[aas[i] + "2"].append(opt_ids[i])
else:
d[aas[i] + str(c[aas[i]])].append(opt_ids[i])
c[aas[i]] += 1
nof_reads = sum((self.__instance.occ[j] * self.__instance.y[j].value for j in self.__instance.y))
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
d['discarded'].append(0)
for j in opt_ids:
if self.__verbosity > 0:
self.__instance.c.pprint()
self.__instance.c.clear()
#fix all but j'th variable
fix = 0
for i in opt_ids:
if i != j:
fix += (1 - self.__instance.x[i])
self.__instance.c.add(fix == 0.0)
#discard j'th allele and all its 4digit equivalent alleles form the next solution
discard = 0
for k in self.__groups_4digit[self.__allele_to_4digit[j]]:
discard += self.__instance.x[k]
self.__instance.c.add(discard == 0.0)
#solve with new constraints
self.__instance.x.reset()
self.__instance.y.reset()
self.__instance.preprocess()
try:
res = self.__solver.solve(self.__instance, tee=self.__verbosity) #,tee=True) verbose solvinf
self.__instance.load(res)
except:
print Warning("There is no replacement for allele " + self.__allele_to_4digit[j])
continue
selected = [al for al in self.__instance.x if 0.99 <= self.__instance.x[al].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for q in xrange(len(aas)):
if aas.count(aas[q]) < 2:
d[aas[q] + "1"].append(selected[q])
d[aas[q] + "2"].append(selected[q])
else:
d[aas[q] + str(c[aas[q]])].append(selected[q])
c[aas[q]] += 1
nof_reads = sum((self.__instance.occ[h] * self.__instance.y[h].value for h in self.__instance.y))
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
d['discarded'].append(j)
return pd.DataFrame(d)
def solve_enforced_zygosity(self, gosity_dict):
"""
EXPERIMENTAL!
solves the ilp without regularization but enforced h**o/heterozygosity for each locus
@param gosity_dict: a dictionary with all loci as keys and value = number of alleles per locus (default is 2)
"""
inst = self.__instance.clone()
#set beta = 0 because we do homozygocity calling manually
getattr(inst, str(inst.beta)).set_value(float(0.0))
inst.del_component("heterozygot_count")
inst.del_component("reg1")
inst.del_component("reg2")
inst.del_component("reg3")
#now delete max_allele_constraint and reconstruct it again
inst.del_component("max_allel_selection")
for locus in inst.LociNames:
cons = 0
for a in inst.Loci[locus]:
cons += inst.x[a]
inst.c.add(cons <= gosity_dict.get(locus, 2))
d = defaultdict(list)
inst.x.reset()
inst.y.reset()
inst.preprocess()
res = self.__solver.solve(inst, tee=self.__verbosity) #,tee=True) verbose solvinf
inst.load(res)
selected = [al for al in inst.x if 0.99 <= inst.x[al].value <= 1.01]
aas = [self.__allele_to_4digit[x].split('*')[0] for x in selected]
c = dict.fromkeys(aas, 1)
for q in xrange(len(aas)):
if aas.count(aas[q]) < 2:
d[aas[q] + "1"].append(selected[q])
d[aas[q] + "2"].append(selected[q])
else:
d[aas[q] + str(c[aas[q]])].append(selected[q])
c[aas[q]] += 1
nof_reads = sum((inst.occ[h] * inst.y[h].value for h in inst.y))
d['obj'].append(res.Solution.Objective.__default_objective__.Value)
d['nof_reads'].append(nof_reads)
return pd.DataFrame(d)
def optimal2DCopulaSampling(data, n_scenario=20, solver="cplex"):
'''
the optimal samples close to the empirical copula functions
it can only to deal with bivariate samples
@data, numpy.array, size: n_rv * 2
'''
assert data.shape[1] == 2
n_rv = data.shape[0]
tgt_copula = buildEmpiricalCopula(data)
# Model
model = ConcreteModel()
#Set, dimension 1 and 2
model.x = range(n_scenario)
model.y = range(n_scenario)
#decision variables
model.X = Var(model.x, model.y, within=Binary)
model.yp = Var(model.x, model.y, within=NonNegativeReals)
model.yn = Var(model.x, model.y, within=NonNegativeReals)
#constraint
def rowConstraint_rule(model, x):
'''to ensure that each rank is used only once in each row'''
val = sum(model.X[x, j] for j in model.y)
return val == 1
model.rowConstraint = Constraint(model.x)
def columnConstraint_rule(model, y):
'''to ensure that each rank is used only once in each column'''
val = sum(model.X[i, y] for i in model.x)
return val == 1
model.columnConstraint = Constraint(model.y)
def copulaConstraint_rule(model, i, j):
'''bias constraint '''
val = 0
for kdx in xrange(i):
for ldx in xrange(j):
val += model.X[kdx, ldx]
val = val - model.yp[i, j] + model.yn[i, j]
point = [(i + 1.) / n_scenario, (j + 1.) / n_scenario]
copula_val = getCopula(tgt_copula, point)
print "point %s copula:%s, S*copula:%s" % (point, copula_val,
n_scenario * copula_val)
return val == n_scenario * copula_val
model.copulaConstraint = Constraint(model.x, model.y)
#objective
def minBias_rule(model):
'''minimize the bias between the sampling and given CDF'''
val = 0
for idx in model.x:
for jdx in model.y:
val += (model.yp[idx, jdx] + model.yn[idx, jdx])
return val
model.minBias = Objective(sense=minimize)
# Create a solver
solver = solver
opt = SolverFactory(solver)
if solver == "cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
display(instance)
results = {}
for v in instance.active_components(Var):
varobject = getattr(instance, v)
if v == "X":
results[v] = np.fromiter(
(varobject[idx, jdx].value for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
parsed = []
for row in xrange(n_scenario):
for col in xrange(n_scenario):
if results[v][row][col] >= 0.9:
parsed.append((row, col))
parsed = (np.asarray(parsed) + 1) / n_scenario
results["parsed"] = parsed
elif v == "yp":
results[v] = np.fromiter(
(varobject[idx, jdx].value for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
elif v == "yn":
results[v] = np.fromiter(
(varobject[idx, jdx].value for jdx in np.arange(n_scenario)
for idx in np.arange(n_scenario)), np.float)
results[v] = results[v].reshape((n_scenario, n_scenario))
# print results
return results
def MinCVaRPortfolioSIP(symbols, riskyRet, riskFreeRet, allocatedWealth,
depositWealth, buyTransFee, sellTransFee, alpha,
predictRiskyRet, predictRiskFreeRet, n_scenario,
probs=None, solver="glpk", n_stock=5):
'''
two-stage stochastic integer programming
given M stocks, choose the best n_stocks to invest
variable:
M: num. of symbols,
S: num. of scenarios
symbols, list of string,
riskRet, numpy.array, size: M
riskFreeRet, float
allocatedWealth, numpy.array, size: M
depositWealth, float,
buyTransFee, numpy.array, size: M
sellTransFee, numpy.array, size: M
alpha, float,
predictRiskRet, numpy.array, size: M * S
predictRiskFreeRet, float
n_scenario, positive integer
probs, numpy.array, size: S
solver, string in {glpk or cplex}
n_stock, positive integer, max number of stock to invest in
'''
t = time.time()
if not probs:
probs = np.ones(n_scenario, dtype=np.float)/n_scenario
# Model
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
model.scenarios = range(n_scenario)
#decision variables
#stage1
#0: stock is not chosen, 1: stock is chosen
model.chosen = Var(model.symbols, within=Binary)
model.buys = Var(model.symbols, within=NonNegativeReals)
model.sells = Var(model.symbols, within=NonNegativeReals)
#stage 2
model.riskyWealth = Var(model.symbols, within=NonNegativeReals)
model.riskFreeWealth = Var(within=NonNegativeReals)
#aux variable, variable in definition of CVaR, equals to VaR at opt. sol.
model.Z = Var()
#aux variable, portfolio wealth less than than VaR (Z)
model.Ys = Var(model.scenarios, within=NonNegativeReals)
#constraint
def riskyWeathConstraint_rule(model, m):
'''
riskyWealth is a decision variable depending on both buys and sells.
(it means riskyWealth depending on scenario).
therefore
buys and sells are fist stage variable,
riskywealth is second stage variable
'''
return (model.riskyWealth[m] ==
(1. + riskyRet[m]) * allocatedWealth[m] +
model.buys[m] - model.sells[m])
model.riskyWeathConstraint = Constraint(model.symbols)
def riskFreeWealthConstraint_rule(model):
'''
riskFreeWealth is decision variable depending on both buys and sells.
therefore
buys and sells are fist stage variable,
riskFreewealth is second stage variable
'''
totalSell = sum((1 - sellTransFee[m]) * model.sells[m]
for m in model.symbols)
totalBuy = sum((1 + buyTransFee[m]) * model.buys[m]
for m in model.symbols)
return (model.riskFreeWealth ==
(1. + riskFreeRet)* depositWealth +
totalSell - totalBuy)
model.riskFreeWealthConstraint = Constraint()
def CVaRConstraint_rule(model, s):
'''auxiliary variable Y depends on scenario. CVaR <= VaR'''
wealth = sum( (1. + predictRiskyRet[m][s] ) * model.riskyWealth[m]
for m in model.symbols)
return model.Ys[s] >= (model.Z - wealth)
model.CVaRConstraint = Constraint(model.scenarios)
def chosenConstraint_rule(model, m):
totalCapital = sum(allocatedWealth) + depositWealth
return (model.riskyWealth[m] <= model.chosen[m] * totalCapital)
model.chosenConstraint = Constraint(model.symbols)
def portoflioSizeConstraint_rule(model):
'''restricted the number of stock to invest'''
return sum(model.chosen[m] for m in model.symbols) <= n_stock
model.portoflioSizeConstraint = Constraint()
#objective
def TotalCostObjective_rule(model):
return model.Z - 1/(1-alpha)* sum(model.Ys[s] * probs[s]
for s in xrange(n_scenario))
model.TotalCostObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 6
instance = model.create()
results = opt.solve(instance)
instance.load(results)
CVaR = results.Solution.Objective.__default_objective__['value']
# display(instance)
M = len(symbols)
results = {"CVaR": CVaR}
for v in instance.active_components(Var):
varobject = getattr(instance, v)
if v == "buys":
results[v] = np.fromiter((varobject[index].value for index in varobject), np.float)
elif v == "sells":
results[v] = np.fromiter((varobject[index].value for index in varobject), np.float)
elif v == "Z":
results["VaR"] = varobject.value
print "CVaR:", CVaR
print "MinCVaRPortfolioSIP elapsed %.3f secs"%(time.time()-t)
return results
def solve_tech_model(idx, solver='gurobi', time=None, gap=None, cutoff=None):
tech = 'Tech{}'.format(idx)
bash_command('rm -f ReferenceModel*')
model = Pyomo_tech(idx)
instance = model.create()
# instance.write('model_{}.lp'.format(idx),symbolic_solver_labels=True)
opt = SolverFactory(solver)
if gap is not None:
if gap < 1:
opt.options["MIPGap"] = gap
else:
opt.options["MIPGapAbs"] = gap
if time is not None:
opt.options["TimeLimit"] = time
if cutoff is not None:
opt.options["Cutoff"] = cutoff
with open('temp.log', 'w') as f:
with Redirect(f, f):
results = opt.solve(model, tee=True)
status = results['Solver'][0]['Termination condition'].key
transformed_results = instance.update_results(results)
# print 'After Loading: {}'.format(status)
if status == 'optimal':
instance.load(results)
rm('temp.log')
obj = instance.Total_Cost_Objective()
SubCost(tech, instance, obj)
# FTI = [tech, instance.alpha_put.value, instance.alpha_pick.value]
#print '\tFinished {0:6}: {1}, {2}'.format(*FTI)
#FTI = [tech, instance.lambda_put.value/8., instance.lambda_pick.value/8.]
#print '\tFinished {0:6}: {1:6} | {2:6}'.format(*FTI)
with open('results_{}.txt'.format(idx), 'w') as f:
f.write('Results from {}\n'.format(tech))
f.write('Putaway Technology: {}\n'.format(instance.lambda_put.value))
f.write('Picking Technology: {}\n\n'.format(instance.lambda_pick.value))
with Redirect(f, f):
display(instance)
else:
obj = 'Cutoff Error'
with open('results_{}.yml'.format(idx), 'w') as f:
from check_sol import T
f.write('Results from {}\n'.format(tech))
f.write('Putaway Technology: {}\n'.format(T[idx][0]))
f.write('Picking Technology: {}\n'.format(T[idx][1]))
if status == 'optimal':
f.write('Total_Cost_Objective:\n\tValue={}\n\n'.format(obj))
else:
f.write('Total_Cost_Objective:\n\tValue={}\n\n'.format(status))
with Redirect(f, f):
transformed_results.write()
bash_command('rm -f ReferenceModel*')
return tech, instance, obj
def orth_constraint(model):
return summation(model.y, model.alpha) == 0
model.orth = Constraint(rule = orth_constraint)
#create objective function
def kernel(x,y):
max_index = len(numpy.shape(y))
return numpy.sum(x*y, axis = max_index - 1)
def Obj(model):
return summation(model.alpha) - 0.5*sum([ sum([model.alpha[i]*model.alpha[j]*model.y[i]*model.y[j]*kernel(model.x[i],model.x[j]) for i in model.i]) for j in model.i])
model.L = Objective(rule = Obj, sense = maximize)
#Create Solver
opt = SolverFactory('ipopt')
#Solve
results = opt.solve(model)
model.load(results)
#Ending pyomo timer
print("Pyomo took " + str(time.time() - start_time) + " seconds")
#Scatter original points
matplotlib.pyplot.figure(1, figsize=(4, 3))
matplotlib.pyplot.clf()
matplotlib.pyplot.scatter(X1[:,0], X1[:,1], color = 'r', zorder=10)
matplotlib.pyplot.scatter(X2[:,0], X2[:,1], color = 'b', zorder=10)
#Get a picture for the classifier
# In[52]:
# (Constraint 2) Nodes t h a t share an edge cannot be c o l o r e d t he same color
def different_colors(model, v, w, c):
return model.x[v, c] + model.x[w, c] <= 1
model.edge_coloring = Constraint(model.edges,
model.colors,
rule=different_colors)
# In[53]:
# (Constraint 3) Provide a lower bound on the minimum number of c o l o r s
def lower_bound(model, v, c):
return c * model.x[v, c] <= model.y
model.lower_bound = Constraint(model.vertices, model.colors, rule=lower_bound)
# In[54]:
model.obj = Objective(expr=model.y, sense=minimize)
opt = SolverFactory("glpk")
results = opt.solve(model)
# In[ ]:
def __init__(self,
cov,
occ,
groups_4digit,
allele_table,
beta,
t_max_allele=2,
solver="glpk",
threads=1,
verbosity=0):
"""
Constructor
"""
self.__allele_table = allele_table
self.__beta = float(beta)
self.__t_max_allele = t_max_allele
self.__solver = SolverFactory(solver)
self.__threads = threads
self.__verbosity = True if verbosity > 0 else False
self.__changed = True
self.__ks = 1
self.__groups_4digit = groups_4digit
loci_alleles = defaultdict(list)
for type_4digit, group_alleles in groups_4digit.iteritems():
#print type_4digit, group_alleles
loci_alleles[type_4digit.split('*')[0]].extend(group_alleles)
loci = loci_alleles
self.__allele_to_4digit = {
allele: type_4digit
for type_4digit, group in groups_4digit.iteritems()
for allele in group
}
'''
generates the basic ILP model
'''
model = ConcreteModel()
#init Sets
model.LociNames = Set(initialize=loci.keys())
model.Loci = Set(model.LociNames, initialize=lambda m, l: loci[l])
L = list(itertools.chain(*loci.values()))
reconst = {allele_id: 0.01 for allele_id in L if '_' in allele_id}
R = set([r for (r, _) in cov.keys()])
model.L = Set(initialize=L)
model.R = Set(initialize=R)
#init Params
model.cov = Param(model.R,
model.L,
initialize=lambda model, r, a: cov.get((r, a), 0))
model.reconst = Param(model.L,
initialize=lambda model, a: reconst.get(a, 0))
model.occ = Param(model.R, initialize=occ)
model.t_allele = Param(initialize=self.__t_max_allele, mutable=True)
model.beta = Param(
initialize=self.__beta,
validate=lambda val, model: 0.0 <= float(self.__beta) <= 0.999,
mutable=True)
model.nof_loci = Param(initialize=len(loci))
#init variables
model.x = Var(model.L, domain=Binary)
model.y = Var(model.R, domain=Binary)
model.re = Var(model.R, bounds=(0.0, None))
model.hetero = Var(bounds=(0.0, model.nof_loci))
#init objective
model.read_cov = Objective(rule=lambda model: sum(
model.occ[r] * (model.y[r] - model.beta * (model.re[r]))
for r in model.R) - sum(model.reconst[a] * model.x[a]
for a in model.L),
sense=maximize)
#init Constraints
model.max_allel_selection = Constraint(
model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]
) <= model.t_allele)
model.min_allel_selection = Constraint(
model.LociNames,
rule=lambda model, l: sum(model.x[a] for a in model.Loci[l]) >= 1)
model.is_read_cov = Constraint(
model.R,
rule=lambda model, r: sum(model.cov[r, a] * model.x[a]
for a in model.L) >= model.y[r])
model.heterozygot_count = Constraint(
rule=lambda model: model.hetero >= sum(model.x[a] for a in model.L
) - model.nof_loci)
#regularization constraints
model.reg1 = Constraint(
model.R,
rule=lambda model, r: model.re[r] <= model.nof_loci * model.y[r])
model.reg2 = Constraint(
model.R, rule=lambda model, r: model.re[r] <= model.hetero)
model.reg3 = Constraint(model.R,
rule=lambda model, r: model.re[r] >= model.
hetero - model.nof_loci * (1 - model.y[r]))
#generate constraint list for solution enumeration
model.c = ConstraintList()
#generate instance
self.__instance = model.create()
def maxRetMultiStagePortfolio(riskyRetMtx, riskFreeRetVec, buyTransFeeMtx,
sellTransFeeMtx, allocatedWealth, symbols,
transDates):
'''
-假設資料共T期, 投資在M個risky assets, 以及1個riskfree asset
-求出每個risky asset每一期應該要買進以及賣出的金額
-最後一期結算不投資
@param riskyRetMtx, numpy.array, size: M * T+1
@param riskFreeRetVec, numpy.array, size: T+1
@param buyTransFeeMtx, numpy.array, size: M * T
@param sellTransFeeMtx, numpy.array, size: M * T
@param allocatedWealth, numpy.array, size: (M+1) (最後一個為cash)
@return (buyMtx, sellMtx), numpy.array, each size: M*T
'''
assert buyTransFeeMtx.shape == sellTransFeeMtx.shape
assert riskyRetMtx.shape[1] == riskFreeRetVec.size
M, T = buyTransFeeMtx.shape
t1 = time.time()
#create model
model = ConcreteModel()
#number of asset and number of periods
model.symbols = range(M)
model.T = range(T)
#decision variables
model.buys = Var(model.symbols, model.T, within=NonNegativeReals)
model.sells = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskyWealth = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskFreeWealth = Var(model.T, within=NonNegativeReals)
#objective
def objective_rule(model):
wealth = sum((1. + riskyRetMtx[m, T]) * model.riskyWealth[m, T - 1]
for m in xrange(M))
wealth += (1. + riskFreeRetVec[T]) * model.riskFreeWealth[T - 1]
return wealth
model.objective = Objective(sense=maximize)
#constraint
def riskyWeathConstraint_rule(model, m, t):
if t >= 1:
preWealth = model.riskyWealth[m, t - 1]
else:
preWealth = allocatedWealth[m]
return (model.riskyWealth[m,
t] == (1. + riskyRetMtx[m, t]) * preWealth +
model.buys[m, t] - model.sells[m, t])
def riskFreeWealthConstraint_rule(model, t):
totalSell = sum((1 - sellTransFeeMtx[mdx, t]) * model.sells[mdx, t]
for mdx in xrange(M))
totalBuy = sum((1 + buyTransFeeMtx[mdx, t]) * model.buys[mdx, t]
for mdx in xrange(M))
if t >= 1:
preWealth = model.riskFreeWealth[t - 1]
else:
preWealth = allocatedWealth[-1]
return (
model.riskFreeWealth[t] == (1. + riskFreeRetVec[t]) * preWealth +
totalSell - totalBuy)
model.riskyWeathConstraint = Constraint(model.symbols, model.T)
model.riskFreeWealthConstraint = Constraint(model.T)
#optimizer
opt = SolverFactory('cplex')
instance = model.create()
results = opt.solve(instance)
print type(results)
print results
#load decision variable
instance.load(results)
display(instance)
# instance.load(results)
# for var in instance.active_components(Var):
# varobj = getattr(instance, var)
# for idx in varobj:
# print varobj[idx], varobj[idx].value
#
# print type(results)
#load objective value
finalWealth = results.Solution.Objective.__default_objective__['value']
print "finalWealth:", finalWealth
print "ret: %.3f %%" % ((finalWealth / 1e5 - 1) * 100)
print "elapsed %.3f secs" % (time.time() - t1)
from os import getcwd, chdir, mkdir
from os.path import isdir
from coopr.pyomo import *
from coopr.opt import SolverFactory
from IPython import embed as II
from Preprocessor import *
from time import time
import csv
import matplotlib.pyplot as plt
from sys import argv
WindMode = True
WaveMode = False
model = ConcreteModel()
opt = SolverFactory("glpk")
#-------------------------------------------------------------------------------
# Data input from csv files, 'data.csv', 'wind.csv' and 'wave.csv'
#--------------------------------------------------------------------------------
data = data_generator() # 1st column: name, 2nd column: value
wind = wind_generator()
wave = wave_generator()
Xw1 = data['Xw1']
etaT1 = data['etaT1']
C1 = data['C1']
F1 = data['F1']
V1 = data['V1']
Xw2 = data['Xw2']
def WorstCVaRPortfolioSP(symbols, riskyRet, riskFreeRet, allocatedWealth,
depositWealth, buyTransFee, sellTransFee, alpha,
predictRiskyRet, predictRiskFreeRet, n_scenario,
probs=None, solver="cplex"):
'''
two-stage stochastic programming
variable:
M: num. of symbols,
S: num. of scenarios
symbols, list of string,
riskRet, numpy.array, size: M
riskFreeRet, float
allocatedWealth, numpy.array, size: M
depositWealth, float,
buyTransFee, numpy.array, size: M
sellTransFee, numpy.array, size: M
alpha, float,
predictRiskRet, numpy.array, size: L * M * S, L個不確定的dist.
predictRiskFreeRet, float
probs, numpy.array, size: S
solver, string in {glpk or cplex}
'''
t = time.time()
assert len(predictRiskyRet.shape) == 3
n_dist = predictRiskyRet.shape[0]
if not probs:
probs = np.ones(n_scenario, dtype=np.float)/n_scenario
# Model
model = ConcreteModel()
#Set
model.symbols = range(len(symbols))
model.scenarios = range(n_scenario)
model.distributions = range(n_dist)
#decision variables
model.buys = Var(model.symbols, within=NonNegativeReals) #stage 1
model.sells = Var(model.symbols, within=NonNegativeReals) #stage 1
model.riskyWealth = Var(model.symbols, within=NonNegativeReals) #stage 2
model.riskFreeWealth = Var(within=NonNegativeReals) #stage 2
#aux variable, variable in definition of CVaR, equals to VaR at opt. sol.
model.Z = Var()
#aux variable, portfolio wealth less than than VaR (Z)
model.Ys = Var(model.distributions, model.scenarios, within=NonNegativeReals)
#constraint
def riskyWeathConstraint_rule(model, m):
'''
riskyWealth is a decision variable depending on both buys and sells.
(it means riskyWealth depending on scenario).
therefore
buys and sells are fist stage variable,
riskywealth is second stage variable
'''
return (model.riskyWealth[m] ==
(1. + riskyRet[m]) * allocatedWealth[m] +
model.buys[m] - model.sells[m])
model.riskyWeathConstraint = Constraint(model.symbols)
def riskFreeWealthConstraint_rule(model):
'''
riskFreeWealth is decision variable depending on both buys and sells.
therefore
buys and sells are fist stage variable,
riskFreewealth is second stage variable
'''
totalSell = sum((1 - sellTransFee[m]) * model.sells[m]
for m in model.symbols)
totalBuy = sum((1 + buyTransFee[m]) * model.buys[m]
for m in model.symbols)
return (model.riskFreeWealth ==
(1. + riskFreeRet)* depositWealth +
totalSell - totalBuy)
model.riskFreeWealthConstraint = Constraint()
def WCVaRConstraint_rule(model, d, s):
'''auxiliary variable Y depends on scenario. CVaR <= VaR'''
wealth = sum( (1. + predictRiskyRet[d, m, s] ) * model.riskyWealth[m]
for m in model.symbols)
return model.Ys[d, s] >= (model.Z - wealth)
model.WCVaRConstraint = Constraint(model.distributions, model.scenarios)
#objective
def WCVaRObjective_rule(model):
val = 0
for d in model.distributions:
for s in model.scenarios:
val += model.Ys[d, s] * probs[s]
val *= 1/(1-alpha)
val = model.Z - val
return val
model.WCVaRObjective = Objective(sense=maximize)
# Create a solver
opt = SolverFactory(solver)
if solver =="cplex":
opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
instance.load(results)
WCVaR = results.Solution.Objective.__default_objective__['value']
# display(instance)
M = len(symbols)
results = {"WCVaR": WCVaR}
for v in instance.active_components(Var):
# print "Variable",v
varobject = getattr(instance, v)
if v == "buys":
results[v] = np.fromiter((varobject[index].value for index in varobject), np.float)
elif v == "sells":
results[v] = np.fromiter((varobject[index].value for index in varobject), np.float)
elif v == "Z":
results["VaR"] = varobject.value
# print results
print "WCVaR:", WCVaR
print "WorstCVaRPortfolioSP elapsed %.3f secs"%(time.time()-t)
return results
def concretePharmacy():
t1 = time.time()
model = ConcreteModel()
model.drug=[1,2]
model.material = [1,2]
model.budget = ["money", "HR", "hour", "storage"]
#set
# model.drug = Set(initialize=[1,2])
# model.material = Set(initialize=[1,2])
# model.budget = Set(initialize=["money", "HR", "hour", "storage"])
#parameter
# model.sellPrice = Param(model.drug, initialize={1:6200, 2:6900})
model.sellPrice = Param(model.drug, mutable=True)
model.sellPrice[1] = 6200
model.sellPrice[2] = 6900
model.grams = Param(model.drug, initialize={1:0.5, 2:0.6})
model.HRhours = Param(model.drug, initialize={1:90, 2:100})
model.EQhours = Param(model.drug, initialize={1:40, 2:50})
model.EQCost = Param(model.drug, initialize={1:700, 2:800})
model.materialCost = Param(model.material, initialize={1:100, 2:199.9},
mutable=True)
model.materialContent = Param(model.material, initialize={1:0.01, 2:0.02})
model.budgeValue = Param(model.budget, initialize={"money":1e5, "HR":2000,
"hour":800, "storage":1000})
#decision variables
model.D = Var(model.drug, domain=NonNegativeReals)
model.R = Var(model.material, domain=NonNegativeReals)
#objective
def objective_rule(model):
return summation(model.sellPrice, model.D) - \
summation(model.EQCost, model.D) - \
summation(model.materialCost, model.R)
model.Obj = Objective(rule=objective_rule, sense=maximize)
#constraint
def balance_constraint_rule(model):
return summation(model.materialContent, model.R) - \
summation(model.grams, model.D) >= 0
def storage_constraint_rule(model):
return summation(model.R) <= model.budgeValue['storage']
def HR_constraint_rule(model):
return summation(model.HRhours, model.D) <= model.budgeValue['HR']
def EQ_constraint_rule(model):
return summation(model.EQhours, model.D) <= model.budgeValue['hour']
def money_constraint_rule(model):
return (summation(model.EQCost, model.D) +
summation(model.materialCost, model.R)) <=model.budgeValue['money']
model.balanceConstraint = Constraint(rule=balance_constraint_rule)
model.storageConstraint = Constraint(rule=storage_constraint_rule)
model.HRConstraint = Constraint(rule=HR_constraint_rule)
model.EQConstraint = Constraint(rule=EQ_constraint_rule)
model.moneyConstraint = Constraint(rule=money_constraint_rule)
# Create a solver
opt = SolverFactory('cplex')
print "opt options:", opt.options
# opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
print type(instance)
print dir(instance)
print results
print
# print results.Solution.Objective.x1.Value
# print results.Solver.Status
# print results.Solution.Status
# print type(results)
# print dir(results)
instance.load(results)
#print variable method 1
print instance.D[1].value
print instance.D[2].value
print instance.R[1].value
print instance.R[2].value
print "obj:", results.Solution.Objective.__default_objective__['value']
#print variable method 2
# for var in instance.active_components(Var):
# varobj = getattr(instance, var)
# for idx in varobj:
# print varobj[idx], varobj[idx].value
display(instance)
#output file to yaml format
# results.write()
print "original elapsed %.3f secs"%(time.time()-t1)
def maxRetPortfolio(riskyRetMtx, riskFreeRetVec, buyTransFeeMtx,
sellTransFeeMtx, allocatedWealth):
'''
-假設有T期的資料, 投資在M個risky assets, 以及1個riskfree asset
-在(T+1)期初結算
goal: 最大化T+1期期初財產
@param riskyRetMtx, numpy.array, size: M * T+1
@param riskFreeRetVec, numpy.array, size: T+1
@param buyTransFeeMtx, numpy.array, size: M * T
@param sellTransFeeMtx, numpy.array, size: M * T
@param allocatedWealth, numpy.array, size: (M+1) (最後一個為cash)
@return (buyMtx, sellMtx), numpy.array, each size: M*T
'''
assert buyTransFeeMtx.shape == sellTransFeeMtx.shape
assert riskyRetMtx.shape[1] == riskFreeRetVec.size
M, T = buyTransFeeMtx.shape
t1 = time.time()
#create model
model = ConcreteModel()
#number of asset and number of periods
model.symbols = range(M)
model.T = range(T)
model.Tp1 = range(T + 1)
model.Mp1 = range(M + 1)
#decision variables
model.buys = Var(model.symbols, model.T, within=NonNegativeReals)
model.sells = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskyWealth = Var(model.symbols, model.T, within=NonNegativeReals)
model.riskFreeWealth = Var(model.T, within=NonNegativeReals)
#objective
def objective_rule(model):
wealth = sum((1. + riskyRetMtx[m, T]) * model.riskyWealth[m, T - 1]
for m in xrange(M))
wealth += (1. + riskFreeRetVec[T]) * model.riskFreeWealth[T - 1]
return wealth
model.objective = Objective(rule=objective_rule, sense=maximize)
#constraint
def riskyWealth_constraint_rule(model, m, t):
if t >= 1:
preWealth = model.riskyWealth[m, t - 1]
else:
preWealth = allocatedWealth[m]
return (model.riskyWealth[m,
t] == (1. + riskyRetMtx[m, t]) * preWealth +
model.buys[m, t] - model.sells[m, t])
def riskyFreeWealth_constraint_rule(model, t):
totalSell = sum((1 - sellTransFeeMtx[mdx, t]) * model.sells[mdx, t]
for mdx in xrange(M))
totalBuy = sum((1 + buyTransFeeMtx[mdx, t]) * model.buys[mdx, t]
for mdx in xrange(M))
if t >= 1:
preWealth = model.riskFreeWealth[t - 1]
else:
preWealth = allocatedWealth[-1]
return (
model.riskFreeWealth[t] == (1. + riskFreeRetVec[t]) * preWealth +
totalSell - totalBuy)
model.riskyWeathConstraint = Constraint(model.symbols,
model.T,
rule=riskyWealth_constraint_rule)
model.riskFreeWealthConstraint = Constraint(
model.T, rule=riskyFreeWealth_constraint_rule)
#optimizer
opt = SolverFactory('cplex')
# opt.options["threads"] = 4
instance = model.create()
results = opt.solve(instance)
print results
# instance.load(results)
# display(instance)
# instance.load(results)
# for var in instance.active_components(Var):
# varobj = getattr(instance, var)
# for idx in varobj:
# print varobj[idx], varobj[idx].value
#
print "elapsed %.3f secs" % (time.time() - t1)
data_master.load(filename=path_datos+'data_particiones.csv',
param=(master_model.barras_zona1, master_model.barras_zona2),
index=master_model.PARTICIONES)
data_slave = data_master
#data_slave.load(filename=path_datos+'data_gen_despachos.csv',
# param=(slave_model.gen_d_uc, slave_model.gen_d_pg, slave_model.gen_d_resup),
# index=(slave_model.GENERADORES, slave_model.ESCENARIOS))
print ("--- Creando Master ---")
master_instance = master_model.create(data_master)
print ("--- Creando Slave")
slave_instance = slave_model.create(data_slave)
opt = SolverFactory("cplex")
# Datos para la iteracion
max_it = 30
GAP_ENS = 1
ngen = {}
planos = {}
ENS = {}
fobj = {}
reqs = {}
particiones = {}
it = 1
eps = 0.1
def EOSS_PRDP_Solve(s, problem_data, fixed_parameters,output_directory):
opt = SolverFactory("cplex")
options = Options()
opt.options.mip_tolerances_mipgap = .000001
opt.options.mip_tolerances_absmipgap = .000001
SSsuccess = {}
### Redefine Success
for i in problem_data.product:
SSsuccess[i] = problem_data.success[(i,s)]
### Redefine Outcome
SSoutcome = problem_data.List_of_Scenarios[s].outcome
### Redefine Probability
SSProbability = problem_data.List_of_Scenarios[s].probability
model = SingleScenario(problem_data.product,problem_data.stage_gate,problem_data.time_step,problem_data.resource_type,problem_data.resource_max,problem_data.gammaL,problem_data.gammaD,problem_data.duration,problem_data.trial_cost,problem_data.resource_required, problem_data.revenue_max,SSProbability, SSsuccess,problem_data.Last_Time_Step, problem_data.last_trial, problem_data.running_revenue, problem_data.open_revenue, problem_data.discounting_factor, SSoutcome)
instance = model.create()
###################################################################
### Determine which fixed parameters are applicable to this problem
###################################################################
new_fixed_parameters = EOSS_Fixed_Parameters(fixed_parameters,SSoutcome)
for items in new_fixed_parameters:
var_i = items[0]
var_j = items[1]
var_t = items[2] + 1
decision = items[3]
instance.Decision_X[var_i,var_j,var_t] = decision
instance.Decision_X[var_i,var_j,var_t].fixed = True
del model
gc.collect()
instance.preprocess()
results= opt.solve(instance)
instance.load(results)
"""
save_file = str(s)
### Open save file
if not os.path.exists(output_directory):
os.makedirs(output_directory)
f = open(os.path.join(output_directory, save_file), "w")
transformed_results = instance.update_results(results)
tr = str(transformed_results)
f.write(tr + '\n')
f.close()
"""
if results.solver.status == SolverStatus.ok and results.solver.termination_condition == TerminationCondition.optimal:
return [s,results.solution.objective['__default_objective__']['Value']]
else:
save_file = "Scenario ", s
if not os.path.exists(output_directory):
os.makedirs(output_directory)
f = open(os.path.join(output_directory, save_file), "w")
transformed_results = instance.update_results(results)
tr = str(transformed_results)
f.write(tr + '\n')
f.close()
exit()
