def solve(self, parallel=True, cores=32):
'''
Solve problems of type example
The results are stored as
./results/{self.output_folder}/{solver}/results.csv
using a pandas table with fields
- 'name': Maros problem name
- 'solver': solver name
- 'status': solver status
- 'run_time': execution time
- 'iter': number of iterations
- 'obj_val': objective value from solver
- 'obj_opt': optimal objective value
- 'n': leading dimension
- 'N': nnz dimension (nnz(P) + nnz(A))
'''
print("Solving Maros Meszaros problems")
print("-------------------------------")
if parallel:
pool = Pool(processes=min(cores, cpu_count()))
# Iterate over all solvers
for solver in self.solvers:
settings = self.settings[solver]
# # Initialize solver results
# results_solver = []
# Solution directory
path = os.path.join('.', 'results', self.output_folder, solver)
# Create directory for the results
make_sure_path_exists(path)
# Get solver file name
results_file_name = os.path.join(path, 'results.csv')
# Check if file name already exists
if not os.path.isfile(results_file_name):
# Solve Maros Meszaros problems
if parallel:
results = pool.starmap(
self.solve_single_example,
zip(self.problems, repeat(solver), repeat(settings)))
else:
results = []
for problem in self.problems:
results.append(
self.solve_single_example(problem, solver,
settings))
# Create dataframe
df = pd.concat(results)
# Store results
df.to_csv(results_file_name, index=False)
# else:
# # Load from file
# df = pd.read_csv(results_file_name)
#
# # Combine list of dataframes
# results_solver.append(df)
if parallel:
pool.close() # Not accepting any more jobs on this pool
pool.join() # Wait for all processes to finish
def solve(self, parallel=True):
'''
Solve problems of type example
The results are stored as
./results/benchmark_problems/{solver}/{class}/n{dimension}.csv
using a pandas table with fields
- 'class': example class
- 'solver': solver name
- 'status': solver status
- 'run_time': execution time
- 'iter': number of iterations
- 'obj_val': objective value
- 'n': leading dimension
- 'N': nnz dimension (nnz(P) + nnz(A))
'''
print("Solving %s" % self.name)
print("-----------------")
if parallel:
pool = Pool(processes=min(self.n_instances, cpu_count()))
# Iterate over all solvers
for solver in self.solvers:
settings = self.settings[solver]
# Initialize solver results
results_solver = []
# Solution directory
path = os.path.join('.', 'results', 'benchmark_problems', solver,
self.name)
# Create directory for the results
make_sure_path_exists(path)
# Get solver file name
solver_file_name = os.path.join(path, 'full.csv')
for n in self.dims:
# Check if solution already exists
n_file_name = os.path.join(path, 'n%i.csv' % n)
if not os.path.isfile(n_file_name):
if parallel:
instances_list = list(range(self.n_instances))
n_results = pool.starmap(
self.solve_single_example,
zip(repeat(n), instances_list, repeat(solver),
repeat(settings)))
else:
n_results = []
for instance in range(self.n_instances):
n_results.append(
self.solve_single_example(
n, instance, solver, settings))
# Combine n_results
df = pd.concat(n_results)
# Store n_results
df.to_csv(n_file_name, index=False)
else:
# Load from file
df = pd.read_csv(n_file_name)
# Combine list of dataframes
results_solver.append(df)
# Create total dataframe for the solver from list
df_solver = pd.concat(results_solver)
# Store dataframe
df_solver.to_csv(solver_file_name, index=False)
if parallel:
pool.close() # Not accepting any more jobs on this pool
pool.join() # Wait for all processes to finish
def solve(self):
"""
Solve Portfolio problem
"""
print("Solve Portfolio problem for dimension %i" % self.n_factors)
# Create example instance
instance = PortfolioExample(self.n_factors, n=self.n_assets)
# Store number of nonzeros in F and D for updates
nnzF = instance.F.nnz
# Store alpha
alpha = self.alpha
'''
Solve problem without warm start
'''
# print("Solving without warm start")
# Solution directory
no_ws_path = os.path.join('.', 'results', 'parametric_problems',
'OSQP no warmstart',
'Portfolio',
)
# Create directory for the results
make_sure_path_exists(no_ws_path)
# Check if solution already exists
n_file_name = os.path.join(no_ws_path, 'n%i.csv' % self.n_factors)
if not os.path.isfile(n_file_name):
res_list_no_ws = [] # Initialize results
for i in range(self.n_problems):
qp = instance.qp_problem
# Solve problem
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
r = m.solve()
# DEBUG
# print("niter = %d" % r.info.iter)
solution_dict = {'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter],
'obj_val': [r.info.obj_val]}
if r.info.status != "solved":
print("OSQP no warmstart did not solve the problem")
import ipdb; ipdb.set_trace()
res_list_no_ws.append(pd.DataFrame(solution_dict))
# Update model
current_mu = instance.mu
current_F_data = instance.F.data
current_D_data = instance.D.data
if i % self.n_qp_per_update == 0:
# print("Update everything: mu, F, D")
# Update everything
new_mu = alpha * np.random.randn(instance.n) + (1 - alpha) * current_mu
new_F = instance.F.copy()
new_F.data = alpha * np.random.randn(nnzF) + (1 - alpha) * current_F_data
new_D = instance.D.copy()
new_D.data = alpha * np.random.rand(instance.n) * \
np.sqrt(instance.k) + (1 - alpha) * current_D_data
instance.update_parameters(new_mu, new_F, new_D)
else:
# print("Update only mu")
# Update only mu
new_mu = alpha * np.random.randn(instance.n) + (1 - alpha) * current_mu
instance.update_parameters(new_mu)
# Get full warm-start
res_no_ws = pd.concat(res_list_no_ws)
# Store file
res_no_ws.to_csv(n_file_name, index=False)
# Plot results
# import matplotlib.pylab as plt
# plt.figure(0)
# plt.plot(X_no_ws.T)
# plt.title("No Warm Start")
# plt.show(block=False)
'''
Solve problem with warm start
'''
# print("Solving with warm start")
# Solution directory
ws_path = os.path.join('.', 'results', 'parametric_problems',
'OSQP warmstart',
'Portfolio',
)
# Create directory for the results
make_sure_path_exists(ws_path)
# Check if solution already exists
n_file_name = os.path.join(ws_path, 'n%i.csv' % self.n_factors)
if not os.path.isfile(n_file_name):
# Setup solver
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
res_list_ws = [] # Initialize results
for i in range(self.n_problems):
# Solve problem
r = m.solve()
# DEBUG
# print("niter = %d" % r.info.iter)
if r.info.status != "solved":
print("OSQP warmstart did not solve the problem")
# Get results
solution_dict = {'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter],
'obj_val': [r.info.obj_val]}
res_list_ws.append(pd.DataFrame(solution_dict))
# Update model
current_mu = instance.mu
current_F_data = instance.F.data
current_D_data = instance.D.data
if i % self.n_qp_per_update == 0:
# print("Update everything: mu, F, D")
# Update everything
new_mu = alpha * np.random.randn(instance.n) + (1 - alpha) * current_mu
new_F = instance.F.copy()
new_F.data = alpha * np.random.randn(nnzF) + (1 - alpha) * current_F_data
new_D = instance.D.copy()
new_D.data = alpha * np.random.rand(instance.n) * \
np.sqrt(instance.k) + (1 - alpha) * current_D_data
instance.update_parameters(new_mu, new_F, new_D)
# Update solver
m.update(q=instance.qp_problem['q'],
Px=instance.qp_problem['P'].data,
Ax=instance.qp_problem['A'].data)
else:
# print("Update only mu")
# Update only mu
new_mu = alpha * np.random.randn(instance.n) + (1 - alpha) * current_mu
instance.update_parameters(new_mu)
# Update solver
m.update(q=instance.qp_problem['q'])
# Get full warm-start
res_ws = pd.concat(res_list_ws)
# Store file
res_ws.to_csv(n_file_name, index=False)
def solve(self):
"""
Solve MPC problem
"""
print("Solve MPC problem for dimension %i" % self.dimension)
# Create example instance
instance = ControlExample(self.dimension)
qp = instance.qp_problem
x0 = np.copy(instance.x0)
'''
Solve problem without warm start
'''
# Solution directory
no_ws_path = os.path.join(
'.',
'results',
'parametric_problems',
'OSQP no warmstart',
'MPC',
)
# Create directory for the results
make_sure_path_exists(no_ws_path)
# Check if solution already exists
n_file_name = os.path.join(no_ws_path, 'n%i.csv' % self.dimension)
if not os.path.isfile(n_file_name):
# Initialize states and inputs for the whole simulation
X_no_ws = np.zeros((instance.nx, self.n_simulation + 1))
U_no_ws = np.zeros((instance.nu, self.n_simulation))
X_no_ws[:, 0] = x0
res_list_no_ws = [] # Initialize results
for i in range(self.n_simulation):
# Solve problem
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
r = m.solve()
solution_dict = {
'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter]
}
if r.info.status != "solved":
print("OSQP no warmstart did not solve the problem")
res_list_no_ws.append(pd.DataFrame(solution_dict))
# Get input
U_no_ws[:,
i] = r.x[instance.nx * (instance.T + 1):instance.nx *
(instance.T + 1) + instance.nu]
# Propagate state
X_no_ws[:, i + 1] = instance.A.dot(X_no_ws[:, i]) + \
instance.B.dot(U_no_ws[:, i])
# Update initial state
instance.update_x0(X_no_ws[:, i + 1])
# Get full warm-start
res_no_ws = pd.concat(res_list_no_ws)
# Store file
res_no_ws.to_csv(n_file_name, index=False)
# Plot results
# import matplotlib.pylab as plt
# plt.figure(1)
# plt.plot(X_no_ws.T)
# plt.title("No Warm Start")
# plt.show(block=False)
'''
Solve problem with warm start
'''
# Solution directory
ws_path = os.path.join(
'.',
'results',
'parametric_problems',
'OSQP warmstart',
'MPC',
)
# Create directory for the results
make_sure_path_exists(ws_path)
# Check if solution already exists
n_file_name = os.path.join(ws_path, 'n%i.csv' % self.dimension)
# Set initial state as x0
instance.update_x0(x0)
if not os.path.isfile(n_file_name):
# Setup solver
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
# Initialize states and inputs for the whole simulation
X_ws = np.zeros((instance.nx, self.n_simulation + 1))
U_ws = np.zeros((instance.nu, self.n_simulation))
X_ws[:, 0] = x0
res_list_ws = [] # Initialize results
for i in range(self.n_simulation):
# Solve problem
r = m.solve()
if r.info.status != "solved":
print("OSQP no warmstart did not solve the problem")
# Get results
solution_dict = {
'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter]
}
res_list_ws.append(pd.DataFrame(solution_dict))
# Get input
U_ws[:, i] = r.x[instance.nx * (instance.T + 1):instance.nx *
(instance.T + 1) + instance.nu]
# Propagate state
X_ws[:, i + 1] = instance.A.dot(X_ws[:, i]) + \
instance.B.dot(U_ws[:, i])
# Update initial state
instance.update_x0(X_ws[:, i + 1])
# Update solver
m.update(l=instance.qp_problem['l'],
u=instance.qp_problem['u'])
# Get full warm-start
res_ws = pd.concat(res_list_ws)
# Store file
res_ws.to_csv(n_file_name, index=False)
def solve(self):
"""
Solve Lasso problem
"""
print("Solve Lasso problem for dimension %i" % self.dimension)
# Create example instance
instance = LassoExample(self.dimension)
qp = instance.qp_problem
# Create lambda array
lambda_array = np.logspace(np.log10(self.minimum_lambda_over_max *
instance.lambda_max),
np.log10(instance.lambda_max),
self.n_problems)[::-1] # From max to min
'''
Solve problem without warm start
'''
# print("Solving without warm start")
# Solution directory
no_ws_path = os.path.join('.', 'results', 'parametric_problems',
'OSQP no warmstart',
'Lasso',
)
# Create directory for the results
make_sure_path_exists(no_ws_path)
# Check if solution already exists
n_file_name = os.path.join(no_ws_path, 'n%i.csv' % self.dimension)
if not os.path.isfile(n_file_name):
res_list_no_ws = [] # Initialize results
for lambda_val in lambda_array:
# Update lambda
instance.update_lambda(lambda_val)
# Solve problem
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
r = m.solve()
# DEBUG
# print("Lambda = %.4e,\t niter = %d" % (lambda_val, r.info.iter))
if r.info.status != "solved":
print("OSQP no warmstart did not solve the problem")
solution_dict = {'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter]}
res_list_no_ws.append(pd.DataFrame(solution_dict))
# Get full warm-start
res_no_ws = pd.concat(res_list_no_ws)
# Store file
res_no_ws.to_csv(n_file_name, index=False)
'''
Solve problem with warm start
'''
# print("Solving with warm start")
# Solution directory
ws_path = os.path.join('.', 'results', 'parametric_problems',
'OSQP warmstart',
'Lasso',
)
# Create directory for the results
make_sure_path_exists(ws_path)
# Check if solution already exists
n_file_name = os.path.join(ws_path, 'n%i.csv' % self.dimension)
# Reset problem to first instance
instance.update_lambda(lambda_array[0])
# Setup solver
qp = instance.qp_problem
m = osqp.OSQP()
m.setup(qp['P'], qp['q'], qp['A'], qp['l'], qp['u'],
**self.osqp_settings)
if not os.path.isfile(n_file_name):
res_list_ws = [] # Initialize results
for lambda_val in lambda_array:
# Update lambda
instance.update_lambda(lambda_val)
m.update(q=qp['q'])
# Solve problem
r = m.solve()
# DEBUG
# print("Lambda = %.4e,\t niter = %d" % (lambda_val, r.info.iter))
if r.info.status != "solved":
print("OSQP warmstart did not solve the problem")
# Get results
solution_dict = {'status': [r.info.status],
'run_time': [r.info.run_time],
'iter': [r.info.iter]}
res_list_ws.append(pd.DataFrame(solution_dict))
# Get full warm-start
res_ws = pd.concat(res_list_ws)
# Store file
res_ws.to_csv(n_file_name, index=False)
else:
res_ws = pd.read_csv(n_file_name)
def __init__(self, configparser, configurationfile):
"""Initialize a particle filter based on a configurationfile"""
self.pool = None
# Parse the configuration file
self.configfile = configurationfile
self.config = collections.OrderedDict() # preserve order of entries
for section in configparser.sections():
self.config[section] = collections.OrderedDict()
for key, value in configparser.items(section):
self.config[section][key] = value
if not self.config['COMPUTING']['poolsize']:
self.poolsize = multiprocessing.cpu_count()
else:
self.poolsize = configparser.getint('COMPUTING', 'poolsize')
self.ID = self.config['FILTER']['runid']
logging.info('Initializing particle filter: {}'.format(self.ID))
ncvarinfo = {}
varlist = filterPick(self.config['VICASC2NC'].keys(),
re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var +
'_longname']
ncvarinfo[var]['column'] = self.config['VICASC2NC'][var +
'_column']
ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
ncvarinfo[var]['divideby'] = float(
self.config['VICASC2NC'][var + '_divideby'])
self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo
# initialize simulation time
self.start = parse_date(self.config['SIMULATION']['start'])
self.now = self.start
self.end = parse_date(self.config['SIMULATION']['end'])
self.runlength = dt.timedelta(
configparser.getint('SIMULATION', 'runlength'))
self.backstep = dt.timedelta(
configparser.getint('SIMULATION', 'backstep'))
# number of timestep to evaluate
self.neval = configparser.getint('FILTER', 'neval')
# Initialize particles
self.np = configparser.getint('FILTER', 'np')
self.particles = [
VicParticle(i + 1, 1. / self.np) for i in range(self.np)
]
for i in range(self.np):
p = self.particles[i]
p.vic['exe'] = self.config['MODELS']['vicexe']
p.route['exe'] = self.config['MODELS']['routeexe']
p.refvar = self.config['FILTER']['referencevar']
p.initstatefile = {}
p.initstatefile['vic'] = self.config['VICCONFIG'][
'INIT_STATE'].format(p.ID)
if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower(
) == 'startup':
p.initstatefile['route'] = \
self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
format(p.ID)
p.neval = self.neval
self.normalize_weights()
logging.info('Initialized {} particles'.format(self.np))
# set up run and archive paths for particles
runpath = self.config['FILTER']['rundir']
archivepath = self.config['FILTER']['archivedir']
subdirs = ['log', 'history', 'settings']
self.paths = {}
self.paths['run'] = {}
self.paths['archive'] = {}
for subdir in subdirs:
self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for filter {}:'.format(self.ID))
for topdir, dirdict in self.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
subdirs = ['config', 'data', 'log', 'state', 'history']
for p in self.particles:
p.paths = {}
p.paths['run'] = {}
p.paths['archive'] = {}
runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'],
p.ID)
archivepath = '{}/pid_{:03d}'.format(
self.config['FILTER']['archivedir'], p.ID)
for subdir in subdirs:
p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
p.paths['archive'][subdir] = '{}/{}'.format(
archivepath, subdir)
logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
for topdir, dirdict in p.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
# set history file for filter
filename = '{}.history'.format(self.ID)
self.hist = open(os.path.join(self.paths['run']['history'], filename),
'w')
# set history file for each particle
for p in self.particles:
filename = '{}.{:03d}.history'.format(self.ID, p.ID)
p.hist = open(os.path.join(p.paths['run']['history'], filename),
'w')
logging.info('Initialization complete')
def __init__(self, configparser, configurationfile):
"""Initialize a particle filter based on a configurationfile"""
self.pool = None
# Parse the configuration file
self.configfile = configurationfile
self.config = collections.OrderedDict() # preserve order of entries
for section in configparser.sections():
self.config[section] = collections.OrderedDict()
for key, value in configparser.items(section):
self.config[section][key] = value
if not self.config['COMPUTING']['poolsize']:
self.poolsize = multiprocessing.cpu_count()
else:
self.poolsize = configparser.getint('COMPUTING','poolsize')
self.ID = self.config['FILTER']['runid']
logging.info('Initializing particle filter: {}'.format(self.ID))
ncvarinfo = {}
varlist = filterPick(self.config['VICASC2NC'].keys(), re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var + '_longname']
ncvarinfo[var]['column'] = self.config['VICASC2NC'][var + '_column']
ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
ncvarinfo[var]['divideby'] = float(self.config['VICASC2NC'][var + '_divideby'])
self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo
# initialize simulation time
self.start = parse_date(self.config['SIMULATION']['start'])
self.now = self.start
self.end = parse_date(self.config['SIMULATION']['end'])
self.runlength = dt.timedelta(configparser.getint('SIMULATION',
'runlength'))
self.backstep = dt.timedelta(configparser.getint('SIMULATION',
'backstep'))
# number of timestep to evaluate
self.neval = configparser.getint('FILTER','neval')
# Initialize particles
self.np = configparser.getint('FILTER', 'np')
self.particles = [VicParticle(i+1, 1./self.np) for i in range(self.np)]
for i in range(self.np):
p = self.particles[i]
p.vic['exe'] = self.config['MODELS']['vicexe']
p.route['exe'] = self.config['MODELS']['routeexe']
p.refvar = self.config['FILTER']['referencevar']
p.initstatefile = {}
p.initstatefile['vic'] = self.config['VICCONFIG']['INIT_STATE'].format(p.ID)
if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower() == 'startup':
p.initstatefile['route'] = \
self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
format(p.ID)
p.neval = self.neval
self.normalize_weights()
logging.info('Initialized {} particles'.format(self.np))
# set up run and archive paths for particles
runpath = self.config['FILTER']['rundir']
archivepath = self.config['FILTER']['archivedir']
subdirs = ['log', 'history', 'settings']
self.paths = {}
self.paths['run'] = {}
self.paths['archive'] = {}
for subdir in subdirs:
self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for filter {}:'.format(self.ID))
for topdir, dirdict in self.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
subdirs = ['config', 'data', 'log', 'state', 'history']
for p in self.particles:
p.paths = {}
p.paths['run'] = {}
p.paths['archive'] = {}
runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'], p.ID)
archivepath = '{}/pid_{:03d}'.format(self.config['FILTER']['archivedir'], p.ID)
for subdir in subdirs:
p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
p.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
for topdir, dirdict in p.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
# set history file for filter
filename = '{}.history'.format(self.ID)
self.hist = open(os.path.join(self.paths['run']['history'], filename), 'w')
# set history file for each particle
for p in self.particles:
filename = '{}.{:03d}.history'.format(self.ID, p.ID)
p.hist = open(os.path.join(p.paths['run']['history'], filename), 'w')
logging.info('Initialization complete')