def print_solvers():
"""
Информация по солверам.
Дополнительные ссылки:
- https://github.com/coin-or/Cbc
- https://en.wikipedia.org/wiki/Branch_and_cut
- https://coin-or.github.io/pulp/guides/how_to_configure_solvers.html
"""
print("\nВозможные солверы:", lst(pulp.list_solvers()))
print("Доступные: ", lst(pulp.list_solvers(onlyAvailable=True)))
default_solver = pulp.LpSolverDefault # 'PULP_CBC_CMD'
print("Использован:", default_solver.name)
print("Где находится:", pulp.get_solver(default_solver.name).path)
def load_pulp_solver(solver_names=ORDERED_SOLVERS, logger=None):
"""Load a pulp solver based on what is available.
Args:
solver_name (`list` of `str`, optional): A list of solver names in the order of
loading preferences.
logger (None, optional): A logging.Logger object
Returns:
pulp.apis.core.LpSolver: A pulp solver instance.
"""
if logger is None:
logger = create_logger('optstoic.load_pulp_solver')
if isinstance(solver_names, str):
solver_names = [solver_names]
elif not isinstance(solver_names, list):
raise Exception("Argument solver_names must be a list!")
# Load solvers in the order of preferences
for solver_name in solver_names:
kwargs = SOLVER_KWARGS.get(solver_name, None)
pulp_solver = pulp.get_solver(**kwargs)
if pulp_solver.available():
logger.warning("Pulp solver set to %s." % solver_name)
if hasattr(pulp_solver, 'tmpDir'):
pulp_solver.tmpDir = './'
if solver_name == 'SCIP_CMD':
scip_parameter_filepath = create_scip_parameter_file(
parameters=SCIP_CMD_PARAMETERS,
filepath=pulp_solver.tmpDir)
pulp_solver.options = [
"-s", "{}".format(scip_parameter_filepath)
]
if solver_name == 'GLPK_CMD':
logger.warning(
"GLPK takes a significantly longer time to solve "
"OptStoic. Please be patient.")
return pulp_solver
logger.warning("No solver is available!")
return None
def _solve_ilp(self, prob):
import pulp
old_path = os.environ["PATH"]
if self.workflow.scheduler_solver_path is None:
# Temporarily prepend the given snakemake env to the path, such that the solver can be found in any case.
# This is needed for cluster envs, where the cluster job might have a different environment but
# still needs access to the solver binary.
os.environ["PATH"] = "{}:{}".format(
self.workflow.scheduler_solver_path, os.environ["PATH"])
try:
solver = (pulp.get_solver(self.scheduler_ilp_solver) if
self.scheduler_ilp_solver else pulp.apis.LpSolverDefault)
finally:
os.environ["PATH"] = old_path
solver.msg = self.workflow.verbose
prob.solve(solver)
def job_selector_ilp(self, jobs):
"""
Job scheduling by optimization of resource usage by solving ILP using pulp
"""
import pulp
from pulp import lpSum
logger.info("Select jobs to execute...")
# assert self.resources["_cores"] > 0
scheduled_jobs = {
job: pulp.LpVariable(
"job_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, job in enumerate(jobs)
}
size_gb = lambda f: f.size / 1e9
temp_files = {
temp_file
for job in jobs for temp_file in self.dag.temp_input(job)
}
temp_job_improvement = {
temp_file: pulp.LpVariable("temp_file_{}".format(idx),
lowBound=0,
upBound=1,
cat="Continuous")
for idx, temp_file in enumerate(temp_files)
}
temp_file_deletable = {
temp_file: pulp.LpVariable(
"deletable_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, temp_file in enumerate(temp_files)
}
prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize)
total_temp_size = max(
sum([size_gb(temp_file) for temp_file in temp_files]), 1)
total_core_requirement = sum(
[max(job.resources.get("_cores", 1), 1) for job in jobs])
# Objective function
# Job priority > Core load
# Core load > temp file removal
# Instant removal > temp size
prob += (2 * total_core_requirement * 2 * total_temp_size * lpSum(
[job.priority * scheduled_jobs[job]
for job in jobs]) + 2 * total_temp_size * lpSum([
max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job]
for job in jobs
]) + total_temp_size * lpSum([
temp_file_deletable[temp_file] * size_gb(temp_file)
for temp_file in temp_files
]) + lpSum([
temp_job_improvement[temp_file] * size_gb(temp_file)
for temp_file in temp_files
]))
# Constraints:
for name in self.workflow.global_resources:
prob += (lpSum([
scheduled_jobs[job] * job.resources.get(name, 0)
for job in jobs
]) <= self.resources[name])
# Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file
remaining_jobs = self.remaining_jobs
for temp_file in temp_files:
prob += temp_job_improvement[temp_file] <= lpSum([
scheduled_jobs[job] * self.required_by_job(temp_file, job)
for job in jobs
]) / lpSum([
self.required_by_job(temp_file, job) for job in remaining_jobs
])
prob += temp_file_deletable[temp_file] <= temp_job_improvement[
temp_file]
solver = (pulp.get_solver(self.scheduler_ilp_solver)
if self.scheduler_ilp_solver else pulp.apis.LpSolverDefault)
solver.msg = self.workflow.verbose
# disable extensive logging
try:
prob.solve(solver)
except pulp.apis.core.PulpSolverError as e:
logger.warning(
"Failed to solve scheduling problem with ILP solver. Falling back to greedy solver. "
"Run Snakemake with --verbose to see the full solver output for debugging the problem."
)
return self.job_selector_greedy(jobs)
selected_jobs = set(job for job, variable in scheduled_jobs.items()
if variable.value() == 1.0)
if not selected_jobs:
logger.warning(
"Failed to solve scheduling problem with ILP solver. Falling back to greedy solver."
"Run Snakemake with --verbose to see the full solver output for debugging the problem."
)
return self.job_selector_greedy(jobs)
for name in self.workflow.global_resources:
self.resources[name] -= sum(
[job.resources.get(name, 0) for job in selected_jobs])
return selected_jobs
def fit(self, A, label):
"""
Function to a decode base of design matrix and test results
Parameters:
A (binary numpy 2d-array): The group testing matrix.
label (binary numpy array): The vector of results of the group tests.
Returns:
self (GroupTestingDecoder): A decoder object including decoding solution
"""
if self.lambda_e is not None:
# Use lambda_e if both lambda_p and lambda_n have same value
self.lambda_p = self.lambda_e
self.lambda_n = self.lambda_e
m, n = A.shape
alpha = A.sum(axis=1)
label = np.array(label)
positive_label = np.where(label == 1)[0]
negative_label = np.where(label == 0)[0]
# positive_label = [idx for idx,i in enumerate(label) if i==1]
# negative_label = [idx for idx,i in enumerate(label) if i==0]
# -------------------------------------
# Checking length of lambda_w
try:
if isinstance(self.lambda_w, list):
assert len(self.lambda_w) == n
except AssertionError:
print(
"length of lambda_w should be equal to number of individuals( numbers of columns in the group "
"testing matrix)")
# -------------------------------------
# Initializing the ILP problem
p = pl.LpProblem('GroupTesting', pl.LpMinimize)
# p.verbose(param['verbose'])
# Variables kind
if self.lp_relaxation:
varCategory = 'Continuous'
#self.solver_options['mip']= True
else:
varCategory = 'Binary'
# Variable w
w = pl.LpVariable.dicts('w',
range(n),
lowBound=0,
upBound=1,
cat=varCategory)
# --------------------------------------
# Noiseless setting
if self.is_it_noiseless:
# Defining the objective function
p += pl.lpSum([
self.lambda_w *
w[i] if isinstance(self.lambda_w,
(int, float)) else self.lambda_w[i] * w[i]
for i in range(n)
])
# Constraints
for i in positive_label:
p += pl.lpSum([A[i][j] * w[j] for j in range(n)]) >= 1
for i in negative_label:
p += pl.lpSum([A[i][j] * w[j] for j in range(n)]) == 0
# Prevalence lower-bound
if self.defective_num_lower_bound is not None:
p += pl.lpSum([w[k] for k in range(n)
]) >= self.defective_num_lower_bound
# --------------------------------------
# Noisy setting
else:
ep = []
en = []
# Variable ep
if len(positive_label) != 0:
ep = pl.LpVariable.dicts(name='ep',
indexs=list(positive_label),
lowBound=0,
upBound=1,
cat=varCategory)
# Variable en
if len(negative_label) != 0:
en = pl.LpVariable.dicts(name='en',
indexs=list(negative_label),
lowBound=0,
upBound=self.en_upBound,
cat=varCategory)
# Defining the objective function
p += pl.lpSum([self.lambda_w * w[i] if isinstance(self.lambda_w, (int, float)) else self.lambda_w[i] * w[i]
for i in range(n)]) + \
pl.lpSum([self.lambda_p * ep[j] for j in positive_label]) + \
pl.lpSum([self.lambda_n * en[k] for k in negative_label])
# Constraints
for i in positive_label:
p += pl.lpSum([A[i][j] * w[j] for j in range(n)] + ep[i]) >= 1
for i in negative_label:
if varCategory == 'Continuous':
p += pl.lpSum([A[i][j] * w[j]
for j in range(n)] + -1 * en[i]) == 0
else:
p += pl.lpSum([-1 * A[i][j] * w[j]
for j in range(n)] + alpha[i] * en[i]) >= 0
# Prevalence lower-bound
if self.defective_num_lower_bound is not None:
p += pl.lpSum([w[i] for i in range(n)
]) >= self.defective_num_lower_bound
if self.solver_options is not None:
solver = pl.get_solver(self.solver_name, **self.solver_options)
else:
solver = pl.get_solver(self.solver_name)
p.solve(solver)
if not self.lp_relaxation:
p.roundSolution()
# ----------------
self.prob_ = p
#print("Status:", pl.LpStatus[p.status])
return self
def job_selector_ilp(self, jobs):
"""
Job scheduling by optimization of resource usage by solving ILP using pulp
"""
import pulp
from pulp import lpSum
# assert self.resources["_cores"] > 0
scheduled_jobs = {
job: pulp.LpVariable(
"job_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, job in enumerate(jobs)
}
temp_files = {
temp_file
for job in jobs for temp_file in self.dag.temp_input(job)
}
temp_job_improvement = {
temp_file: pulp.LpVariable("temp_file_{}".format(idx),
lowBound=0,
upBound=1,
cat="Continuous")
for idx, temp_file in enumerate(temp_files)
}
temp_file_deletable = {
temp_file: pulp.LpVariable(
"deletable_{}".format(idx),
lowBound=0,
upBound=1,
cat=pulp.LpInteger,
)
for idx, temp_file in enumerate(temp_files)
}
prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize)
total_temp_size = max(
sum([temp_file.size for temp_file in temp_files]), 1)
total_core_requirement = sum(
[max(job.resources.get("_cores", 1), 1) for job in jobs])
# Objective function
# Job priority > Core load
# Core load > temp file removal
# Instant removal > temp size
prob += (2 * total_core_requirement * 2 * total_temp_size * lpSum(
[job.priority * scheduled_jobs[job]
for job in jobs]) + 2 * total_temp_size * lpSum([
max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job]
for job in jobs
]) + total_temp_size * lpSum([
temp_file_deletable[temp_file] * temp_file.size
for temp_file in temp_files
]) + lpSum([
temp_job_improvement[temp_file] * temp_file.size
for temp_file in temp_files
]))
# Constraints:
for name in self.workflow.global_resources:
prob += (lpSum([
scheduled_jobs[job] * job.resources.get(name, 0)
for job in jobs
]) <= self.resources[name])
# Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file
remaining_jobs = self.remaining_jobs
for temp_file in temp_files:
prob += temp_job_improvement[temp_file] <= lpSum([
scheduled_jobs[job] * self.required_by_job(temp_file, job)
for job in jobs
]) / lpSum([
self.required_by_job(temp_file, job) for job in remaining_jobs
])
prob += temp_file_deletable[temp_file] <= temp_job_improvement[
temp_file]
solver = (pulp.get_solver(self.scheduler_ilp_solver)
if self.scheduler_ilp_solver else pulp.apis.LpSolverDefault)
solver.msg = False
# disable extensive logging
try:
prob.solve(solver)
except pulp.apis.core.PulpSolverError as e:
raise WorkflowError(
"Failed to solve the job scheduling problem with pulp. "
"Please report a bug and use --scheduler greedy as a workaround:\n{}"
.format(e))
selected_jobs = set(job for job, variable in scheduled_jobs.items()
if variable.value() == 1.0)
for name in self.workflow.global_resources:
self.resources[name] -= sum(
[job.resources.get(name, 0) for job in selected_jobs])
return selected_jobs
def fit(self, x, y):
"""
:param x:
train input, ndarray, shape: (n_sample, n_feature)
:param y:
train output, ndarray, shape: (n_sample, 1)
:return:
None
"""
# number of samples
n_sample = x.shape[0]
# normalization
# x, y = self.normalize(x, y)
# get ELM parameters
np.random.seed(self.random_state)
array = np.random.normal(size=[self.n_feature + 1, self.n_hidden])
self.w, self.b = array[:-1, :], array[-1, :]
bb = np.repeat(np.expand_dims(self.b, axis=0), n_sample, axis=0)
H = np.dot(x, self.w) + bb
# self.func = Layer['func'].lower()
H = self.activate(H)
# make LP
prob = pl.LpProblem("myProblem", pl.LpMinimize)
# set variables --- f_alpha_time, w_alpha_hidden
# i think set the bound for weight is necessary,
# otherwise it would be likely to get thousand and more
# which result in unstable prediction
ff = pl.LpVariable.dicts('f',
indexs=(range(self.n_quantile),
range(n_sample)))
ww = pl.LpVariable.dicts('w',
indexs=(range(self.n_quantile),
range(self.n_hidden)))
# ww = pl.LpVariable.dicts('w', indexs=(range(self.n_quantile), range(self.n_hidden)), lowBound=-10, upBound=10)
# objection
prob += pl.lpSum([
1.0 * ff[i][j] for i in range(self.n_quantile)
for j in range(n_sample)
])
# constrains
for i in range(self.n_quantile):
qt0, qt1 = 1 / self.quantiles[i], 1 / (self.quantiles[i] - 1)
for j in range(n_sample):
# Hw
# 这里为什么是个列表，不应该求和吗？
# 相当于所有分位点共用一套输入权重与偏置？
temp1 = [H[j, k] * ww[i][k] for k in range(self.n_hidden)]
prob += pl.lpSum([qt0 * ff[i][j]] + temp1 + [-y[j]]) >= 0
prob += pl.lpSum([qt1 * ff[i][j]] + temp1 +
[-y[j]]) <= 0 # 是小于0吗？不应该也是大于0？
prob += pl.lpSum(temp1) >= 0
prob += pl.lpSum(temp1) <= 1
if i != 0:
temp2 = [
-H[j, k] * ww[i - 1][k] for k in range(self.n_hidden)
]
prob += pl.lpSum(temp1 + temp2) >= 0
# prob.writeLP("MM.lp")
# print(prob)
# solve the LP
# pl.list_solvers(1)
# 'CPLEX_PY', 'PULP_CBC_CMD'(default), 'PULP_CHOCO_CMD'
solver = pl.get_solver('PULP_CBC_CMD')
prob.solve(solver)
# prob.solve()
# get Weight
self.W = np.zeros((self.n_hidden, self.n_quantile))
for i in range(self.n_quantile):
for j in range(self.n_hidden):
self.W[j, i] = pl.value(ww[i][j])
print("objective=", pl.value(prob.objective))
# constraint: Q_ij is 1 for final states (equation 12)
for s_i, s_j in product(lts_1.states, lts_2.states):
if not lts_1[s_i]:
q_ij = ns[f'Q_{s_i}_{s_j}']
lp += (
q_ij == 1.,
f'CSTR_5__({s_i}, {s_j})'
)
# optionally store lp
if path := self._lp_path:
lp.writeLP(str(path))
# search for a solution
logging.info(f'start solver ({lp.numVariables()} variables, {lp.numConstraints()} constraints)')
duration, *_ = timeit(lambda: lp.solve(get_solver(self._backend, msg=False, threads=16)))
logging.info(f'status: {LpStatus[lp.status]} ({duration:.3f}s)')
assignments = {v.name: v.varValue for v in lp.variables()}
return SimilarityTable(
(s_i, s_j, assignments[f'Q_{s_i}_{s_j}']) for s_i, s_j in product(lts_1.states, lts_2.states)
)
def simulate(self, lts_1: LTS, lts_2: LTS) -> SimilarityTable:
logging.info(f'simulate {lts_1.name} by {lts_2.name}')
duration, similarities = timeit(lambda: self._simulate(lts_1, lts_2))
logging.info(f'complete simulation in {duration:.3f}s')
return similarities
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 17 21:52:00 2021
@author: Amlan Ghosh
"""
import pulp as plp
'''Setting a solver'''
solver = plp.get_solver()
'''Defining the problem and setting the sense'''
model = plp.LpProblem('Name', sense = plp.LpMinimize) #For Minimization
model = plp.LpProblem('Name', sense = plp.LpMaximize) #For Maximization
'''Defining variables'''
#Integer
var_name = plp.LpVariable.dicts('Name', ((i) for i in iterator), upBound = None, lowBound = None, cat = plp.LpInteger) #dictionary of variables
var_name = plp.LpVariable('Name', upBound = None, lowBound = None, cat = plp.LpInteger) #single varible
#Binary
var_name = plp.LpVariable.dicts('Name', ((i) for i in iterator), cat = plp.LpBinary) #dictionary of variables
var_name = plp.LpVariable('Name', cat = plp.LpBiary) #single varible
'''Defining constraints'''
cons = {i: model.addConstraint(plp.LpConstraint(e = var_name, sense = plp.LpConstraintGE. rhs = 0, name = 'Name_'+i)) for i in iterator} #Greater than
cons = {i: model.addConstraint(plp.LpConstraint(e = var_name, sense = plp.LpConstraintLE. rhs = 0, name = 'Name_'+i)) for i in iterator} #Less than
'''Objective'''