def print_solvers(): """ Информация по солверам. Дополнительные ссылки: - https://github.com/coin-or/Cbc - https://en.wikipedia.org/wiki/Branch_and_cut - https://coin-or.github.io/pulp/guides/how_to_configure_solvers.html """ print("\nВозможные солверы:", lst(pulp.list_solvers())) print("Доступные: ", lst(pulp.list_solvers(onlyAvailable=True))) default_solver = pulp.LpSolverDefault # 'PULP_CBC_CMD' print("Использован:", default_solver.name) print("Где находится:", pulp.get_solver(default_solver.name).path)
def load_pulp_solver(solver_names=ORDERED_SOLVERS, logger=None): """Load a pulp solver based on what is available. Args: solver_name (`list` of `str`, optional): A list of solver names in the order of loading preferences. logger (None, optional): A logging.Logger object Returns: pulp.apis.core.LpSolver: A pulp solver instance. """ if logger is None: logger = create_logger('optstoic.load_pulp_solver') if isinstance(solver_names, str): solver_names = [solver_names] elif not isinstance(solver_names, list): raise Exception("Argument solver_names must be a list!") # Load solvers in the order of preferences for solver_name in solver_names: kwargs = SOLVER_KWARGS.get(solver_name, None) pulp_solver = pulp.get_solver(**kwargs) if pulp_solver.available(): logger.warning("Pulp solver set to %s." % solver_name) if hasattr(pulp_solver, 'tmpDir'): pulp_solver.tmpDir = './' if solver_name == 'SCIP_CMD': scip_parameter_filepath = create_scip_parameter_file( parameters=SCIP_CMD_PARAMETERS, filepath=pulp_solver.tmpDir) pulp_solver.options = [ "-s", "{}".format(scip_parameter_filepath) ] if solver_name == 'GLPK_CMD': logger.warning( "GLPK takes a significantly longer time to solve " "OptStoic. Please be patient.") return pulp_solver logger.warning("No solver is available!") return None
def _solve_ilp(self, prob): import pulp old_path = os.environ["PATH"] if self.workflow.scheduler_solver_path is None: # Temporarily prepend the given snakemake env to the path, such that the solver can be found in any case. # This is needed for cluster envs, where the cluster job might have a different environment but # still needs access to the solver binary. os.environ["PATH"] = "{}:{}".format( self.workflow.scheduler_solver_path, os.environ["PATH"]) try: solver = (pulp.get_solver(self.scheduler_ilp_solver) if self.scheduler_ilp_solver else pulp.apis.LpSolverDefault) finally: os.environ["PATH"] = old_path solver.msg = self.workflow.verbose prob.solve(solver)
def job_selector_ilp(self, jobs): """ Job scheduling by optimization of resource usage by solving ILP using pulp """ import pulp from pulp import lpSum logger.info("Select jobs to execute...") # assert self.resources["_cores"] > 0 scheduled_jobs = { job: pulp.LpVariable( "job_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, job in enumerate(jobs) } size_gb = lambda f: f.size / 1e9 temp_files = { temp_file for job in jobs for temp_file in self.dag.temp_input(job) } temp_job_improvement = { temp_file: pulp.LpVariable("temp_file_{}".format(idx), lowBound=0, upBound=1, cat="Continuous") for idx, temp_file in enumerate(temp_files) } temp_file_deletable = { temp_file: pulp.LpVariable( "deletable_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, temp_file in enumerate(temp_files) } prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize) total_temp_size = max( sum([size_gb(temp_file) for temp_file in temp_files]), 1) total_core_requirement = sum( [max(job.resources.get("_cores", 1), 1) for job in jobs]) # Objective function # Job priority > Core load # Core load > temp file removal # Instant removal > temp size prob += (2 * total_core_requirement * 2 * total_temp_size * lpSum( [job.priority * scheduled_jobs[job] for job in jobs]) + 2 * total_temp_size * lpSum([ max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job] for job in jobs ]) + total_temp_size * lpSum([ temp_file_deletable[temp_file] * size_gb(temp_file) for temp_file in temp_files ]) + lpSum([ temp_job_improvement[temp_file] * size_gb(temp_file) for temp_file in temp_files ])) # Constraints: for name in self.workflow.global_resources: prob += (lpSum([ scheduled_jobs[job] * job.resources.get(name, 0) for job in jobs ]) <= self.resources[name]) # Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file remaining_jobs = self.remaining_jobs for temp_file in temp_files: prob += temp_job_improvement[temp_file] <= lpSum([ scheduled_jobs[job] * self.required_by_job(temp_file, job) for job in jobs ]) / lpSum([ self.required_by_job(temp_file, job) for job in remaining_jobs ]) prob += temp_file_deletable[temp_file] <= temp_job_improvement[ temp_file] solver = (pulp.get_solver(self.scheduler_ilp_solver) if self.scheduler_ilp_solver else pulp.apis.LpSolverDefault) solver.msg = self.workflow.verbose # disable extensive logging try: prob.solve(solver) except pulp.apis.core.PulpSolverError as e: logger.warning( "Failed to solve scheduling problem with ILP solver. Falling back to greedy solver. " "Run Snakemake with --verbose to see the full solver output for debugging the problem." ) return self.job_selector_greedy(jobs) selected_jobs = set(job for job, variable in scheduled_jobs.items() if variable.value() == 1.0) if not selected_jobs: logger.warning( "Failed to solve scheduling problem with ILP solver. Falling back to greedy solver." "Run Snakemake with --verbose to see the full solver output for debugging the problem." ) return self.job_selector_greedy(jobs) for name in self.workflow.global_resources: self.resources[name] -= sum( [job.resources.get(name, 0) for job in selected_jobs]) return selected_jobs
def fit(self, A, label): """ Function to a decode base of design matrix and test results Parameters: A (binary numpy 2d-array): The group testing matrix. label (binary numpy array): The vector of results of the group tests. Returns: self (GroupTestingDecoder): A decoder object including decoding solution """ if self.lambda_e is not None: # Use lambda_e if both lambda_p and lambda_n have same value self.lambda_p = self.lambda_e self.lambda_n = self.lambda_e m, n = A.shape alpha = A.sum(axis=1) label = np.array(label) positive_label = np.where(label == 1)[0] negative_label = np.where(label == 0)[0] # positive_label = [idx for idx,i in enumerate(label) if i==1] # negative_label = [idx for idx,i in enumerate(label) if i==0] # ------------------------------------- # Checking length of lambda_w try: if isinstance(self.lambda_w, list): assert len(self.lambda_w) == n except AssertionError: print( "length of lambda_w should be equal to number of individuals( numbers of columns in the group " "testing matrix)") # ------------------------------------- # Initializing the ILP problem p = pl.LpProblem('GroupTesting', pl.LpMinimize) # p.verbose(param['verbose']) # Variables kind if self.lp_relaxation: varCategory = 'Continuous' #self.solver_options['mip']= True else: varCategory = 'Binary' # Variable w w = pl.LpVariable.dicts('w', range(n), lowBound=0, upBound=1, cat=varCategory) # -------------------------------------- # Noiseless setting if self.is_it_noiseless: # Defining the objective function p += pl.lpSum([ self.lambda_w * w[i] if isinstance(self.lambda_w, (int, float)) else self.lambda_w[i] * w[i] for i in range(n) ]) # Constraints for i in positive_label: p += pl.lpSum([A[i][j] * w[j] for j in range(n)]) >= 1 for i in negative_label: p += pl.lpSum([A[i][j] * w[j] for j in range(n)]) == 0 # Prevalence lower-bound if self.defective_num_lower_bound is not None: p += pl.lpSum([w[k] for k in range(n) ]) >= self.defective_num_lower_bound # -------------------------------------- # Noisy setting else: ep = [] en = [] # Variable ep if len(positive_label) != 0: ep = pl.LpVariable.dicts(name='ep', indexs=list(positive_label), lowBound=0, upBound=1, cat=varCategory) # Variable en if len(negative_label) != 0: en = pl.LpVariable.dicts(name='en', indexs=list(negative_label), lowBound=0, upBound=self.en_upBound, cat=varCategory) # Defining the objective function p += pl.lpSum([self.lambda_w * w[i] if isinstance(self.lambda_w, (int, float)) else self.lambda_w[i] * w[i] for i in range(n)]) + \ pl.lpSum([self.lambda_p * ep[j] for j in positive_label]) + \ pl.lpSum([self.lambda_n * en[k] for k in negative_label]) # Constraints for i in positive_label: p += pl.lpSum([A[i][j] * w[j] for j in range(n)] + ep[i]) >= 1 for i in negative_label: if varCategory == 'Continuous': p += pl.lpSum([A[i][j] * w[j] for j in range(n)] + -1 * en[i]) == 0 else: p += pl.lpSum([-1 * A[i][j] * w[j] for j in range(n)] + alpha[i] * en[i]) >= 0 # Prevalence lower-bound if self.defective_num_lower_bound is not None: p += pl.lpSum([w[i] for i in range(n) ]) >= self.defective_num_lower_bound if self.solver_options is not None: solver = pl.get_solver(self.solver_name, **self.solver_options) else: solver = pl.get_solver(self.solver_name) p.solve(solver) if not self.lp_relaxation: p.roundSolution() # ---------------- self.prob_ = p #print("Status:", pl.LpStatus[p.status]) return self
def job_selector_ilp(self, jobs): """ Job scheduling by optimization of resource usage by solving ILP using pulp """ import pulp from pulp import lpSum # assert self.resources["_cores"] > 0 scheduled_jobs = { job: pulp.LpVariable( "job_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, job in enumerate(jobs) } temp_files = { temp_file for job in jobs for temp_file in self.dag.temp_input(job) } temp_job_improvement = { temp_file: pulp.LpVariable("temp_file_{}".format(idx), lowBound=0, upBound=1, cat="Continuous") for idx, temp_file in enumerate(temp_files) } temp_file_deletable = { temp_file: pulp.LpVariable( "deletable_{}".format(idx), lowBound=0, upBound=1, cat=pulp.LpInteger, ) for idx, temp_file in enumerate(temp_files) } prob = pulp.LpProblem("JobScheduler", pulp.LpMaximize) total_temp_size = max( sum([temp_file.size for temp_file in temp_files]), 1) total_core_requirement = sum( [max(job.resources.get("_cores", 1), 1) for job in jobs]) # Objective function # Job priority > Core load # Core load > temp file removal # Instant removal > temp size prob += (2 * total_core_requirement * 2 * total_temp_size * lpSum( [job.priority * scheduled_jobs[job] for job in jobs]) + 2 * total_temp_size * lpSum([ max(job.resources.get("_cores", 1), 1) * scheduled_jobs[job] for job in jobs ]) + total_temp_size * lpSum([ temp_file_deletable[temp_file] * temp_file.size for temp_file in temp_files ]) + lpSum([ temp_job_improvement[temp_file] * temp_file.size for temp_file in temp_files ])) # Constraints: for name in self.workflow.global_resources: prob += (lpSum([ scheduled_jobs[job] * job.resources.get(name, 0) for job in jobs ]) <= self.resources[name]) # Choose jobs that lead to "fastest" (minimum steps) removal of existing temp file remaining_jobs = self.remaining_jobs for temp_file in temp_files: prob += temp_job_improvement[temp_file] <= lpSum([ scheduled_jobs[job] * self.required_by_job(temp_file, job) for job in jobs ]) / lpSum([ self.required_by_job(temp_file, job) for job in remaining_jobs ]) prob += temp_file_deletable[temp_file] <= temp_job_improvement[ temp_file] solver = (pulp.get_solver(self.scheduler_ilp_solver) if self.scheduler_ilp_solver else pulp.apis.LpSolverDefault) solver.msg = False # disable extensive logging try: prob.solve(solver) except pulp.apis.core.PulpSolverError as e: raise WorkflowError( "Failed to solve the job scheduling problem with pulp. " "Please report a bug and use --scheduler greedy as a workaround:\n{}" .format(e)) selected_jobs = set(job for job, variable in scheduled_jobs.items() if variable.value() == 1.0) for name in self.workflow.global_resources: self.resources[name] -= sum( [job.resources.get(name, 0) for job in selected_jobs]) return selected_jobs
def fit(self, x, y): """ :param x: train input, ndarray, shape: (n_sample, n_feature) :param y: train output, ndarray, shape: (n_sample, 1) :return: None """ # number of samples n_sample = x.shape[0] # normalization # x, y = self.normalize(x, y) # get ELM parameters np.random.seed(self.random_state) array = np.random.normal(size=[self.n_feature + 1, self.n_hidden]) self.w, self.b = array[:-1, :], array[-1, :] bb = np.repeat(np.expand_dims(self.b, axis=0), n_sample, axis=0) H = np.dot(x, self.w) + bb # self.func = Layer['func'].lower() H = self.activate(H) # make LP prob = pl.LpProblem("myProblem", pl.LpMinimize) # set variables --- f_alpha_time, w_alpha_hidden # i think set the bound for weight is necessary, # otherwise it would be likely to get thousand and more # which result in unstable prediction ff = pl.LpVariable.dicts('f', indexs=(range(self.n_quantile), range(n_sample))) ww = pl.LpVariable.dicts('w', indexs=(range(self.n_quantile), range(self.n_hidden))) # ww = pl.LpVariable.dicts('w', indexs=(range(self.n_quantile), range(self.n_hidden)), lowBound=-10, upBound=10) # objection prob += pl.lpSum([ 1.0 * ff[i][j] for i in range(self.n_quantile) for j in range(n_sample) ]) # constrains for i in range(self.n_quantile): qt0, qt1 = 1 / self.quantiles[i], 1 / (self.quantiles[i] - 1) for j in range(n_sample): # Hw # 这里为什么是个列表,不应该求和吗? # 相当于所有分位点共用一套输入权重与偏置? temp1 = [H[j, k] * ww[i][k] for k in range(self.n_hidden)] prob += pl.lpSum([qt0 * ff[i][j]] + temp1 + [-y[j]]) >= 0 prob += pl.lpSum([qt1 * ff[i][j]] + temp1 + [-y[j]]) <= 0 # 是小于0吗?不应该也是大于0? prob += pl.lpSum(temp1) >= 0 prob += pl.lpSum(temp1) <= 1 if i != 0: temp2 = [ -H[j, k] * ww[i - 1][k] for k in range(self.n_hidden) ] prob += pl.lpSum(temp1 + temp2) >= 0 # prob.writeLP("MM.lp") # print(prob) # solve the LP # pl.list_solvers(1) # 'CPLEX_PY', 'PULP_CBC_CMD'(default), 'PULP_CHOCO_CMD' solver = pl.get_solver('PULP_CBC_CMD') prob.solve(solver) # prob.solve() # get Weight self.W = np.zeros((self.n_hidden, self.n_quantile)) for i in range(self.n_quantile): for j in range(self.n_hidden): self.W[j, i] = pl.value(ww[i][j]) print("objective=", pl.value(prob.objective))
# constraint: Q_ij is 1 for final states (equation 12) for s_i, s_j in product(lts_1.states, lts_2.states): if not lts_1[s_i]: q_ij = ns[f'Q_{s_i}_{s_j}'] lp += ( q_ij == 1., f'CSTR_5__({s_i}, {s_j})' ) # optionally store lp if path := self._lp_path: lp.writeLP(str(path)) # search for a solution logging.info(f'start solver ({lp.numVariables()} variables, {lp.numConstraints()} constraints)') duration, *_ = timeit(lambda: lp.solve(get_solver(self._backend, msg=False, threads=16))) logging.info(f'status: {LpStatus[lp.status]} ({duration:.3f}s)') assignments = {v.name: v.varValue for v in lp.variables()} return SimilarityTable( (s_i, s_j, assignments[f'Q_{s_i}_{s_j}']) for s_i, s_j in product(lts_1.states, lts_2.states) ) def simulate(self, lts_1: LTS, lts_2: LTS) -> SimilarityTable: logging.info(f'simulate {lts_1.name} by {lts_2.name}') duration, similarities = timeit(lambda: self._simulate(lts_1, lts_2)) logging.info(f'complete simulation in {duration:.3f}s') return similarities
# -*- coding: utf-8 -*- """ Created on Wed Mar 17 21:52:00 2021 @author: Amlan Ghosh """ import pulp as plp '''Setting a solver''' solver = plp.get_solver() '''Defining the problem and setting the sense''' model = plp.LpProblem('Name', sense = plp.LpMinimize) #For Minimization model = plp.LpProblem('Name', sense = plp.LpMaximize) #For Maximization '''Defining variables''' #Integer var_name = plp.LpVariable.dicts('Name', ((i) for i in iterator), upBound = None, lowBound = None, cat = plp.LpInteger) #dictionary of variables var_name = plp.LpVariable('Name', upBound = None, lowBound = None, cat = plp.LpInteger) #single varible #Binary var_name = plp.LpVariable.dicts('Name', ((i) for i in iterator), cat = plp.LpBinary) #dictionary of variables var_name = plp.LpVariable('Name', cat = plp.LpBiary) #single varible '''Defining constraints''' cons = {i: model.addConstraint(plp.LpConstraint(e = var_name, sense = plp.LpConstraintGE. rhs = 0, name = 'Name_'+i)) for i in iterator} #Greater than cons = {i: model.addConstraint(plp.LpConstraint(e = var_name, sense = plp.LpConstraintLE. rhs = 0, name = 'Name_'+i)) for i in iterator} #Less than '''Objective'''