def compute_features(self, task, cpu_seconds = None): """Read or compute features of an instance.""" # grab precomputed feature data csv_path = task + ".features.csv" assert os.path.exists(csv_path) features_array = numpy.recfromcsv(csv_path) features = features_array.tolist() names = features_array.dtype.names # accumulate their cost assert names[0] == "cpu_cost" cpu_cost = features[0] borg.get_accountant().charge_cpu(cpu_cost) # handle timeout logic, and we're done if cpu_seconds is not None: if cpu_cost >= cpu_seconds: return (["cpu_cost"], [cpu_seconds]) else: assert len(names) > 1 return (names, features)
def run_then_pause(self, budget): """Unpause the solver for the specified duration.""" assert not self._terminated position = self._elapsed + budget logger.detail( "moving %s run to %.0f of %.0f (from %.0f)", self._run.solver, position, self._run.cost, self._elapsed, ) if position >= self._run.cost: borg.get_accountant().charge_cpu(self._run.cost - self._elapsed) self._elapsed = self._run.cost self._terminated = True return self._run.success else: borg.get_accountant().charge_cpu(budget) self._elapsed = position return None
def __call__(self, task, budget, cores = 1): queue = multiprocessing.Queue() solvers = [] for _ in xrange(cores): solver_id = uuid.uuid4() solver = cargo.grab(self._domain.solvers.values())(task, queue, solver_id) solvers.append(solver) try: for solver in solvers: solver.unpause_for(borg.unicore_cpu_budget(budget)) remaining = len(solvers) while remaining > 0: (solver_id, run_cpu_cost, answer, _) = queue.get() remaining -= 1 borg.get_accountant().charge_cpu(run_cpu_cost) if self._domain.is_final(task, answer): return answer return None finally: for solver in solvers: solver.stop()
def get_claspfolio_features_for(asp_path, binaries_path): """Invoke claspre to compute features of an ASP instance.""" previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime # get feature names claspre_path = os.path.join(binaries_path, "claspfolio-0.8.0-x86-linux/clasp+pre-1.3.4") (names_out, _) = borg.util.check_call_capturing([claspre_path, "--list-features"]) (dynamic_names_out, static_names_out) = names_out.splitlines() dynamic_names = normalized_claspre_names(dynamic_names_out.split(",")) static_names = normalized_claspre_names(static_names_out.split(",")) # compute feature values values_command = [claspre_path, "--rand-prob=10,30", "--search-limit=300,10", "--features=C1", "--file", asp_path] num_restarts = 10 logger.info("running %s", values_command) (values_out, _, _) = borg.util.call_capturing(values_command) values_per = [map(parse_claspre_value, l.split(",")) for l in values_out.strip().splitlines()] if len(values_per) < num_restarts + 1: # claspre failed, or the instance was solved in preprocessing if len(values_per) == 0: # (claspre died) values_per = [[0.0] * len(static_names)] missing = num_restarts - len(values_per) + 1 values_per = values_per[:-1] + ([[0.0] * len(dynamic_names)] * missing) + values_per[-1:] else: assert len(values_per) == num_restarts + 1 # pull them together names = [] values = [] for i in xrange(num_restarts): names += ["restart{0}-{1}".format(i, n) for n in dynamic_names] values += values_per[i] names += static_names values += values_per[-1] # ... cost = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime - previous_utime borg.get_accountant().charge_cpu(cost) logger.info("collected features of %s in %.2fs", asp_path, cost) assert len(names) == len(values) return (names, values)
def __call__(self, budget): """Unpause the solver, block for some limit, and terminate it.""" self.unpause_for(budget) (solver_id, run_cpu_cost, answer, terminated) = self._stm_queue.get() assert solver_id == self._solver_id self.stop() borg.get_accountant().charge_cpu(run_cpu_cost) return answer
def compute_features(self, instance): """Return static features of an instance.""" # get features with cost with_cost = self._suite.run_data.get_feature_vector(instance) borg.get_accountant().charge_cpu(with_cost["cpu_cost"]) # return the features features = dict(with_cost.iteritems()) del features["cpu_cost"] return (features.keys(), features.values())
def __call__(self, budget): """Unpause the solver, block for some limit, and terminate it.""" self.unpause_for(budget) response = self._stm_queue.get() if isinstance(response, Exception): raise response else: (solver_id, run_cpu_cost, answer, terminated) = response assert solver_id == self._solver_id self.stop() borg.get_accountant().charge_cpu(run_cpu_cost) return answer
def main( model_path, solvers_path, input_path, seed = 42, budget = 3600.0, cores = 1, speed = borg.defaults.machine_speed, quiet = False ): """Solve a problem instance.""" # XXX hackish borg.defaults.machine_speed = speed try: # general setup enable_output() if not quiet: cargo.get_logger("borg.solvers", level = "DETAIL") numpy.random.seed(seed) random.seed(numpy.random.randint(2**31)) # run the solver bundle = borg.load_solvers(solvers_path) logger.info("loaded portfolio model from %s", model_path) with open(model_path) as file: portfolio = pickle.load(file) logger.info("solving %s", input_path) with bundle.domain.task_from_path(input_path) as task: remaining = budget - borg.get_accountant().total.cpu_seconds answer = portfolio(task, bundle, borg.Cost(cpu_seconds = remaining), cores) return bundle.domain.show_answer(task, answer) except KeyboardInterrupt: print "\nc terminating on SIGINT"
def main(model_path, solvers_path, input_path, seed=42, budget=3600.0, cores=1, speed=borg.defaults.machine_speed, quiet=False): """Solve a problem instance.""" # XXX hackish borg.defaults.machine_speed = speed try: # general setup enable_output() if not quiet: borg.get_logger("borg.solvers", level="DETAIL") borg.statistics.set_prng_seeds(seed) # run the solver bundle = borg.load_solvers(solvers_path) logger.info("loaded portfolio model from %s", model_path) with open(model_path) as file: portfolio = pickle.load(file) logger.info("solving %s", input_path) with bundle.domain.task_from_path(input_path) as task: remaining = budget - borg.get_accountant().total.cpu_seconds answer = portfolio(task, bundle, borg.Cost(cpu_seconds=remaining), cores) return bundle.domain.show_answer(task, answer) except KeyboardInterrupt: print "\nc terminating on SIGINT"
def _solve(self, task, bundle, budget, cores): # print oracle knowledge, if any #(runs,) = borg.portfolios.get_task_run_data([task]).values() #(oracle_history, oracle_counts, _) = \ #borg.portfolios.action_rates_from_runs( #self._solver_name_index, #self._budget_index, #runs.tolist(), #) #true_rates = oracle_history / oracle_counts #hopeless = numpy.sum(true_rates) < 1e-2 #messages = [] #messages.append("true rates:\n%s" % cargo.pretty_probability_matrix(true_rates)) # obtain features (_, features) = self._domain.compute_features(task) # select a solver queue = multiprocessing.Queue() running = {} paused = [] failed = [] answer = None while True: # obtain model predictions failed_indices = [] for solver in failed + paused + running.values(): (total_b,) = numpy.digitize([solver.cpu_budgeted], self._budgets) if total_b == 0: # XXX hack total_b += 1 failed_indices.append((solver.s, total_b - 1)) (tclass_weights_L, tclass_rates_LSB) = self._model.predict(failed_indices, features) (L, S, B) = tclass_rates_LSB.shape # XXX force determinism #for (s, b) in failed_indices: #tclass_rates_LSB[:, s, :b + 1] = 1e-6 # prepare augmented PMF matrix augmented_tclass_arrays = [tclass_rates_LSB] for solver in paused: s = solver.s (c,) = numpy.digitize([solver.cpu_cost], self._budgets) if c > 0: # XXX hack c -= 1 paused_tclass_LB = numpy.zeros((L, B)) paused_tclass_LB[:, :B - c - 1] = tclass_rates_LSB[:, s, c + 1:] paused_tclass_LB /= 1.0 - numpy.sum(tclass_rates_LSB[:, s, :c + 1], axis = -1)[..., None] augmented_tclass_arrays.append(paused_tclass_LB[:, None, :]) augmented_tclass_rates_LAB = numpy.hstack(augmented_tclass_arrays) # make a plan... remaining = budget - borg.get_accountant().total normal_cpu_budget = borg.machine_to_normal(borg.unicore_cpu_budget(remaining)) (feasible_b,) = numpy.digitize([normal_cpu_budget], self._budgets) if feasible_b == 0: break raw_plan = \ plan_knapsack_multiverse( tclass_weights_L, augmented_tclass_rates_LAB[..., :feasible_b], ) # interpret the plan's first action (a, c) = raw_plan[0] planned_cpu_cost = self._budgets[c] if a >= S: solver = paused.pop(a - S) s = solver.s name = self._solver_names[s] else: s = a name = self._solver_names[s] solver = bundle.solvers[name](task, queue, uuid.uuid4()) solver.s = s solver.cpu_cost = 0.0 # don't waste our final seconds before the buzzer if normal_cpu_budget - planned_cpu_cost < self._budgets[0]: planned_cpu_cost = normal_cpu_budget else: planned_cpu_cost = planned_cpu_cost # be informative augmented_tclass_cmf_LAB = numpy.cumsum(augmented_tclass_rates_LAB, axis = -1) augmented_mean_cmf_AB = numpy.sum(tclass_weights_L[:, None, None] * augmented_tclass_cmf_LAB, axis = 0) subjective_rate = augmented_mean_cmf_AB[a, c] logger.info( #tclass_cmf_LSB = numpy.cumsum(tclass_rates_LSB, axis = -1) #mean_cmf_SB = numpy.sum(tclass_weights_L[:, None, None] * tclass_cmf_LSB, axis = 0) #messages.append("mean augmented subjective CMF:\n%s" % cargo.pretty_probability_matrix(mean_cmf_SB)) #messages.append( "running %s@%i for %i with %i remaining (b = %.2f)" % ( name, borg.normal_to_machine(solver.cpu_cost), borg.normal_to_machine(planned_cpu_cost), remaining.cpu_seconds, subjective_rate, ), ) # ... and follow through solver.unpause_for(borg.normal_to_machine(planned_cpu_cost)) running[solver._solver_id] = solver solver.cpu_budgeted = solver.cpu_cost + planned_cpu_cost if len(running) == cores: (solver_id, run_cpu_seconds, answer, terminated) = queue.get() borg.get_accountant().charge_cpu(run_cpu_seconds) solver = running.pop(solver_id) solver.cpu_cost += borg.machine_to_normal(run_cpu_seconds) if bundle.domain.is_final(task, answer): break elif terminated: failed.append(solver) else: paused.append(solver) for process in paused + running.values(): process.stop() # XXX #if not self._domain.is_final(task, answer) and not hopeless: #for message in messages: #logger.info("%s", message) return answer
def run_lp2sat(binaries_path, asp_file, cnf_file): """Convert a grounded ASP instance to CNF.""" # prepare the pipeline commands = [ ["lp2sat-bin/smodels-2.34", "-internal", "-nolookahead"], ["lp2sat-bin/lpcat-1.18"], # XXX is lpcat truly necessary? ["lp2sat-bin/lp2normal-1.11"], ["lp2sat-bin/igen-1.7"], ["lp2sat-bin/smodels-2.34", "-internal", "-nolookahead"], ["lp2sat-bin/lpcat-1.18", "-s=/dev/null"], # XXX is lpcat truly necessary? ["lp2sat-bin/lp2lp2-1.17", "-g"], ["lp2sat-bin/lp2sat-1.15", "-n"], ] full_commands = \ [[os.path.join(binaries_path, c[0])] + c[1:] for c in commands] \ + [["grep", "-v", "^c"]] # run the pipeline processes = [] previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime try: with open("/dev/null", "wb") as dev_null_file: # start the pipeline processes input_pipe = asp_file for (i, command) in enumerate(full_commands): if i == len(full_commands) - 1: output_pipe = cnf_file else: output_pipe = subprocess.PIPE process = \ subprocess.Popen( command, stdin = input_pipe, stderr = dev_null_file, stdout = output_pipe, ) processes.append(process) input_pipe.close() input_pipe = process.stdout # wait for them to terminate for (process, command) in zip(processes, full_commands): process.wait() if process.returncode != 0: message = \ "process {0} in lp2sat pipeline failed: {1}".format( process.pid, command, ) raise LP2SAT_FailedException(message) except: logger.warning("failed to convert ASP to CNF") raise finally: for process in processes: if process.returncode is None: process.kill() process.wait() # accumulate the cost of children cost = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime - previous_utime borg.get_accountant().charge_cpu(cost) logger.info("lp2sat pipeline cost was %.2f s", cost)
def get_claspfolio_features_for(asp_path, binaries_path): """Invoke claspre to compute features of an ASP instance.""" previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime # get feature names claspre_path = os.path.join(binaries_path, "claspfolio-0.8.0-x86-linux/clasp+pre-1.3.4") (names_out, _) = borg.util.check_call_capturing([claspre_path, "--list-features"]) (dynamic_names_out, static_names_out) = names_out.splitlines() dynamic_names = normalized_claspre_names(dynamic_names_out.split(",")) static_names = normalized_claspre_names(static_names_out.split(",")) # compute feature values values_command = [ claspre_path, "--rand-prob=10,30", "--search-limit=300,10", "--features=C1", "--file", asp_path, ] num_restarts = 10 logger.info("running %s", values_command) (values_out, _, _) = borg.util.call_capturing(values_command) values_per = [ map(parse_claspre_value, l.split(",")) for l in values_out.strip().splitlines() ] if len(values_per) < num_restarts + 1: # claspre failed, or the instance was solved in preprocessing if len(values_per) == 0: # (claspre died) values_per = [[0.0] * len(static_names)] missing = (num_restarts - len(values_per) + 1) values_per = values_per[:-1] + ([[0.0] * len(dynamic_names)] * missing) + values_per[-1:] else: assert len(values_per) == num_restarts + 1 # pull them together names = [] values = [] for i in xrange(num_restarts): names += ["restart{0}-{1}".format(i, n) for n in dynamic_names] values += values_per[i] names += static_names values += values_per[-1] # ... cost = resource.getrusage( resource.RUSAGE_CHILDREN).ru_utime - previous_utime borg.get_accountant().charge_cpu(cost) logger.info("collected features of %s in %.2fs", asp_path, cost) assert len(names) == len(values) return (names, values)