def __call__(self, task, budget, cores = 1): queue = multiprocessing.Queue() solvers = [] for _ in xrange(cores): solver_id = uuid.uuid4() solver = cargo.grab(self._domain.solvers.values())(task, queue, solver_id) solvers.append(solver) try: for solver in solvers: solver.unpause_for(borg.unicore_cpu_budget(budget)) remaining = len(solvers) while remaining > 0: (solver_id, run_cpu_cost, answer, _) = queue.get() remaining -= 1 borg.get_accountant().charge_cpu(run_cpu_cost) if self._domain.is_final(task, answer): return answer return None finally: for solver in solvers: solver.stop()
def __call__(self, task, budget, cores = 1): assert cores == 1 # grab known run data (runs,) = get_task_run_data([task]).values() (_, _, rates) = \ action_rates_from_runs( self._solver_index, self._budget_index, runs.tolist(), ) # make a plan, and follow through plan = \ knapsack_plan( rates, self._solver_rindex, self._budgets, borg.unicore_cpu_budget(budget), ) for (name, cost) in plan: answer = self._domain.solvers[name](task)(cost) if self._domain.is_final(task, answer): return answer return None
def __call__(self, task, budget, cores = 1): assert cores == 1 # obtain features with borg.accounting() as features_accountant: (_, features) = self._domain.compute_features(task) # select a solver scores = [(s, m.predict_proba([features])[0, -1]) for (s, m) in self._models.items()] (name, probability) = max(scores, key = lambda (_, p): p) # run the solver run_cpu_budget = borg.unicore_cpu_budget(budget - features_accountant.total) return self._domain.solvers[name](task)(run_cpu_budget)
def _solve(self, task, bundle, budget, cores): # print oracle knowledge, if any #(runs,) = borg.portfolios.get_task_run_data([task]).values() #(oracle_history, oracle_counts, _) = \ #borg.portfolios.action_rates_from_runs( #self._solver_name_index, #self._budget_index, #runs.tolist(), #) #true_rates = oracle_history / oracle_counts #hopeless = numpy.sum(true_rates) < 1e-2 #messages = [] #messages.append("true rates:\n%s" % cargo.pretty_probability_matrix(true_rates)) # obtain features (_, features) = self._domain.compute_features(task) # select a solver queue = multiprocessing.Queue() running = {} paused = [] failed = [] answer = None while True: # obtain model predictions failed_indices = [] for solver in failed + paused + running.values(): (total_b,) = numpy.digitize([solver.cpu_budgeted], self._budgets) if total_b == 0: # XXX hack total_b += 1 failed_indices.append((solver.s, total_b - 1)) (tclass_weights_L, tclass_rates_LSB) = self._model.predict(failed_indices, features) (L, S, B) = tclass_rates_LSB.shape # XXX force determinism #for (s, b) in failed_indices: #tclass_rates_LSB[:, s, :b + 1] = 1e-6 # prepare augmented PMF matrix augmented_tclass_arrays = [tclass_rates_LSB] for solver in paused: s = solver.s (c,) = numpy.digitize([solver.cpu_cost], self._budgets) if c > 0: # XXX hack c -= 1 paused_tclass_LB = numpy.zeros((L, B)) paused_tclass_LB[:, :B - c - 1] = tclass_rates_LSB[:, s, c + 1:] paused_tclass_LB /= 1.0 - numpy.sum(tclass_rates_LSB[:, s, :c + 1], axis = -1)[..., None] augmented_tclass_arrays.append(paused_tclass_LB[:, None, :]) augmented_tclass_rates_LAB = numpy.hstack(augmented_tclass_arrays) # make a plan... remaining = budget - borg.get_accountant().total normal_cpu_budget = borg.machine_to_normal(borg.unicore_cpu_budget(remaining)) (feasible_b,) = numpy.digitize([normal_cpu_budget], self._budgets) if feasible_b == 0: break raw_plan = \ plan_knapsack_multiverse( tclass_weights_L, augmented_tclass_rates_LAB[..., :feasible_b], ) # interpret the plan's first action (a, c) = raw_plan[0] planned_cpu_cost = self._budgets[c] if a >= S: solver = paused.pop(a - S) s = solver.s name = self._solver_names[s] else: s = a name = self._solver_names[s] solver = bundle.solvers[name](task, queue, uuid.uuid4()) solver.s = s solver.cpu_cost = 0.0 # don't waste our final seconds before the buzzer if normal_cpu_budget - planned_cpu_cost < self._budgets[0]: planned_cpu_cost = normal_cpu_budget else: planned_cpu_cost = planned_cpu_cost # be informative augmented_tclass_cmf_LAB = numpy.cumsum(augmented_tclass_rates_LAB, axis = -1) augmented_mean_cmf_AB = numpy.sum(tclass_weights_L[:, None, None] * augmented_tclass_cmf_LAB, axis = 0) subjective_rate = augmented_mean_cmf_AB[a, c] logger.info( #tclass_cmf_LSB = numpy.cumsum(tclass_rates_LSB, axis = -1) #mean_cmf_SB = numpy.sum(tclass_weights_L[:, None, None] * tclass_cmf_LSB, axis = 0) #messages.append("mean augmented subjective CMF:\n%s" % cargo.pretty_probability_matrix(mean_cmf_SB)) #messages.append( "running %s@%i for %i with %i remaining (b = %.2f)" % ( name, borg.normal_to_machine(solver.cpu_cost), borg.normal_to_machine(planned_cpu_cost), remaining.cpu_seconds, subjective_rate, ), ) # ... and follow through solver.unpause_for(borg.normal_to_machine(planned_cpu_cost)) running[solver._solver_id] = solver solver.cpu_budgeted = solver.cpu_cost + planned_cpu_cost if len(running) == cores: (solver_id, run_cpu_seconds, answer, terminated) = queue.get() borg.get_accountant().charge_cpu(run_cpu_seconds) solver = running.pop(solver_id) solver.cpu_cost += borg.machine_to_normal(run_cpu_seconds) if bundle.domain.is_final(task, answer): break elif terminated: failed.append(solver) else: paused.append(solver) for process in paused + running.values(): process.stop() # XXX #if not self._domain.is_final(task, answer) and not hopeless: #for message in messages: #logger.info("%s", message) return answer
def __call__(self, task, budget, cores = 1): assert cores == 1 return self._best(task)(borg.unicore_cpu_budget(budget))