def compute_features(self, task, cpu_seconds = None):
"""Read or compute features of an instance."""
# grab precomputed feature data
csv_path = task + ".features.csv"
assert os.path.exists(csv_path)
features_array = numpy.recfromcsv(csv_path)
features = features_array.tolist()
names = features_array.dtype.names
# accumulate their cost
assert names[0] == "cpu_cost"
cpu_cost = features[0]
borg.get_accountant().charge_cpu(cpu_cost)
# handle timeout logic, and we're done
if cpu_seconds is not None:
if cpu_cost >= cpu_seconds:
return (["cpu_cost"], [cpu_seconds])
else:
assert len(names) > 1
return (names, features)
def run_then_pause(self, budget):
"""Unpause the solver for the specified duration."""
assert not self._terminated
position = self._elapsed + budget
logger.detail(
"moving %s run to %.0f of %.0f (from %.0f)",
self._run.solver,
position,
self._run.cost,
self._elapsed,
)
if position >= self._run.cost:
borg.get_accountant().charge_cpu(self._run.cost - self._elapsed)
self._elapsed = self._run.cost
self._terminated = True
return self._run.success
else:
borg.get_accountant().charge_cpu(budget)
self._elapsed = position
return None
def __call__(self, task, budget, cores = 1):
queue = multiprocessing.Queue()
solvers = []
for _ in xrange(cores):
solver_id = uuid.uuid4()
solver = cargo.grab(self._domain.solvers.values())(task, queue, solver_id)
solvers.append(solver)
try:
for solver in solvers:
solver.unpause_for(borg.unicore_cpu_budget(budget))
remaining = len(solvers)
while remaining > 0:
(solver_id, run_cpu_cost, answer, _) = queue.get()
remaining -= 1
borg.get_accountant().charge_cpu(run_cpu_cost)
if self._domain.is_final(task, answer):
return answer
return None
finally:
for solver in solvers:
solver.stop()
def get_claspfolio_features_for(asp_path, binaries_path):
"""Invoke claspre to compute features of an ASP instance."""
previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
# get feature names
claspre_path = os.path.join(binaries_path, "claspfolio-0.8.0-x86-linux/clasp+pre-1.3.4")
(names_out, _) = borg.util.check_call_capturing([claspre_path, "--list-features"])
(dynamic_names_out, static_names_out) = names_out.splitlines()
dynamic_names = normalized_claspre_names(dynamic_names_out.split(","))
static_names = normalized_claspre_names(static_names_out.split(","))
# compute feature values
values_command = [claspre_path, "--rand-prob=10,30", "--search-limit=300,10", "--features=C1", "--file", asp_path]
num_restarts = 10
logger.info("running %s", values_command)
(values_out, _, _) = borg.util.call_capturing(values_command)
values_per = [map(parse_claspre_value, l.split(",")) for l in values_out.strip().splitlines()]
if len(values_per) < num_restarts + 1:
# claspre failed, or the instance was solved in preprocessing
if len(values_per) == 0:
# (claspre died)
values_per = [[0.0] * len(static_names)]
missing = num_restarts - len(values_per) + 1
values_per = values_per[:-1] + ([[0.0] * len(dynamic_names)] * missing) + values_per[-1:]
else:
assert len(values_per) == num_restarts + 1
# pull them together
names = []
values = []
for i in xrange(num_restarts):
names += ["restart{0}-{1}".format(i, n) for n in dynamic_names]
values += values_per[i]
names += static_names
values += values_per[-1]
# ...
cost = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime - previous_utime
borg.get_accountant().charge_cpu(cost)
logger.info("collected features of %s in %.2fs", asp_path, cost)
assert len(names) == len(values)
return (names, values)
def __call__(self, budget):
"""Unpause the solver, block for some limit, and terminate it."""
self.unpause_for(budget)
(solver_id, run_cpu_cost, answer, terminated) = self._stm_queue.get()
assert solver_id == self._solver_id
self.stop()
borg.get_accountant().charge_cpu(run_cpu_cost)
return answer
def compute_features(self, instance):
"""Return static features of an instance."""
# get features with cost
with_cost = self._suite.run_data.get_feature_vector(instance)
borg.get_accountant().charge_cpu(with_cost["cpu_cost"])
# return the features
features = dict(with_cost.iteritems())
del features["cpu_cost"]
return (features.keys(), features.values())
def __call__(self, budget):
"""Unpause the solver, block for some limit, and terminate it."""
self.unpause_for(budget)
response = self._stm_queue.get()
if isinstance(response, Exception):
raise response
else:
(solver_id, run_cpu_cost, answer, terminated) = response
assert solver_id == self._solver_id
self.stop()
borg.get_accountant().charge_cpu(run_cpu_cost)
return answer
def __call__(self, budget):
"""Unpause the solver, block for some limit, and terminate it."""
self.unpause_for(budget)
response = self._stm_queue.get()
if isinstance(response, Exception):
raise response
else:
(solver_id, run_cpu_cost, answer, terminated) = response
assert solver_id == self._solver_id
self.stop()
borg.get_accountant().charge_cpu(run_cpu_cost)
return answer
def main(
model_path,
solvers_path,
input_path,
seed = 42,
budget = 3600.0,
cores = 1,
speed = borg.defaults.machine_speed,
quiet = False
):
"""Solve a problem instance."""
# XXX hackish
borg.defaults.machine_speed = speed
try:
# general setup
enable_output()
if not quiet:
cargo.get_logger("borg.solvers", level = "DETAIL")
numpy.random.seed(seed)
random.seed(numpy.random.randint(2**31))
# run the solver
bundle = borg.load_solvers(solvers_path)
logger.info("loaded portfolio model from %s", model_path)
with open(model_path) as file:
portfolio = pickle.load(file)
logger.info("solving %s", input_path)
with bundle.domain.task_from_path(input_path) as task:
remaining = budget - borg.get_accountant().total.cpu_seconds
answer = portfolio(task, bundle, borg.Cost(cpu_seconds = remaining), cores)
return bundle.domain.show_answer(task, answer)
except KeyboardInterrupt:
print "\nc terminating on SIGINT"
def main(model_path,
solvers_path,
input_path,
seed=42,
budget=3600.0,
cores=1,
speed=borg.defaults.machine_speed,
quiet=False):
"""Solve a problem instance."""
# XXX hackish
borg.defaults.machine_speed = speed
try:
# general setup
enable_output()
if not quiet:
borg.get_logger("borg.solvers", level="DETAIL")
borg.statistics.set_prng_seeds(seed)
# run the solver
bundle = borg.load_solvers(solvers_path)
logger.info("loaded portfolio model from %s", model_path)
with open(model_path) as file:
portfolio = pickle.load(file)
logger.info("solving %s", input_path)
with bundle.domain.task_from_path(input_path) as task:
remaining = budget - borg.get_accountant().total.cpu_seconds
answer = portfolio(task, bundle, borg.Cost(cpu_seconds=remaining),
cores)
return bundle.domain.show_answer(task, answer)
except KeyboardInterrupt:
print "\nc terminating on SIGINT"
def _solve(self, task, bundle, budget, cores):
# print oracle knowledge, if any
#(runs,) = borg.portfolios.get_task_run_data([task]).values()
#(oracle_history, oracle_counts, _) = \
#borg.portfolios.action_rates_from_runs(
#self._solver_name_index,
#self._budget_index,
#runs.tolist(),
#)
#true_rates = oracle_history / oracle_counts
#hopeless = numpy.sum(true_rates) < 1e-2
#messages = []
#messages.append("true rates:\n%s" % cargo.pretty_probability_matrix(true_rates))
# obtain features
(_, features) = self._domain.compute_features(task)
# select a solver
queue = multiprocessing.Queue()
running = {}
paused = []
failed = []
answer = None
while True:
# obtain model predictions
failed_indices = []
for solver in failed + paused + running.values():
(total_b,) = numpy.digitize([solver.cpu_budgeted], self._budgets)
if total_b == 0: # XXX hack
total_b += 1
failed_indices.append((solver.s, total_b - 1))
(tclass_weights_L, tclass_rates_LSB) = self._model.predict(failed_indices, features)
(L, S, B) = tclass_rates_LSB.shape
# XXX force determinism
#for (s, b) in failed_indices:
#tclass_rates_LSB[:, s, :b + 1] = 1e-6
# prepare augmented PMF matrix
augmented_tclass_arrays = [tclass_rates_LSB]
for solver in paused:
s = solver.s
(c,) = numpy.digitize([solver.cpu_cost], self._budgets)
if c > 0: # XXX hack
c -= 1
paused_tclass_LB = numpy.zeros((L, B))
paused_tclass_LB[:, :B - c - 1] = tclass_rates_LSB[:, s, c + 1:]
paused_tclass_LB /= 1.0 - numpy.sum(tclass_rates_LSB[:, s, :c + 1], axis = -1)[..., None]
augmented_tclass_arrays.append(paused_tclass_LB[:, None, :])
augmented_tclass_rates_LAB = numpy.hstack(augmented_tclass_arrays)
# make a plan...
remaining = budget - borg.get_accountant().total
normal_cpu_budget = borg.machine_to_normal(borg.unicore_cpu_budget(remaining))
(feasible_b,) = numpy.digitize([normal_cpu_budget], self._budgets)
if feasible_b == 0:
break
raw_plan = \
plan_knapsack_multiverse(
tclass_weights_L,
augmented_tclass_rates_LAB[..., :feasible_b],
)
# interpret the plan's first action
(a, c) = raw_plan[0]
planned_cpu_cost = self._budgets[c]
if a >= S:
solver = paused.pop(a - S)
s = solver.s
name = self._solver_names[s]
else:
s = a
name = self._solver_names[s]
solver = bundle.solvers[name](task, queue, uuid.uuid4())
solver.s = s
solver.cpu_cost = 0.0
# don't waste our final seconds before the buzzer
if normal_cpu_budget - planned_cpu_cost < self._budgets[0]:
planned_cpu_cost = normal_cpu_budget
else:
planned_cpu_cost = planned_cpu_cost
# be informative
augmented_tclass_cmf_LAB = numpy.cumsum(augmented_tclass_rates_LAB, axis = -1)
augmented_mean_cmf_AB = numpy.sum(tclass_weights_L[:, None, None] * augmented_tclass_cmf_LAB, axis = 0)
subjective_rate = augmented_mean_cmf_AB[a, c]
logger.info(
#tclass_cmf_LSB = numpy.cumsum(tclass_rates_LSB, axis = -1)
#mean_cmf_SB = numpy.sum(tclass_weights_L[:, None, None] * tclass_cmf_LSB, axis = 0)
#messages.append("mean augmented subjective CMF:\n%s" % cargo.pretty_probability_matrix(mean_cmf_SB))
#messages.append(
"running %s@%i for %i with %i remaining (b = %.2f)" % (
name,
borg.normal_to_machine(solver.cpu_cost),
borg.normal_to_machine(planned_cpu_cost),
remaining.cpu_seconds,
subjective_rate,
),
)
# ... and follow through
solver.unpause_for(borg.normal_to_machine(planned_cpu_cost))
running[solver._solver_id] = solver
solver.cpu_budgeted = solver.cpu_cost + planned_cpu_cost
if len(running) == cores:
(solver_id, run_cpu_seconds, answer, terminated) = queue.get()
borg.get_accountant().charge_cpu(run_cpu_seconds)
solver = running.pop(solver_id)
solver.cpu_cost += borg.machine_to_normal(run_cpu_seconds)
if bundle.domain.is_final(task, answer):
break
elif terminated:
failed.append(solver)
else:
paused.append(solver)
for process in paused + running.values():
process.stop()
# XXX
#if not self._domain.is_final(task, answer) and not hopeless:
#for message in messages:
#logger.info("%s", message)
return answer
def run_lp2sat(binaries_path, asp_file, cnf_file):
"""Convert a grounded ASP instance to CNF."""
# prepare the pipeline
commands = [
["lp2sat-bin/smodels-2.34", "-internal", "-nolookahead"],
["lp2sat-bin/lpcat-1.18"], # XXX is lpcat truly necessary?
["lp2sat-bin/lp2normal-1.11"],
["lp2sat-bin/igen-1.7"],
["lp2sat-bin/smodels-2.34", "-internal", "-nolookahead"],
["lp2sat-bin/lpcat-1.18", "-s=/dev/null"], # XXX is lpcat truly necessary?
["lp2sat-bin/lp2lp2-1.17", "-g"],
["lp2sat-bin/lp2sat-1.15", "-n"],
]
full_commands = \
[[os.path.join(binaries_path, c[0])] + c[1:] for c in commands] \
+ [["grep", "-v", "^c"]]
# run the pipeline
processes = []
previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
try:
with open("/dev/null", "wb") as dev_null_file:
# start the pipeline processes
input_pipe = asp_file
for (i, command) in enumerate(full_commands):
if i == len(full_commands) - 1:
output_pipe = cnf_file
else:
output_pipe = subprocess.PIPE
process = \
subprocess.Popen(
command,
stdin = input_pipe,
stderr = dev_null_file,
stdout = output_pipe,
)
processes.append(process)
input_pipe.close()
input_pipe = process.stdout
# wait for them to terminate
for (process, command) in zip(processes, full_commands):
process.wait()
if process.returncode != 0:
message = \
"process {0} in lp2sat pipeline failed: {1}".format(
process.pid,
command,
)
raise LP2SAT_FailedException(message)
except:
logger.warning("failed to convert ASP to CNF")
raise
finally:
for process in processes:
if process.returncode is None:
process.kill()
process.wait()
# accumulate the cost of children
cost = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime - previous_utime
borg.get_accountant().charge_cpu(cost)
logger.info("lp2sat pipeline cost was %.2f s", cost)
def get_claspfolio_features_for(asp_path, binaries_path):
"""Invoke claspre to compute features of an ASP instance."""
previous_utime = resource.getrusage(resource.RUSAGE_CHILDREN).ru_utime
# get feature names
claspre_path = os.path.join(binaries_path,
"claspfolio-0.8.0-x86-linux/clasp+pre-1.3.4")
(names_out,
_) = borg.util.check_call_capturing([claspre_path, "--list-features"])
(dynamic_names_out, static_names_out) = names_out.splitlines()
dynamic_names = normalized_claspre_names(dynamic_names_out.split(","))
static_names = normalized_claspre_names(static_names_out.split(","))
# compute feature values
values_command = [
claspre_path,
"--rand-prob=10,30",
"--search-limit=300,10",
"--features=C1",
"--file",
asp_path,
]
num_restarts = 10
logger.info("running %s", values_command)
(values_out, _, _) = borg.util.call_capturing(values_command)
values_per = [
map(parse_claspre_value, l.split(","))
for l in values_out.strip().splitlines()
]
if len(values_per) < num_restarts + 1:
# claspre failed, or the instance was solved in preprocessing
if len(values_per) == 0:
# (claspre died)
values_per = [[0.0] * len(static_names)]
missing = (num_restarts - len(values_per) + 1)
values_per = values_per[:-1] + ([[0.0] * len(dynamic_names)] *
missing) + values_per[-1:]
else:
assert len(values_per) == num_restarts + 1
# pull them together
names = []
values = []
for i in xrange(num_restarts):
names += ["restart{0}-{1}".format(i, n) for n in dynamic_names]
values += values_per[i]
names += static_names
values += values_per[-1]
# ...
cost = resource.getrusage(
resource.RUSAGE_CHILDREN).ru_utime - previous_utime
borg.get_accountant().charge_cpu(cost)
logger.info("collected features of %s in %.2fs", asp_path, cost)
assert len(names) == len(values)
return (names, values)
