def main(
solvers_path,
tasks_root,
budget,
discard = False,
runs = 4,
only_solver = None,
suffix = ".runs.csv",
workers = 0,
):
"""Collect solver running-time data."""
cargo.enable_default_logging()
def yield_runs():
bundle = borg.load_solvers(solvers_path)
paths = list(cargo.files_under(tasks_root, bundle.domain.extensions))
if not paths:
raise ValueError("no paths found under specified root")
if only_solver is None:
solver_names = bundle.solvers.keys()
else:
if only_solver not in bundle.solvers:
raise ArgumentError("no such solver")
solver_names = [only_solver]
for _ in xrange(runs):
for solver_name in solver_names:
for path in paths:
yield (run_solver_on, [solvers_path, solver_name, path, budget])
def collect_run(task, row):
if not discard:
# unpack run outcome
(solver_name, budget, cost, succeeded, answer) = row
if answer is None:
answer_text = None
else:
answer_text = base64.b64encode(zlib.compress(pickle.dumps(answer)))
# write it to disk
(_, _, cnf_path, _) = task.args
csv_path = cnf_path + suffix
existed = os.path.exists(csv_path)
with open(csv_path, "a") as csv_file:
writer = csv.writer(csv_file)
if not existed:
writer.writerow(["solver", "budget", "cost", "succeeded", "answer"])
writer.writerow([solver_name, budget, cost, succeeded, answer_text])
cargo.do_or_distribute(yield_runs(), workers, collect_run)
def main(local_path, train_path):
"""Compute the machine speed calibration ratio."""
cargo.enable_default_logging()
local_median = read_median_cost(local_path)
train_median = read_median_cost(train_path)
logger.info("local median run time is %.2f CPU seconds", local_median)
logger.info("model median run time is %.2f CPU seconds", train_median)
logger.info("local speed ratio is thus %f", local_median / train_median)
def main(path,
executable,
no_tmux=False,
timeout=250,
window=None,
*arguments):
"""
Run something in response to changes in a directory.
"""
# enable logging
cargo.enable_default_logging()
# prepare the notification framework
command = [executable] + list(arguments)
manager = pyinotify.WatchManager()
handler = TriggerHandler()
notifier = pyinotify.Notifier(manager, handler)
manager.add_watch(
path,
pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY,
rec=True,
)
# watch for and respond to events
try:
while True:
triggered = notifier.check_events()
if triggered:
# coalesce events
notifier.read_events()
notifier.process_events()
if timeout is not None:
while notifier.check_events(timeout=timeout):
notifier.read_events()
notifier.process_events()
# run the command
execute_command(command, not no_tmux, window)
finally:
if not no_tmux:
subprocess.call([
"tmux", "-q", "setw", "-t",
str(window), "window-status-bg", "default"
])
def main(out_path, setup_path):
"""Prepare to visualize run data."""
cargo.enable_default_logging()
# build the configuration
with open(setup_path) as setup_file:
setup = json.load(setup_file)
view = ViewData(os.path.dirname(setup_path), setup)
# write it to disk
logger.info("writing visualization data to %s", out_path)
with open(out_path, "w") as out_file:
pickle.dump(view, out_file)
def main(root_path, encoding_name, min_width, max_width, min_height, max_height):
"""Encode a grid-coloring problem in CNF."""
cargo.enable_default_logging()
for width in xrange(min_width, max_width + 1):
for height in xrange(min_height, max_height + 1):
# encode the instance
grid = gridc.Grid(width, height)
encoding = gridc.encoding(encoding_name)
instance = encoding(grid).encode()
# write it to disk
out_name = "{0}x{1}.{2}.cnf".format(width, height, encoding_name)
out_path = os.path.join(root_path, out_name)
with open(out_path, "w") as out_file:
instance.write(out_file)
def main(out_path, fit_path):
"""Visualize model parameters."""
cargo.enable_default_logging()
# copy over static content
static_path = os.path.join(borg_explorer.__path__[0], "static")
distutils.dir_util.copy_tree(static_path, out_path)
# load the model(s)
logger.info("loading visualization data from %s", fit_path)
with open(fit_path) as fit_file:
fit = pickle.load(fit_file)
# write data directories
logger.info("writing inputs archive")
with open(os.path.join(out_path, "inputs.tar.gz"), "w") as archive_file:
archive_file.write(fit.data_archive)
for (name, category) in fit.categories.items():
write_category(out_path, name, category)
# generate the visualization
loader = jinja2.PackageLoader("borg_explorer", "templates")
environment = jinja2.Environment(loader = loader)
def write_rendered(template_name, output_name, **kwargs):
template = environment.get_template(template_name)
with open(os.path.join(out_path, output_name), "w") as output_file:
output_file.write(template.render(**kwargs).encode("utf-8"))
write_rendered("index.html", "index.html", base_url = fit.setup["base_url"])
write_rendered("borgview.js", "borgview.js", base_url = fit.setup["base_url"])
write_rendered("borgview.css", "borgview.css")
write_rendered("analytics.js", "analytics.js")
with open(os.path.join(out_path, "categories.json"), "w") as output_file:
category_list = [{"name": k, "path": sanitize(k)} for k in fit.categories.keys()]
json.dump(category_list, output_file)
def main(domain_name, tasks_root, workers=0):
"""Collect task features."""
cargo.enable_default_logging()
def yield_runs():
domain = borg.get_domain(domain_name)
paths = list(cargo.files_under(tasks_root, domain.extensions))
for path in paths:
yield (features_for_path, [domain, path])
def collect_run(task, (names, values)):
(_, cnf_path) = task.args
csv_path = cnf_path + ".features.csv"
with open(csv_path, "w") as csv_file:
csv.writer(csv_file).writerow(names)
csv.writer(csv_file).writerow(values)
def main(out_path, portfolio_name, solvers_path, *tasks_roots):
"""Train a solver."""
cargo.enable_default_logging()
# load the solvers bundle
bundle = borg.load_solvers(solvers_path)
# train the portfolio
training = borg.storage.TrainingData(tasks_roots, bundle.domain)
portfolio = borg.portfolios.named[portfolio_name](bundle, training, 50.0, 42) # XXX
logger.info("portfolio training complete")
# write it to disk
with open(out_path, "w") as out_file:
pickle.dump(portfolio, out_file, protocol = -1)
logger.info("portfolio written to %s", out_path)
def main(out_path, domain_name, budget, tasks_root, tests_root = None, live = False, runs = 16, workers = 0):
"""Collect validation results."""
cargo.enable_default_logging()
cargo.get_logger("borg.portfolios", level = "DETAIL")
def yield_runs():
# build solvers and train / test sets
if live:
domain = borg.get_domain(domain_name)
else:
domain = FakeDomain(borg.get_domain(domain_name))
paths = list(cargo.files_under(tasks_root, domain.extensions))
examples = int(round(len(paths) * 0.50))
logger.info("found %i tasks", len(paths))
if tests_root is not None:
tests_root_paths = list(cargo.files_under(tests_root, domain.extensions))
# build validation runs
for _ in xrange(runs):
split = uuid.uuid4()
shuffled = sorted(paths, key = lambda _ : numpy.random.rand())
train_paths = shuffled[:examples]
if tests_root is None:
test_paths = shuffled[examples:]
else:
test_paths = tests_root_paths
for name in borg.portfolios.named:
yield (run_validation, [name, domain, train_paths, test_paths, budget, split])
with open(out_path, "w") as out_file:
writer = csv.writer(out_file)
writer.writerow(["name", "budget", "cost", "rate", "split"])
cargo.distribute_or_labor(yield_runs(), workers, lambda _, r: writer.writerows(r))
def main(path, executable, no_tmux = False, timeout = 250, window = None, *arguments):
"""
Run something in response to changes in a directory.
"""
# enable logging
cargo.enable_default_logging()
# prepare the notification framework
command = [executable] + list(arguments)
manager = pyinotify.WatchManager()
handler = TriggerHandler()
notifier = pyinotify.Notifier(manager, handler)
manager.add_watch(
path,
pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY,
rec = True,
)
# watch for and respond to events
try:
while True:
triggered = notifier.check_events()
if triggered:
# coalesce events
notifier.read_events()
notifier.process_events()
if timeout is not None:
while notifier.check_events(timeout = timeout):
notifier.read_events()
notifier.process_events()
# run the command
execute_command(command, not no_tmux, window)
finally:
if not no_tmux:
subprocess.call(["tmux", "-q", "setw", "-t", str(window), "window-status-bg", "default"])
def main(req_address, condor_id):
"""Do arbitrary distributed work."""
cargo.enable_default_logging()
# connect to the work server
logger.info("connecting to %s", req_address)
context = zmq.Context()
req_socket = context.socket(zmq.REQ)
req_socket.connect(req_address)
# enter the work loop
try:
work_loop(condor_id, req_socket)
finally:
logger.info("flushing sockets and terminating zeromq context")
req_socket.close()
context.term()
logger.info("zeromq cleanup complete")
def main(req_address, condor_id):
"""Do arbitrary distributed work."""
cargo.enable_default_logging()
# connect to the work server
logger.info("connecting to %s", req_address)
context = zmq.Context()
req_socket = context.socket(zmq.REQ)
req_socket.connect(req_address)
# enter the work loop
try:
work_loop(condor_id, req_socket)
finally:
logger.info("flushing sockets and terminating zeromq context")
req_socket.close()
context.term()
logger.info("zeromq cleanup complete")
