def enable_output():
"""Set up competition-compliant output."""
# configure the default global level
cargo.get_logger(level = cargo.defaults.root_log_level)
# set up output
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(CompetitionFormatter())
handler.setLevel(logging.NOTSET)
logging.root.addHandler(handler)
def main(out_path, domain_name, budget, tasks_root, tests_root = None, live = False, runs = 16, workers = 0):
"""Collect validation results."""
cargo.enable_default_logging()
cargo.get_logger("borg.portfolios", level = "DETAIL")
def yield_runs():
# build solvers and train / test sets
if live:
domain = borg.get_domain(domain_name)
else:
domain = FakeDomain(borg.get_domain(domain_name))
paths = list(cargo.files_under(tasks_root, domain.extensions))
examples = int(round(len(paths) * 0.50))
logger.info("found %i tasks", len(paths))
if tests_root is not None:
tests_root_paths = list(cargo.files_under(tests_root, domain.extensions))
# build validation runs
for _ in xrange(runs):
split = uuid.uuid4()
shuffled = sorted(paths, key = lambda _ : numpy.random.rand())
train_paths = shuffled[:examples]
if tests_root is None:
test_paths = shuffled[examples:]
else:
test_paths = tests_root_paths
for name in borg.portfolios.named:
yield (run_validation, [name, domain, train_paths, test_paths, budget, split])
with open(out_path, "w") as out_file:
writer = csv.writer(out_file)
writer.writerow(["name", "budget", "cost", "rate", "split"])
cargo.distribute_or_labor(yield_runs(), workers, lambda _, r: writer.writerows(r))
def main(
model_path,
solvers_path,
input_path,
seed = 42,
budget = 3600.0,
cores = 1,
speed = borg.defaults.machine_speed,
quiet = False
):
"""Solve a problem instance."""
# XXX hackish
borg.defaults.machine_speed = speed
try:
# general setup
enable_output()
if not quiet:
cargo.get_logger("borg.solvers", level = "DETAIL")
numpy.random.seed(seed)
random.seed(numpy.random.randint(2**31))
# run the solver
bundle = borg.load_solvers(solvers_path)
logger.info("loaded portfolio model from %s", model_path)
with open(model_path) as file:
portfolio = pickle.load(file)
logger.info("solving %s", input_path)
with bundle.domain.task_from_path(input_path) as task:
remaining = budget - borg.get_accountant().total.cpu_seconds
answer = portfolio(task, bundle, borg.Cost(cpu_seconds = remaining), cores)
return bundle.domain.show_answer(task, answer)
except KeyboardInterrupt:
print "\nc terminating on SIGINT"
if __name__ == "__main__":
from borg.tools.run_validation import main
plac.call(main)
import os.path
import csv
import uuid
import itertools
import contextlib
import multiprocessing
import numpy
import cargo
import borg
logger = cargo.get_logger(__name__, default_level = "INFO")
class FakeSolver(object):
"""Provide a standard interface to a solver process."""
def __init__(self, name, task_path, stm_queue = None, solver_id = None):
# prepare communication channels
if stm_queue is None:
self._stm_queue = multiprocessing.Queue()
else:
self._stm_queue = stm_queue
if solver_id is None:
self._solver_id = uuid.uuid4()
else:
self._solver_id = solver_id
"""@author: Bryan Silverthorn"""
import cargo
logger = cargo.get_logger(__name__)
_encodings_by_name = {}
def named_encoding(class_):
_encodings_by_name[class_.name] = class_
logger.info("registered encoding %s", class_.name)
return class_
def encoding(name):
return _encodings_by_name[name]
class Grid(object):
"""Instance of the grid-coloring problem."""
def __init__(self, width, height):
self.width = width
self.height = height
self.colors = 4
def is_coloring(self, coloring):
"""Is the specified coloring a valid coloring?"""
for x0 in xrange(self.width):
for y0 in xrange(self.height):
"""@author: Bryan Silverthorn <*****@*****.**>"""
import plac
if __name__ == "__main__":
from cargo.tools.labor.work2 import main
plac.call(main)
import numpy
import random
import traceback
import zmq
import cargo
logger = cargo.get_logger(__name__, level = "NOTSET")
def work_once(condor_id, req_socket, task):
"""Request and/or complete a single unit of work."""
# get an assignment
if task is None:
cargo.send_pyobj_gz(
req_socket,
cargo.labor2.ApplyMessage(condor_id),
)
task = cargo.recv_pyobj_gz(req_socket)
if task is None:
logger.info("received null assignment; terminating")
"""@author: Bryan Silverthorn <*****@*****.**>"""
import plac
if __name__ == "__main__":
from cargo.tools.labor.work2 import main
plac.call(main)
import numpy
import random
import traceback
import zmq
import cargo
logger = cargo.get_logger(__name__, level="NOTSET")
def work_once(condor_id, req_socket, task):
"""Request and/or complete a single unit of work."""
# get an assignment
if task is None:
cargo.send_pyobj_gz(
req_socket,
cargo.labor2.ApplyMessage(condor_id),
)
task = cargo.recv_pyobj_gz(req_socket)
if task is None:
import os
import sys
import time
import zlib
import socket
import signal
import random
import traceback
import collections
import multiprocessing
import cPickle as pickle
import numpy
import cargo
logger = cargo.get_logger(__name__, level="INFO")
_current_task = None
def get_task():
"""Get the currently-executing task, if any."""
return _current_task
def send_pyobj_gz(zmq_socket, message):
pickled = pickle.dumps(message)
compressed = zlib.compress(pickled, 1)
zmq_socket.send(compressed)
import plac
if __name__ == "__main__":
from borg.tools.armada import main
plac.call(main)
import sys
import random
import logging
import cPickle as pickle
import numpy
import cargo
import borg
logger = cargo.get_logger(__name__, default_level = "INFO")
@plac.annotations(
in_path = ("path to instance"),
seed = ("PRNG seed", "option", None, int),
workers = ("units of execution", "option", None, int),
)
def main(in_path, seed = 42, budget = 2e6, workers = 1):
"""Solve a problem instance with an armada."""
# general setup
numpy.random.seed(seed)
random.seed(numpy.random.randint(2**31))
# parse the instance
with open(in_path) as in_file:
import os
import sys
import time
import zlib
import socket
import signal
import random
import traceback
import subprocess
import collections
import multiprocessing
import cPickle as pickle
import numpy
import cargo
logger = cargo.get_logger(__name__, level = "INFO")
def send_pyobj_gz(zmq_socket, message):
pickled = pickle.dumps(message)
compressed = zlib.compress(pickled, 1)
zmq_socket.send(compressed)
def recv_pyobj_gz(zmq_socket):
compressed = zmq_socket.recv()
decompressed = zlib.decompress(compressed)
unpickled = pickle.loads(decompressed)
return unpickled
class Message(object):
"""@author: Bryan Silverthorn <*****@*****.**>"""
import plac
if __name__ == "__main__":
from borg.tools.get_calibration import main
plac.call(main)
import numpy
import cargo
import borg
logger = cargo.get_logger(__name__, default_level="DETAIL")
def read_median_cost(path):
runs_data = numpy.recfromcsv(path, usemask=True)
return numpy.median(runs_data["cost"])
@plac.annotations()
def main(local_path, train_path):
"""Compute the machine speed calibration ratio."""
cargo.enable_default_logging()
local_median = read_median_cost(local_path)
train_median = read_median_cost(train_path)