def test_main_search_clear_existing(trials):
# -- main_search (the CLI) has no way of specifying an
# exp_key of None.
assert trials._exp_key == 'hello'
doc = hyperopt.tests.test_base.ok_trial(70, 0)
doc['exp_key'] = 'hello'
trials.insert_trial_doc(doc)
print 'Setting up trials with exp_key', doc['exp_key']
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key=doc['exp_key'],
clear_existing=True,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foo'),
)
args = ('hyperopt.bandits.n_arms', 'hyperopt.Random')
def input():
return 'y'
trials.refresh()
assert len(trials) == 1
main_search_helper(options, args, input=input)
trials.refresh()
assert len(trials) == 0
def test_main_search_clear_existing(trials):
# -- main_search (the CLI) has no way of specifying an
# exp_key of None.
assert trials._exp_key == "hello"
doc = hyperopt.tests.test_base.ok_trial(70, 0)
doc["exp_key"] = "hello"
trials.insert_trial_doc(doc)
print "Setting up trials with exp_key", doc["exp_key"]
options = FakeOptions(
bandit_argfile="",
bandit_algo_argfile="",
exp_key=doc["exp_key"],
clear_existing=True,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str("localhost:22334/foo"),
)
args = ("hyperopt.bandits.n_arms", "hyperopt.Random")
def input():
return "y"
trials.refresh()
assert len(trials) == 1
main_search_helper(options, args, input=input)
trials.refresh()
assert len(trials) == 0
def _fmin_parallel(
queue: multiprocessing.Queue,
fn: Callable,
exp_key: str,
space: dict,
algo: Callable = tpe.suggest,
max_evals: int = 100,
show_progressbar: bool = False,
mongo_port_address: str = "localhost:1234/scvi_db",
):
"""Launches a ``hyperopt`` minimization procedure.
"""
logger.debug("Instantiating trials object.")
# instantiate Trials object
trials = MongoTrials(as_mongo_str(os.path.join(mongo_port_address,
"jobs")),
exp_key=exp_key)
# run hyperoptimization
logger.debug("Calling fmin.")
_ = fmin(
fn=fn,
space=space,
algo=algo,
max_evals=max_evals,
trials=trials,
show_progressbar=show_progressbar,
)
logger.debug("fmin returned.")
# queue.put uses pickle so remove attribute containing thread.lock
if hasattr(trials, "handle"):
logger.debug("Deleting Trial handle for pickling.")
del trials.handle
logger.debug("Putting Trials in Queue.")
queue.put(trials)
def _fmin_parallel(
queue: multiprocessing.Queue,
fn: Callable,
exp_key: str,
space: dict,
algo: Callable = tpe.suggest,
max_evals: int = 100,
fmin_timer: float = None,
show_progressbar: bool = False,
mongo_port_address: str = "localhost:1234/scvi_db",
):
"""Launches a ``hyperopt`` minimization procedure.
"""
logger.debug("Instantiating trials object.")
# instantiate Trials object
trials = MongoTrials(
as_mongo_str(os.path.join(mongo_port_address, "jobs")), exp_key=exp_key
)
# run hyperoptimization in another fork to enable the use of fmin_timer
fmin_kwargs = {
"fn": fn,
"space": space,
"algo": algo,
"max_evals": max_evals,
"trials": trials,
"show_progressbar": show_progressbar,
}
fmin_thread = threading.Thread(target=fmin, kwargs=fmin_kwargs)
logger.debug("Calling fmin.")
# set fmin thread as daemon so it stops when the main process terminates
fmin_thread.daemon = True
fmin_thread.start()
started_threads.append(fmin_thread)
if fmin_timer:
logging.debug(
"Timer set, fmin will run for at most {timer}".format(timer=fmin_timer)
)
start_time = time.monotonic()
run_time = 0
while run_time < fmin_timer and fmin_thread.is_alive():
time.sleep(10)
run_time = time.monotonic() - start_time
else:
logging.debug("No timer, waiting for fmin")
while True:
if not fmin_thread.is_alive():
break
else:
time.sleep(10)
logger.debug("fmin returned or timer ran out.")
# queue.put uses pickle so remove attribute containing thread.lock
if hasattr(trials, "handle"):
logger.debug("Deleting Trial handle for pickling.")
del trials.handle
logger.debug("Putting Trials in Queue.")
queue.put(trials)
def hyperopt_worker(
progress_queue: multiprocessing.Queue,
logging_queue: multiprocessing.Queue,
exp_key: str,
workdir: str = ".",
gpu: bool = True,
hw_id: str = None,
poll_interval: float = 1.0,
reserve_timeout: float = 30.0,
mongo_port_address: str = "localhost:1234/scvi_db",
):
"""Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised.
:param progress_queue: Queue in which to put None when a job is done.
:param logging_queue: Queue to send logs to using a ``QueueHandler``.
:param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb
which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``.
:param workdir:
:param gpu: If ``True`` means a GPU is to be used.
:param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``.
:param poll_interval: Time to wait between attempts to reserve a job.
:param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for
before throwing a ``ReserveTimeout`` Exception.
:param mongo_port_address: Addres to the running MongoDb service.
"""
# write all logs to queue
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
queue_handler = QueueHandler(logging_queue)
queue_handler.setLevel(logging.DEBUG)
root_logger.addHandler(queue_handler)
logger.debug("Worker working...")
os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str()
# FIXME is this stil necessary?
sys.path.append(".")
mjobs = MongoJobs.new_from_connection_str(
os.path.join(as_mongo_str(mongo_port_address), "jobs"))
mworker = MongoWorker(mjobs,
float(poll_interval),
workdir=workdir,
exp_key=exp_key)
while True:
# FIXME we don't protect ourselves from memory leaks, bad cleanup, etc.
try:
mworker.run_one(reserve_timeout=float(reserve_timeout))
progress_queue.put(None)
except ReserveTimeout:
logger.debug(
"Caught ReserveTimeout. "
"Exiting after failing to reserve job for {time} seconds.".
format(time=reserve_timeout))
break
def test_main_worker(trials):
options = FakeOptions(
max_jobs=1,
# XXX: sync this with TempMongo
mongo=as_mongo_str('localhost:22334/foodb'),
reserve_timeout=1,
poll_interval=.5,
workdir=None,
exp_key='foo')
# -- check that it runs
# and that the reserve timeout is respected
main_worker_helper(options, ())
def test_main_worker(trials):
options = FakeOptions(
max_jobs=1,
# XXX: sync this with TempMongo
mongo=as_mongo_str('localhost:22334/foodb'),
reserve_timeout=1,
poll_interval=.5,
workdir=None,
exp_key='foo'
)
# -- check that it runs
# and that the reserve timeout is respected
main_worker_helper(options, ())
def test_main_search_runs(trials):
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key=None,
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foodb'),
)
args = ('hyperopt.bandits.n_arms', 'hyperopt.Random')
main_search_helper(options, args)
def test_main_search_driver_attachment(trials):
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key='hello',
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foo'),
)
args = ('hyperopt.bandits.n_arms', 'hyperopt.Random')
main_search_helper(options, args, cmd_type='D.A.')
print trials.handle.gfs._GridFS__collection
assert 'driver_attachment_hello.pkl' in trials.attachments
def test_main_search_driver_reattachment(trials):
# pretend we already attached a different bandit
trials.attachments["driver_attachment_hello.pkl"] = cPickle.dumps((1, 2, 3))
options = FakeOptions(
bandit_argfile="",
bandit_algo_argfile="",
exp_key="hello",
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str("localhost:22334/foo"),
)
args = ("hyperopt.bandits.n_arms", "hyperopt.Random")
main_search_helper(options, args, cmd_type="D.A.")
def test_main_search_driver_attachment(trials):
options = FakeOptions(
bandit_argfile="",
bandit_algo_argfile="",
exp_key="hello",
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str("localhost:22334/foo"),
)
args = ("hyperopt.bandits.n_arms", "hyperopt.Random")
main_search_helper(options, args, cmd_type="D.A.")
print trials.handle.gfs._GridFS__collection
assert "driver_attachment_hello.pkl" in trials.attachments
def test_main_search_driver_reattachment(trials):
# pretend we already attached a different bandit
trials.attachments['driver_attachment_hello.pkl'] = cPickle.dumps(
(1, 2, 3))
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key='hello',
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foo'),
)
args = ('hyperopt.bandits.n_arms', 'hyperopt.Random')
main_search_helper(options, args, cmd_type='D.A.')
def test_main_search_driver_reattachment(trials):
# pretend we already attached a different bandit
trials.attachments['driver_attachment_hello.pkl'] = cPickle.dumps(
(1, 2, 3))
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key='hello',
clear_existing=False,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foo'),
)
args = ('hyperopt.bandits.TwoArms', 'hyperopt.Random')
main_search_helper(options, args, cmd_type='D.A.')
def __init__(self, num_features, bandit_func, bandit_algo_class, exp_prefix,
trials=None, mongo_opts=None, walltime_cutoff=np.inf,
ntrials=None, use_injected=True):
# if trials is None, then mongo_opts is used to create a MongoTrials,
# otherwise it is ignored.
#
self.num_features = num_features
self.bandit_algo_class = bandit_algo_class
self.bandit = bandit_func(num_features)
self.init_bandit_algo()
self.exp_prefix = exp_prefix
self.walltime_cutoff = walltime_cutoff
assert ntrials is not None
self.ntrials = ntrials
self.use_injected = use_injected
if trials is None:
trials = MongoTrials(as_mongo_str(mongo_opts) + '/jobs',
exp_key=self.get_exp_key())
#trials = Trials()
self.trials = trials
self.exp_key = self.trials._exp_key
def test_main_search_clear_existing(trials):
doc = hyperopt.tests.test_base.ok_trial(70, 0)
doc['exp_key'] = 'hello'
trials.insert_trial_doc(doc)
options = FakeOptions(
bandit_argfile='',
bandit_algo_argfile='',
exp_key=doc['exp_key'],
clear_existing=True,
steps=0,
block=True,
workdir=None,
poll_interval=1,
max_queue_len=1,
mongo=as_mongo_str('localhost:22334/foo'),
)
args = ('hyperopt.bandits.n_arms', 'hyperopt.Random')
def input():
return 'y'
trials.refresh()
assert len(trials) == 1
main_search_helper(options, args, input=input)
trials.refresh()
assert len(trials) == 0
def connection_string(dbname):
return as_mongo_str('localhost:22334/%s' % dbname) + '/jobs'
def connection_string(dbname):
return as_mongo_str('localhost:22334/%s' % dbname) + '/jobs'
def connection_string(dbname):
return as_mongo_str(f"localhost:22334/{dbname}/jobs")
def mongo_jobs(self, name):
return MongoJobs.new_from_connection_str(
as_mongo_str('localhost:22334/%s' % name) + '/jobs')
def connection_string(dbname):
return as_mongo_str('localhost:22334/{}/jobs'.format(dbname))
def connection_string(dbname):
return as_mongo_str('localhost:22334/{}/jobs'.format(dbname))
def connection_string(dbname):
return as_mongo_str("localhost:22334/%s" % dbname) + "/jobs"
