def test_main_search_clear_existing(trials): # -- main_search (the CLI) has no way of specifying an # exp_key of None. assert trials._exp_key == 'hello' doc = hyperopt.tests.test_base.ok_trial(70, 0) doc['exp_key'] = 'hello' trials.insert_trial_doc(doc) print 'Setting up trials with exp_key', doc['exp_key'] options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key=doc['exp_key'], clear_existing=True, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foo'), ) args = ('hyperopt.bandits.n_arms', 'hyperopt.Random') def input(): return 'y' trials.refresh() assert len(trials) == 1 main_search_helper(options, args, input=input) trials.refresh() assert len(trials) == 0
def test_main_search_clear_existing(trials): # -- main_search (the CLI) has no way of specifying an # exp_key of None. assert trials._exp_key == "hello" doc = hyperopt.tests.test_base.ok_trial(70, 0) doc["exp_key"] = "hello" trials.insert_trial_doc(doc) print "Setting up trials with exp_key", doc["exp_key"] options = FakeOptions( bandit_argfile="", bandit_algo_argfile="", exp_key=doc["exp_key"], clear_existing=True, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str("localhost:22334/foo"), ) args = ("hyperopt.bandits.n_arms", "hyperopt.Random") def input(): return "y" trials.refresh() assert len(trials) == 1 main_search_helper(options, args, input=input) trials.refresh() assert len(trials) == 0
def _fmin_parallel( queue: multiprocessing.Queue, fn: Callable, exp_key: str, space: dict, algo: Callable = tpe.suggest, max_evals: int = 100, show_progressbar: bool = False, mongo_port_address: str = "localhost:1234/scvi_db", ): """Launches a ``hyperopt`` minimization procedure. """ logger.debug("Instantiating trials object.") # instantiate Trials object trials = MongoTrials(as_mongo_str(os.path.join(mongo_port_address, "jobs")), exp_key=exp_key) # run hyperoptimization logger.debug("Calling fmin.") _ = fmin( fn=fn, space=space, algo=algo, max_evals=max_evals, trials=trials, show_progressbar=show_progressbar, ) logger.debug("fmin returned.") # queue.put uses pickle so remove attribute containing thread.lock if hasattr(trials, "handle"): logger.debug("Deleting Trial handle for pickling.") del trials.handle logger.debug("Putting Trials in Queue.") queue.put(trials)
def _fmin_parallel( queue: multiprocessing.Queue, fn: Callable, exp_key: str, space: dict, algo: Callable = tpe.suggest, max_evals: int = 100, fmin_timer: float = None, show_progressbar: bool = False, mongo_port_address: str = "localhost:1234/scvi_db", ): """Launches a ``hyperopt`` minimization procedure. """ logger.debug("Instantiating trials object.") # instantiate Trials object trials = MongoTrials( as_mongo_str(os.path.join(mongo_port_address, "jobs")), exp_key=exp_key ) # run hyperoptimization in another fork to enable the use of fmin_timer fmin_kwargs = { "fn": fn, "space": space, "algo": algo, "max_evals": max_evals, "trials": trials, "show_progressbar": show_progressbar, } fmin_thread = threading.Thread(target=fmin, kwargs=fmin_kwargs) logger.debug("Calling fmin.") # set fmin thread as daemon so it stops when the main process terminates fmin_thread.daemon = True fmin_thread.start() started_threads.append(fmin_thread) if fmin_timer: logging.debug( "Timer set, fmin will run for at most {timer}".format(timer=fmin_timer) ) start_time = time.monotonic() run_time = 0 while run_time < fmin_timer and fmin_thread.is_alive(): time.sleep(10) run_time = time.monotonic() - start_time else: logging.debug("No timer, waiting for fmin") while True: if not fmin_thread.is_alive(): break else: time.sleep(10) logger.debug("fmin returned or timer ran out.") # queue.put uses pickle so remove attribute containing thread.lock if hasattr(trials, "handle"): logger.debug("Deleting Trial handle for pickling.") del trials.handle logger.debug("Putting Trials in Queue.") queue.put(trials)
def hyperopt_worker( progress_queue: multiprocessing.Queue, logging_queue: multiprocessing.Queue, exp_key: str, workdir: str = ".", gpu: bool = True, hw_id: str = None, poll_interval: float = 1.0, reserve_timeout: float = 30.0, mongo_port_address: str = "localhost:1234/scvi_db", ): """Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised. :param progress_queue: Queue in which to put None when a job is done. :param logging_queue: Queue to send logs to using a ``QueueHandler``. :param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``. :param workdir: :param gpu: If ``True`` means a GPU is to be used. :param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``. :param poll_interval: Time to wait between attempts to reserve a job. :param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for before throwing a ``ReserveTimeout`` Exception. :param mongo_port_address: Addres to the running MongoDb service. """ # write all logs to queue root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) queue_handler = QueueHandler(logging_queue) queue_handler.setLevel(logging.DEBUG) root_logger.addHandler(queue_handler) logger.debug("Worker working...") os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str() # FIXME is this stil necessary? sys.path.append(".") mjobs = MongoJobs.new_from_connection_str( os.path.join(as_mongo_str(mongo_port_address), "jobs")) mworker = MongoWorker(mjobs, float(poll_interval), workdir=workdir, exp_key=exp_key) while True: # FIXME we don't protect ourselves from memory leaks, bad cleanup, etc. try: mworker.run_one(reserve_timeout=float(reserve_timeout)) progress_queue.put(None) except ReserveTimeout: logger.debug( "Caught ReserveTimeout. " "Exiting after failing to reserve job for {time} seconds.". format(time=reserve_timeout)) break
def test_main_worker(trials): options = FakeOptions( max_jobs=1, # XXX: sync this with TempMongo mongo=as_mongo_str('localhost:22334/foodb'), reserve_timeout=1, poll_interval=.5, workdir=None, exp_key='foo') # -- check that it runs # and that the reserve timeout is respected main_worker_helper(options, ())
def test_main_worker(trials): options = FakeOptions( max_jobs=1, # XXX: sync this with TempMongo mongo=as_mongo_str('localhost:22334/foodb'), reserve_timeout=1, poll_interval=.5, workdir=None, exp_key='foo' ) # -- check that it runs # and that the reserve timeout is respected main_worker_helper(options, ())
def test_main_search_runs(trials): options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key=None, clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foodb'), ) args = ('hyperopt.bandits.n_arms', 'hyperopt.Random') main_search_helper(options, args)
def test_main_search_driver_attachment(trials): options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key='hello', clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foo'), ) args = ('hyperopt.bandits.n_arms', 'hyperopt.Random') main_search_helper(options, args, cmd_type='D.A.') print trials.handle.gfs._GridFS__collection assert 'driver_attachment_hello.pkl' in trials.attachments
def test_main_search_driver_reattachment(trials): # pretend we already attached a different bandit trials.attachments["driver_attachment_hello.pkl"] = cPickle.dumps((1, 2, 3)) options = FakeOptions( bandit_argfile="", bandit_algo_argfile="", exp_key="hello", clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str("localhost:22334/foo"), ) args = ("hyperopt.bandits.n_arms", "hyperopt.Random") main_search_helper(options, args, cmd_type="D.A.")
def test_main_search_driver_attachment(trials): options = FakeOptions( bandit_argfile="", bandit_algo_argfile="", exp_key="hello", clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str("localhost:22334/foo"), ) args = ("hyperopt.bandits.n_arms", "hyperopt.Random") main_search_helper(options, args, cmd_type="D.A.") print trials.handle.gfs._GridFS__collection assert "driver_attachment_hello.pkl" in trials.attachments
def test_main_search_driver_reattachment(trials): # pretend we already attached a different bandit trials.attachments['driver_attachment_hello.pkl'] = cPickle.dumps( (1, 2, 3)) options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key='hello', clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foo'), ) args = ('hyperopt.bandits.n_arms', 'hyperopt.Random') main_search_helper(options, args, cmd_type='D.A.')
def test_main_search_driver_reattachment(trials): # pretend we already attached a different bandit trials.attachments['driver_attachment_hello.pkl'] = cPickle.dumps( (1, 2, 3)) options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key='hello', clear_existing=False, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foo'), ) args = ('hyperopt.bandits.TwoArms', 'hyperopt.Random') main_search_helper(options, args, cmd_type='D.A.')
def __init__(self, num_features, bandit_func, bandit_algo_class, exp_prefix, trials=None, mongo_opts=None, walltime_cutoff=np.inf, ntrials=None, use_injected=True): # if trials is None, then mongo_opts is used to create a MongoTrials, # otherwise it is ignored. # self.num_features = num_features self.bandit_algo_class = bandit_algo_class self.bandit = bandit_func(num_features) self.init_bandit_algo() self.exp_prefix = exp_prefix self.walltime_cutoff = walltime_cutoff assert ntrials is not None self.ntrials = ntrials self.use_injected = use_injected if trials is None: trials = MongoTrials(as_mongo_str(mongo_opts) + '/jobs', exp_key=self.get_exp_key()) #trials = Trials() self.trials = trials self.exp_key = self.trials._exp_key
def test_main_search_clear_existing(trials): doc = hyperopt.tests.test_base.ok_trial(70, 0) doc['exp_key'] = 'hello' trials.insert_trial_doc(doc) options = FakeOptions( bandit_argfile='', bandit_algo_argfile='', exp_key=doc['exp_key'], clear_existing=True, steps=0, block=True, workdir=None, poll_interval=1, max_queue_len=1, mongo=as_mongo_str('localhost:22334/foo'), ) args = ('hyperopt.bandits.n_arms', 'hyperopt.Random') def input(): return 'y' trials.refresh() assert len(trials) == 1 main_search_helper(options, args, input=input) trials.refresh() assert len(trials) == 0
def connection_string(dbname): return as_mongo_str('localhost:22334/%s' % dbname) + '/jobs'
def connection_string(dbname): return as_mongo_str(f"localhost:22334/{dbname}/jobs")
def mongo_jobs(self, name): return MongoJobs.new_from_connection_str( as_mongo_str('localhost:22334/%s' % name) + '/jobs')
def connection_string(dbname): return as_mongo_str('localhost:22334/{}/jobs'.format(dbname))
def connection_string(dbname): return as_mongo_str("localhost:22334/%s" % dbname) + "/jobs"