def test_trial_attachments(): exp_key = "A" with TempMongo() as tm: mj = tm.mongo_jobs("foo") trials = MongoTrials(tm.connection_string("foo"), exp_key=exp_key) space = hp.uniform("x", -10, 10) max_evals = 3 fmin_thread = threading.Thread(target=fmin_thread_fn, args=(space, trials, max_evals)) fmin_thread.start() mw = MongoWorker(mj=mj, logfilename=None, workdir="mongoexp_test_dir") n_jobs = max_evals while n_jobs: try: mw.run_one("hostname", 10.0, erase_created_workdir=True) print("worker: ran job") except Exception as exc: print(f"worker: encountered error : {str(exc)}") traceback.print_exc() n_jobs -= 1 fmin_thread.join() all_trials = MongoTrials(tm.connection_string("foo")) assert len(all_trials) == max_evals assert trials.count_by_state_synced(JOB_STATE_DONE) == max_evals assert trials.count_by_state_unsynced(JOB_STATE_DONE) == max_evals
def worker_thread_fn(host_id, n_jobs, timeout): mw = MongoWorker(mj=TempMongo.mongo_jobs('foodb'), logfilename=None, workdir="mongoexp_test_dir") while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print('worker: %s ran job' % str(host_id)) n_jobs -= 1
def worker_thread_fn(host_id, n_jobs, timeout): mw = MongoWorker( mj=TempMongo.mongo_jobs('foodb'), logfilename=None) while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print('worker: %s ran job' % str(host_id)) n_jobs -= 1
def worker(host_id, n_jobs, timeout): mw = MongoWorker( mj=TempMongo.mongo_jobs('foodb'), ) t0 = time.time() while n_jobs: mw.run_one(host_id, timeout) print 'worker: ran job' n_jobs -= 1
def hyperopt_worker( progress_queue: multiprocessing.Queue, logging_queue: multiprocessing.Queue, exp_key: str, workdir: str = ".", gpu: bool = True, hw_id: str = None, poll_interval: float = 1.0, reserve_timeout: float = 30.0, mongo_port_address: str = "localhost:1234/scvi_db", ): """Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised. :param progress_queue: Queue in which to put None when a job is done. :param logging_queue: Queue to send logs to using a ``QueueHandler``. :param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``. :param workdir: :param gpu: If ``True`` means a GPU is to be used. :param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``. :param poll_interval: Time to wait between attempts to reserve a job. :param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for before throwing a ``ReserveTimeout`` Exception. :param mongo_port_address: Addres to the running MongoDb service. """ # write all logs to queue root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) queue_handler = QueueHandler(logging_queue) queue_handler.setLevel(logging.DEBUG) root_logger.addHandler(queue_handler) logger.debug("Worker working...") os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str() # FIXME is this stil necessary? sys.path.append(".") mjobs = MongoJobs.new_from_connection_str( os.path.join(as_mongo_str(mongo_port_address), "jobs")) mworker = MongoWorker(mjobs, float(poll_interval), workdir=workdir, exp_key=exp_key) while True: # FIXME we don't protect ourselves from memory leaks, bad cleanup, etc. try: mworker.run_one(reserve_timeout=float(reserve_timeout)) progress_queue.put(None) except ReserveTimeout: logger.debug( "Caught ReserveTimeout. " "Exiting after failing to reserve job for {time} seconds.". format(time=reserve_timeout)) break
def _worker_thread_fn(host_id, n_jobs, timeout, dbname='foo'): mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname)) try: while n_jobs: mw.run_one(host_id, timeout) print 'worker: %s ran job' % str(host_id) n_jobs -= 1 except ReserveTimeout: print 'worker timed out:', host_id pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname="foo", logfilename=None): mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename) try: while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print "worker: %s ran job" % str(host_id) n_jobs -= 1 except ReserveTimeout: print "worker timed out:", host_id pass
def run(self): print "Starting a worker for exp_key = %s" % str(self.exp_key) mworker = MongoWorker(self.mj, float(self.options["poll_interval"]), workdir=self.options["workdir"], exp_key=self.exp_key) while True: res = mworker.run_one(reserve_timeout=float(self.options["reserve_timeout"]))
def run(self): print "Starting a worker for exp_key = %s" % str(self.exp_key) mworker = MongoWorker(self.mj, float(self.options["poll_interval"]), workdir=self.options["workdir"], exp_key=self.exp_key) while True: res = mworker.run_one( reserve_timeout=float(self.options["reserve_timeout"]))
def _worker_thread_fn(host_id, n_jobs, timeout, dbname='foo', logfilename=None): mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename) try: while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print 'worker: %s ran job' % str(host_id) n_jobs -= 1 except ReserveTimeout: print 'worker timed out:', host_id pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname='foo', logfilename=None): mw = MongoWorker( mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename, workdir="mongoexp_test_dir", ) try: while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print('worker: %s ran job' % str(host_id)) n_jobs -= 1 except ReserveTimeout: print('worker timed out:', host_id) pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname="foo", logfilename=None): mw = MongoWorker( mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename, workdir="mongoexp_test_dir", ) try: while n_jobs: mw.run_one(host_id, timeout, erase_created_workdir=True) print("worker: %s ran job" % str(host_id)) n_jobs -= 1 except ReserveTimeout: print("worker timed out:", host_id) pass
def worker_thread_fn(host_id, n_jobs, timeout): mw = MongoWorker(mj=TempMongo.mongo_jobs('foodb')) while n_jobs: mw.run_one(host_id, timeout) print 'worker: %s ran job' % str(host_id) n_jobs -= 1