def __init__(self, mongo_uri, options): self.options = options self.mj = MongoJobs.new_from_connection_str(mongo_uri + "netron/hyperopt_jobs") last_job = self.mj.db["hyperopt_jobs"].find({}, {"exp_key": 1}).sort("exp_key", pymongo.DESCENDING).limit(1) last_job = list(last_job) if len(last_job) > 0: self.exp_key = last_job[0]["exp_key"] else: self.exp_key = "0"
def hyperopt_worker( progress_queue: multiprocessing.Queue, logging_queue: multiprocessing.Queue, exp_key: str, workdir: str = ".", gpu: bool = True, hw_id: str = None, poll_interval: float = 1.0, reserve_timeout: float = 30.0, mongo_port_address: str = "localhost:1234/scvi_db", ): """Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised. :param progress_queue: Queue in which to put None when a job is done. :param logging_queue: Queue to send logs to using a ``QueueHandler``. :param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``. :param workdir: :param gpu: If ``True`` means a GPU is to be used. :param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``. :param poll_interval: Time to wait between attempts to reserve a job. :param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for before throwing a ``ReserveTimeout`` Exception. :param mongo_port_address: Addres to the running MongoDb service. """ # write all logs to queue root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) queue_handler = QueueHandler(logging_queue) queue_handler.setLevel(logging.DEBUG) root_logger.addHandler(queue_handler) logger.debug("Worker working...") os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str() # FIXME is this stil necessary? sys.path.append(".") mjobs = MongoJobs.new_from_connection_str( os.path.join(as_mongo_str(mongo_port_address), "jobs")) mworker = MongoWorker(mjobs, float(poll_interval), workdir=workdir, exp_key=exp_key) while True: # FIXME we don't protect ourselves from memory leaks, bad cleanup, etc. try: mworker.run_one(reserve_timeout=float(reserve_timeout)) progress_queue.put(None) except ReserveTimeout: logger.debug( "Caught ReserveTimeout. " "Exiting after failing to reserve job for {time} seconds.". format(time=reserve_timeout)) break
def __init__(self, mongo_uri, options): self.options = options self.mj = MongoJobs.new_from_connection_str(mongo_uri + "netron/hyperopt_jobs") last_job = self.mj.db["hyperopt_jobs"].find({}, { "exp_key": 1 }).sort("exp_key", pymongo.DESCENDING).limit(1) last_job = list(last_job) if len(last_job) > 0: self.exp_key = last_job[0]["exp_key"] else: self.exp_key = "0"
def main_plot_histories(cls): import plotting conn_str_template = sys.argv[2] algos = sys.argv[3].split(',') dataset_name = sys.argv[4] start = int(sys.argv[5]) if len(sys.argv) > 5 else 0 stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint mh = plotting.MultiHistory() colors = ['r', 'y', 'b', 'g', 'c', 'k'] def custom_err_fn(trial): if 2 == trial['status']: rval = 1.0 - trial['result']['best_epoch_valid'] if rval > dict( convex=.4, mnist_rotated_background_images=2)[dataset_name]: return None else: return rval for c, algo in zip(colors, algos): conn_str = conn_str_template % (algo, dataset_name) print('algo', algo) mh.add_experiment( mj=MongoJobs.new_from_connection_str(conn_str), y_fn=custom_err_fn, color=c, label=algo, start=start, stop=stop) plt = plotting.plt # TODO: icml07 undefined plt.axhline( 1.0 - icml07.dbn3_scores[dataset_name], c='k', label='manual+grid') # , dashes=[0,1]) mh.add_scatters() plt.legend() plt.title(dataset_name) plt.show()
def main_plot_histories(cls): import plotting conn_str_template = sys.argv[2] algos = sys.argv[3].split(',') dataset_name = sys.argv[4] start = int(sys.argv[5]) if len(sys.argv) > 5 else 0 stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint mh = plotting.MultiHistory() colors = ['r', 'y', 'b', 'g', 'c', 'k'] def custom_err_fn(trial): if 2 == trial['status']: rval = 1.0 - trial['result']['best_epoch_valid'] if rval > dict( convex=.4, mnist_rotated_background_images=2)[dataset_name]: return None else: return rval for c, algo in zip(colors, algos): conn_str = conn_str_template % (algo, dataset_name) print('algo', algo) mh.add_experiment(mj=MongoJobs.new_from_connection_str(conn_str), y_fn=custom_err_fn, color=c, label=algo, start=start, stop=stop) plt = plotting.plt # TODO: icml07 undefined plt.axhline(1.0 - icml07.dbn3_scores[dataset_name], c='k', label='manual+grid') # , dashes=[0,1]) mh.add_scatters() plt.legend() plt.title(dataset_name) plt.show()
def mongo_jobs(dbname): return MongoJobs.new_from_connection_str( TempMongo.connection_string(dbname))
def mongo_jobs(self, name): return MongoJobs.new_from_connection_str( as_mongo_str('localhost:22334/%s' % name) + '/jobs')