Exemplo n.º 1
0
 def __init__(self, mongo_uri, options):
     self.options = options
     self.mj = MongoJobs.new_from_connection_str(mongo_uri + "netron/hyperopt_jobs")
     last_job = self.mj.db["hyperopt_jobs"].find({}, {"exp_key": 1}).sort("exp_key", pymongo.DESCENDING).limit(1)
     last_job = list(last_job)
     if len(last_job) > 0:
         self.exp_key = last_job[0]["exp_key"]
     else:
         self.exp_key = "0"
Exemplo n.º 2
0
def hyperopt_worker(
    progress_queue: multiprocessing.Queue,
    logging_queue: multiprocessing.Queue,
    exp_key: str,
    workdir: str = ".",
    gpu: bool = True,
    hw_id: str = None,
    poll_interval: float = 1.0,
    reserve_timeout: float = 30.0,
    mongo_port_address: str = "localhost:1234/scvi_db",
):
    """Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised.

    :param progress_queue: Queue in which to put None when a job is done.
    :param logging_queue: Queue to send logs to using a ``QueueHandler``.
    :param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb
        which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``.
    :param workdir:
    :param gpu: If ``True`` means a GPU is to be used.
    :param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``.
    :param poll_interval: Time to wait between attempts to reserve a job.
    :param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for
        before throwing a ``ReserveTimeout`` Exception.
    :param mongo_port_address: Addres to the running MongoDb service.
    """
    # write all logs to queue
    root_logger = logging.getLogger()
    root_logger.setLevel(logging.DEBUG)
    queue_handler = QueueHandler(logging_queue)
    queue_handler.setLevel(logging.DEBUG)
    root_logger.addHandler(queue_handler)
    logger.debug("Worker working...")

    os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str()

    # FIXME is this stil necessary?
    sys.path.append(".")

    mjobs = MongoJobs.new_from_connection_str(
        os.path.join(as_mongo_str(mongo_port_address), "jobs"))
    mworker = MongoWorker(mjobs,
                          float(poll_interval),
                          workdir=workdir,
                          exp_key=exp_key)

    while True:
        # FIXME we don't protect ourselves from memory leaks, bad cleanup, etc.
        try:
            mworker.run_one(reserve_timeout=float(reserve_timeout))
            progress_queue.put(None)
        except ReserveTimeout:
            logger.debug(
                "Caught ReserveTimeout. "
                "Exiting after failing to reserve job for {time} seconds.".
                format(time=reserve_timeout))
            break
Exemplo n.º 3
0
 def __init__(self, mongo_uri, options):
     self.options = options
     self.mj = MongoJobs.new_from_connection_str(mongo_uri +
                                                 "netron/hyperopt_jobs")
     last_job = self.mj.db["hyperopt_jobs"].find({}, {
         "exp_key": 1
     }).sort("exp_key", pymongo.DESCENDING).limit(1)
     last_job = list(last_job)
     if len(last_job) > 0:
         self.exp_key = last_job[0]["exp_key"]
     else:
         self.exp_key = "0"
Exemplo n.º 4
0
    def main_plot_histories(cls):
        import plotting
        conn_str_template = sys.argv[2]
        algos = sys.argv[3].split(',')
        dataset_name = sys.argv[4]
        start = int(sys.argv[5]) if len(sys.argv) > 5 else 0
        stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint
        mh = plotting.MultiHistory()
        colors = ['r', 'y', 'b', 'g', 'c', 'k']

        def custom_err_fn(trial):
            if 2 == trial['status']:
                rval = 1.0 - trial['result']['best_epoch_valid']
                if rval > dict(
                        convex=.4,
                        mnist_rotated_background_images=2)[dataset_name]:
                    return None
                else:
                    return rval

        for c, algo in zip(colors, algos):
            conn_str = conn_str_template % (algo, dataset_name)
            print('algo', algo)
            mh.add_experiment(
                mj=MongoJobs.new_from_connection_str(conn_str),
                y_fn=custom_err_fn,
                color=c,
                label=algo,
                start=start,
                stop=stop)
        plt = plotting.plt
        # TODO: icml07 undefined
        plt.axhline(
            1.0 - icml07.dbn3_scores[dataset_name],
            c='k', label='manual+grid')  # , dashes=[0,1])
        mh.add_scatters()
        plt.legend()
        plt.title(dataset_name)
        plt.show()
Exemplo n.º 5
0
    def main_plot_histories(cls):
        import plotting
        conn_str_template = sys.argv[2]
        algos = sys.argv[3].split(',')
        dataset_name = sys.argv[4]
        start = int(sys.argv[5]) if len(sys.argv) > 5 else 0
        stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint
        mh = plotting.MultiHistory()
        colors = ['r', 'y', 'b', 'g', 'c', 'k']

        def custom_err_fn(trial):
            if 2 == trial['status']:
                rval = 1.0 - trial['result']['best_epoch_valid']
                if rval > dict(
                        convex=.4,
                        mnist_rotated_background_images=2)[dataset_name]:
                    return None
                else:
                    return rval

        for c, algo in zip(colors, algos):
            conn_str = conn_str_template % (algo, dataset_name)
            print('algo', algo)
            mh.add_experiment(mj=MongoJobs.new_from_connection_str(conn_str),
                              y_fn=custom_err_fn,
                              color=c,
                              label=algo,
                              start=start,
                              stop=stop)
        plt = plotting.plt
        # TODO: icml07 undefined
        plt.axhline(1.0 - icml07.dbn3_scores[dataset_name],
                    c='k',
                    label='manual+grid')  # , dashes=[0,1])
        mh.add_scatters()
        plt.legend()
        plt.title(dataset_name)
        plt.show()
Exemplo n.º 6
0
 def mongo_jobs(dbname):
     return MongoJobs.new_from_connection_str(
         TempMongo.connection_string(dbname))
Exemplo n.º 7
0
 def mongo_jobs(dbname):
     return MongoJobs.new_from_connection_str(
             TempMongo.connection_string(dbname))
Exemplo n.º 8
0
 def mongo_jobs(self, name):
     return MongoJobs.new_from_connection_str(
             as_mongo_str('localhost:22334/%s' % name) + '/jobs')