def __init__(self, mongo_uri, options):
self.options = options
self.mj = MongoJobs.new_from_connection_str(mongo_uri + "netron/hyperopt_jobs")
last_job = self.mj.db["hyperopt_jobs"].find({}, {"exp_key": 1}).sort("exp_key", pymongo.DESCENDING).limit(1)
last_job = list(last_job)
if len(last_job) > 0:
self.exp_key = last_job[0]["exp_key"]
else:
self.exp_key = "0"
def hyperopt_worker(
progress_queue: multiprocessing.Queue,
logging_queue: multiprocessing.Queue,
exp_key: str,
workdir: str = ".",
gpu: bool = True,
hw_id: str = None,
poll_interval: float = 1.0,
reserve_timeout: float = 30.0,
mongo_port_address: str = "localhost:1234/scvi_db",
):
"""Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised.
:param progress_queue: Queue in which to put None when a job is done.
:param logging_queue: Queue to send logs to using a ``QueueHandler``.
:param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb
which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``.
:param workdir:
:param gpu: If ``True`` means a GPU is to be used.
:param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``.
:param poll_interval: Time to wait between attempts to reserve a job.
:param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for
before throwing a ``ReserveTimeout`` Exception.
:param mongo_port_address: Addres to the running MongoDb service.
"""
# write all logs to queue
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
queue_handler = QueueHandler(logging_queue)
queue_handler.setLevel(logging.DEBUG)
root_logger.addHandler(queue_handler)
logger.debug("Worker working...")
os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str()
# FIXME is this stil necessary?
sys.path.append(".")
mjobs = MongoJobs.new_from_connection_str(
os.path.join(as_mongo_str(mongo_port_address), "jobs"))
mworker = MongoWorker(mjobs,
float(poll_interval),
workdir=workdir,
exp_key=exp_key)
while True:
# FIXME we don't protect ourselves from memory leaks, bad cleanup, etc.
try:
mworker.run_one(reserve_timeout=float(reserve_timeout))
progress_queue.put(None)
except ReserveTimeout:
logger.debug(
"Caught ReserveTimeout. "
"Exiting after failing to reserve job for {time} seconds.".
format(time=reserve_timeout))
break
def __init__(self, mongo_uri, options):
self.options = options
self.mj = MongoJobs.new_from_connection_str(mongo_uri +
"netron/hyperopt_jobs")
last_job = self.mj.db["hyperopt_jobs"].find({}, {
"exp_key": 1
}).sort("exp_key", pymongo.DESCENDING).limit(1)
last_job = list(last_job)
if len(last_job) > 0:
self.exp_key = last_job[0]["exp_key"]
else:
self.exp_key = "0"
def main_plot_histories(cls):
import plotting
conn_str_template = sys.argv[2]
algos = sys.argv[3].split(',')
dataset_name = sys.argv[4]
start = int(sys.argv[5]) if len(sys.argv) > 5 else 0
stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint
mh = plotting.MultiHistory()
colors = ['r', 'y', 'b', 'g', 'c', 'k']
def custom_err_fn(trial):
if 2 == trial['status']:
rval = 1.0 - trial['result']['best_epoch_valid']
if rval > dict(
convex=.4,
mnist_rotated_background_images=2)[dataset_name]:
return None
else:
return rval
for c, algo in zip(colors, algos):
conn_str = conn_str_template % (algo, dataset_name)
print('algo', algo)
mh.add_experiment(
mj=MongoJobs.new_from_connection_str(conn_str),
y_fn=custom_err_fn,
color=c,
label=algo,
start=start,
stop=stop)
plt = plotting.plt
# TODO: icml07 undefined
plt.axhline(
1.0 - icml07.dbn3_scores[dataset_name],
c='k', label='manual+grid') # , dashes=[0,1])
mh.add_scatters()
plt.legend()
plt.title(dataset_name)
plt.show()
def main_plot_histories(cls):
import plotting
conn_str_template = sys.argv[2]
algos = sys.argv[3].split(',')
dataset_name = sys.argv[4]
start = int(sys.argv[5]) if len(sys.argv) > 5 else 0
stop = int(sys.argv[6]) if len(sys.argv) > 6 else sys.maxint
mh = plotting.MultiHistory()
colors = ['r', 'y', 'b', 'g', 'c', 'k']
def custom_err_fn(trial):
if 2 == trial['status']:
rval = 1.0 - trial['result']['best_epoch_valid']
if rval > dict(
convex=.4,
mnist_rotated_background_images=2)[dataset_name]:
return None
else:
return rval
for c, algo in zip(colors, algos):
conn_str = conn_str_template % (algo, dataset_name)
print('algo', algo)
mh.add_experiment(mj=MongoJobs.new_from_connection_str(conn_str),
y_fn=custom_err_fn,
color=c,
label=algo,
start=start,
stop=stop)
plt = plotting.plt
# TODO: icml07 undefined
plt.axhline(1.0 - icml07.dbn3_scores[dataset_name],
c='k',
label='manual+grid') # , dashes=[0,1])
mh.add_scatters()
plt.legend()
plt.title(dataset_name)
plt.show()
def mongo_jobs(dbname):
return MongoJobs.new_from_connection_str(
TempMongo.connection_string(dbname))
def mongo_jobs(dbname):
return MongoJobs.new_from_connection_str(
TempMongo.connection_string(dbname))
def mongo_jobs(self, name):
return MongoJobs.new_from_connection_str(
as_mongo_str('localhost:22334/%s' % name) + '/jobs')