def test_trial_attachments():
exp_key = "A"
with TempMongo() as tm:
mj = tm.mongo_jobs("foo")
trials = MongoTrials(tm.connection_string("foo"), exp_key=exp_key)
space = hp.uniform("x", -10, 10)
max_evals = 3
fmin_thread = threading.Thread(target=fmin_thread_fn,
args=(space, trials, max_evals))
fmin_thread.start()
mw = MongoWorker(mj=mj, logfilename=None, workdir="mongoexp_test_dir")
n_jobs = max_evals
while n_jobs:
try:
mw.run_one("hostname", 10.0, erase_created_workdir=True)
print("worker: ran job")
except Exception as exc:
print(f"worker: encountered error : {str(exc)}")
traceback.print_exc()
n_jobs -= 1
fmin_thread.join()
all_trials = MongoTrials(tm.connection_string("foo"))
assert len(all_trials) == max_evals
assert trials.count_by_state_synced(JOB_STATE_DONE) == max_evals
assert trials.count_by_state_unsynced(JOB_STATE_DONE) == max_evals
def worker_thread_fn(host_id, n_jobs, timeout):
mw = MongoWorker(mj=TempMongo.mongo_jobs('foodb'),
logfilename=None,
workdir="mongoexp_test_dir")
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print('worker: %s ran job' % str(host_id))
n_jobs -= 1
def worker_thread_fn(host_id, n_jobs, timeout):
mw = MongoWorker(
mj=TempMongo.mongo_jobs('foodb'),
logfilename=None)
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print('worker: %s ran job' % str(host_id))
n_jobs -= 1
def worker(host_id, n_jobs, timeout):
mw = MongoWorker(
mj=TempMongo.mongo_jobs('foodb'),
)
t0 = time.time()
while n_jobs:
mw.run_one(host_id, timeout)
print 'worker: ran job'
n_jobs -= 1
def hyperopt_worker(
progress_queue: multiprocessing.Queue,
logging_queue: multiprocessing.Queue,
exp_key: str,
workdir: str = ".",
gpu: bool = True,
hw_id: str = None,
poll_interval: float = 1.0,
reserve_timeout: float = 30.0,
mongo_port_address: str = "localhost:1234/scvi_db",
):
"""Launches a ``hyperopt`` ``MongoWorker`` which runs jobs until ``ReserveTimeout`` is raised.
:param progress_queue: Queue in which to put None when a job is done.
:param logging_queue: Queue to send logs to using a ``QueueHandler``.
:param exp_key: This key is used by hyperopt as a suffix to the part of the MongoDb
which corresponds to the current experiment. In particular, it has to be passed to ``MongoWorker``.
:param workdir:
:param gpu: If ``True`` means a GPU is to be used.
:param hw_id: Id of the GPU to use. set via env variable ``CUDA_VISIBLE_DEVICES``.
:param poll_interval: Time to wait between attempts to reserve a job.
:param reserve_timeout: Amount of time, in seconds, a worker tries to reserve a job for
before throwing a ``ReserveTimeout`` Exception.
:param mongo_port_address: Addres to the running MongoDb service.
"""
# write all logs to queue
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
queue_handler = QueueHandler(logging_queue)
queue_handler.setLevel(logging.DEBUG)
root_logger.addHandler(queue_handler)
logger.debug("Worker working...")
os.environ["CUDA_VISIBLE_DEVICES"] = hw_id if gpu else str()
# FIXME is this stil necessary?
sys.path.append(".")
mjobs = MongoJobs.new_from_connection_str(
os.path.join(as_mongo_str(mongo_port_address), "jobs"))
mworker = MongoWorker(mjobs,
float(poll_interval),
workdir=workdir,
exp_key=exp_key)
while True:
# FIXME we don't protect ourselves from memory leaks, bad cleanup, etc.
try:
mworker.run_one(reserve_timeout=float(reserve_timeout))
progress_queue.put(None)
except ReserveTimeout:
logger.debug(
"Caught ReserveTimeout. "
"Exiting after failing to reserve job for {time} seconds.".
format(time=reserve_timeout))
break
def _worker_thread_fn(host_id, n_jobs, timeout, dbname='foo'):
mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname))
try:
while n_jobs:
mw.run_one(host_id, timeout)
print 'worker: %s ran job' % str(host_id)
n_jobs -= 1
except ReserveTimeout:
print 'worker timed out:', host_id
pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname="foo", logfilename=None):
mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename)
try:
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print "worker: %s ran job" % str(host_id)
n_jobs -= 1
except ReserveTimeout:
print "worker timed out:", host_id
pass
def run(self):
print "Starting a worker for exp_key = %s" % str(self.exp_key)
mworker = MongoWorker(self.mj,
float(self.options["poll_interval"]),
workdir=self.options["workdir"],
exp_key=self.exp_key)
while True:
res = mworker.run_one(reserve_timeout=float(self.options["reserve_timeout"]))
def run(self):
print "Starting a worker for exp_key = %s" % str(self.exp_key)
mworker = MongoWorker(self.mj,
float(self.options["poll_interval"]),
workdir=self.options["workdir"],
exp_key=self.exp_key)
while True:
res = mworker.run_one(
reserve_timeout=float(self.options["reserve_timeout"]))
def _worker_thread_fn(host_id,
n_jobs,
timeout,
dbname='foo',
logfilename=None):
mw = MongoWorker(mj=TempMongo.mongo_jobs(dbname), logfilename=logfilename)
try:
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print 'worker: %s ran job' % str(host_id)
n_jobs -= 1
except ReserveTimeout:
print 'worker timed out:', host_id
pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname='foo', logfilename=None):
mw = MongoWorker(
mj=TempMongo.mongo_jobs(dbname),
logfilename=logfilename,
workdir="mongoexp_test_dir",
)
try:
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print('worker: %s ran job' % str(host_id))
n_jobs -= 1
except ReserveTimeout:
print('worker timed out:', host_id)
pass
def _worker_thread_fn(host_id, n_jobs, timeout, dbname="foo", logfilename=None):
mw = MongoWorker(
mj=TempMongo.mongo_jobs(dbname),
logfilename=logfilename,
workdir="mongoexp_test_dir",
)
try:
while n_jobs:
mw.run_one(host_id, timeout, erase_created_workdir=True)
print("worker: %s ran job" % str(host_id))
n_jobs -= 1
except ReserveTimeout:
print("worker timed out:", host_id)
pass
def worker_thread_fn(host_id, n_jobs, timeout):
mw = MongoWorker(mj=TempMongo.mongo_jobs('foodb'))
while n_jobs:
mw.run_one(host_id, timeout)
print 'worker: %s ran job' % str(host_id)
n_jobs -= 1
