class CPUResourceManagerTestCase(unittest.TestCase):
n_parallel = 1
auppath = os.path.join("tests", "data", ".aup")
job = Job("test_Job.py", BasicConfig(), "./tests/EE")
job_fail = Job("task1.py", BasicConfig(), "./tests/data")
job_fail2 = Job("task2.py", BasicConfig(), "./tests/data")
job_fail3 = Job("task3.py", BasicConfig(), "./tests/data")
ori_db = os.path.join(auppath, "sqlite3.db")
bk_db = os.path.join(auppath, "bk.db")
val = 0
def setUp(self):
copyfile(self.ori_db, self.bk_db)
self.connector = get_default_connector(self.auppath)
self.rm = CPUResourceManager(self.connector, self.n_parallel)
def tearDown(self):
copyfile(self.bk_db, self.ori_db)
os.remove(self.bk_db)
def test_get_resource(self):
self.assertIsInstance(self.rm.get_available("test", "cpu"), int)
# !IMPORTANT - currently, there is no user authentication.
self.assertIsInstance(self.rm.get_available("test2", "cpu"), int)
def test_run(self):
def callback(*args):
self.val = -1
self.rm.run(self.job, self.rm.get_available("test", "cpu"), {},
callback)
self.rm.executor.shutdown(wait=True)
self.assertEqual(self.val, -1)
def test_fail_run(self):
def callback(*args):
raise ChildProcessError
self.rm.run(self.job_fail, self.rm.get_available("test", "cpu"), {},
callback)
self.rm.run(self.job_fail2, self.rm.get_available("test", "cpu"), {},
callback)
self.rm.run(self.job_fail3, self.rm.get_available("test", "cpu"), {},
callback)
def test_full_run(self):
def callback(*args):
pass
self.rm.run_job(self.job, self.rm.get_available("test", "cpu"), {},
callback)
self.rm.finish_job(self.job.jid, 2.0)
self.rm.executor.shutdown(wait=True)
self.assertListEqual([1, 0.1], self.rm.finish(1))
def test_finish(self):
self.rm.suspend()
self.assertListEqual([1, 0.1], self.rm.finish(1))
def test_verify_folder(self):
job = Job("test_Job.py", {}, self.path)
self.assertTrue(job.verify_local()) # working demo
job = Job("missing_script", {}, self.path)
self.assertRaises(EnvironmentError, job.verify_local) # missing script
job = Job("__init__.py", {}, self.path)
self.assertRaises(EnvironmentError, job.verify_local) # not executable
job = Job("test_Job.py", {}, os.path.join("tests", "data", "wrong"))
self.assertRaises(EnvironmentError, job.verify_local)
class GPUResourceManagerTestCase(unittest.TestCase):
n_parallel = 1
auppath = os.path.join("tests", "data", ".aup")
connector = get_default_connector(auppath)
val = 0
ori_db = os.path.join(auppath, "sqlite3.db")
bk_db = os.path.join(auppath, "bk.db")
job = Job("test_Job.py", BasicConfig(), "./tests/EE")
def setUp(self):
copyfile(self.ori_db, self.bk_db)
self.connector = get_default_connector(self.auppath)
self.rm = GPUResourceManager(self.connector,
self.n_parallel,
auppath=self.auppath)
def tearDown(self):
copyfile(self.bk_db, self.ori_db)
os.remove(self.bk_db)
def test_run(self):
def callback(*args):
self.val = -1
self.rm.run(self.job, self.rm.get_available("test", "gpu"), {},
callback)
sleep(3) # wait till subprocess finished - handled by rm.finish()
self.assertEqual(self.val, -1)
def test_failed(self):
p = hp.HyperbandProposer(self.pc)
try:
p.failed(Job("none", {"tid": 1}))
self.assertTrue(False)
except NotImplementedError:
pass
def test_runtime(self):
p = hp.HyperoptProposer(self.pc)
c = p.get()
job = Job("none", c)
p.update(0.1, job)
job.config['tid'] = -100
self.assertRaises(KeyError, p.update, 0.0, job)
path = tempfile.mkdtemp()
sav = os.path.join(path, "save.pkl")
p.save(sav)
idx = p.counter
p = hp.HyperoptProposer(self.pc)
self.assertNotEqual(idx, p.counter)
p.reload(sav)
self.assertEqual(idx, p.counter)
os.remove(sav)
os.rmdir(path)
def test_failed(self):
self.pc["parameter_config"] = [
{"name": "x1", "type": "choice", "range": [2, 4, 6, 8, 10]},
{"name": "x2", "type": "choice", "range": ["a", "b", "c"]}
]
p = get_proposer("sequence")(self.pc)
c = p.get()
job = Job("none", c)
p.failed(job)
def test_hp_min(self):
self.pc['target'] = 'max'
p = hp.HyperbandProposer(self.pc)
for j in range(6):
tid = []
for i in range(100):
v = p.get()
if v:
tid.append(v['tid'])
for i in tid:
job = Job("none", {"tid": i})
p.update(0.1, job)
def test_get(self):
p = sp.SpearmintProposer(self.pc)
for j in range(3):
tid = []
for i in range(10):
v = p.get()
tid.append(v['job_id'])
for i in tid:
job = Job("none", {'job_id': i})
p.update(1, job)
self.assertTrue(True)
self.assertTrue(sp.SpearmintProposer(self.pc))
def test_failed(self):
config = {
'n_samples': 10,
'random_seed': 10,
'parameter_config': [{
'name': 'x',
'type': 'int'
}]
}
p = rp.RandomProposer(config)
c = p.get()
job = Job("none", c)
p.failed(job)
def test_failed(self):
from aup.Proposer import BOHBProposer as bohb
proposer = bohb.BOHBProposer({
"parameter_config": self.config,
"target": "min",
"n_iterations": 2,
'max_budget': 3
})
try:
proposer.failed(Job("none", {"tid": 1}))
self.assertTrue(False)
except NotImplementedError:
pass
def test_hp(self):
p = hp.HyperbandProposer(self.pc)
for j in range(3):
tid = []
for i in range(100):
v = p.get()
if v:
tid.append(v['tid'])
for i in tid:
job = Job("none", {"tid": i})
p.update(0.1, job)
self.assertRaises(NotImplementedError, p.save, '')
self.assertRaises(NotImplementedError, p.reload, '')
def test_param(self):
from aup.Proposer import BOHBProposer as bohb
from aup.EE.Job import Job
proposer = bohb.BOHBProposer({
"parameter_config": self.config,
"target": "min",
"n_iterations": 2,
'max_budget': 3
})
for i in range(proposer.nSamples):
config = proposer.get_param()
self.assertTrue(0 <= config['float'] <= 1)
self.assertTrue(0 <= config['int'] <= 10)
self.assertTrue(config['choice'] in (1, 2, 3))
job = Job('.', config)
proposer.update(0, job)
self.assertIsNone(proposer.get_param())
class PassiveRMTestCase(unittest.TestCase):
auppath = os.path.join("tests", "data", ".aup")
job = Job("task1.py", BasicConfig(), "./tests/EE")
ori_db = os.path.join(auppath, "sqlite3.db")
bk_db = os.path.join(auppath, "bk.db")
val = 0
def setUp(self):
copyfile(self.ori_db, self.bk_db)
self.connector = get_default_connector(self.auppath)
self.rm = PassiveResourceManager(self.connector)
def tearDown(self):
copyfile(self.bk_db, self.ori_db)
os.remove(self.bk_db)
def test_get_available(self):
self.assertEqual(10, self.rm.get_available("test", "passive"))
self.connector.take_available_resource(10)
self.assertEqual(None, self.rm.get_available("test", "passive"))
self.rm.running = True
self.assertEqual(None, self.rm.get_available("test", "passive"))
def test_ssh_session(self):
pkey = pkg_resources.resource_filename("mockssh", 'server-key')
data_folder = pkg_resources.resource_filename(__name__, 'data')
users = {
"mockuser": pkey,
}
scores = {}
mutex = Lock()
config = {
'workingdir':
data_folder,
'n_parallel':
2,
'resource':
'node',
'proposer':
'random',
'script':
os.path.join(data_folder, "script", "rosenbrock_hpo.py"),
'n_samples':
10,
'random_seed':
10,
'parameter_config': [{
'name': 'x',
'type': 'float',
'range': [-5, 5]
}, {
'name': 'y',
'type': 'float',
'range': [-5, 5]
}],
"runtime_args": {
"prescript": "export CUDA_VISIBLE_DEVICES=-1",
"postscript": "echo $CUDA_VISIBLE_DEVICES",
"overwrite": 'true'
}
}
m = rp.RandomProposer(config)
def callback_fun(score, jid):
mutex.acquire()
scores[jid] = score
mutex.release()
with mockssh.Server(users) as server1, mockssh.Server(
users) as server2:
mngr = SSHResourceManager(None,
config['n_parallel'],
auppath=data_folder,
reconn_wait_time=3,
max_retries=3)
mngr.mapping[1] = "[email protected]:" + str(
server1.port) + " " + pkey
mngr.mapping[2] = "[email protected]:" + str(
server2.port) + " " + pkey
for i in range(0, config['n_samples'], 2):
proposal = m.get()
job1 = Job(config['script'], BasicConfig(**proposal),
config['workingdir'])
job1.jid = i
mngr.run(job1, 1, None, callback_fun, overwrite=True)
proposal = m.get()
job2 = Job(config['script'], BasicConfig(**proposal),
config['workingdir'])
job2.jid = i + 1
mngr.run(job2, 2, None, callback_fun, overwrite=True)
mngr.executor.shutdown(wait=True)
for key in scores:
self.assertTrue(scores[key] != "ERROR")
def test_failed(self):
p = hp.HyperoptProposer(self.pc)
c = p.get()
job = Job("none", c)
p.failed(job)
def test_failed(self):
p = sp.SpearmintProposer(self.pc)
c = p.get()
job = Job("none", c)
p.failed(job)