def test_add_get_experiment_artifacts(self):
experiment_tuple = get_test_experiment()
e_experiment = experiment_tuple[0]
e_artifacts = e_experiment.artifacts
a1_filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
a2_filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
with open(a1_filename, 'w') as f:
f.write('hello world')
e_artifacts['a1'] = {
'local': a1_filename,
'mutable': False
}
e_artifacts['a2'] = {
'local': a2_filename,
'mutable': True
}
with self.get_db_provider('test_config_http_client.yaml') as db:
db.add_experiment(e_experiment)
experiment = db.get_experiment(e_experiment.key)
self.assertEquals(experiment.key, e_experiment.key)
self.assertEquals(experiment.filename, e_experiment.filename)
self.assertEquals(experiment.args, e_experiment.args)
db.delete_experiment(e_experiment.key)
os.remove(a1_filename)
def test_add_get_experiment(self):
experiment_tuple = get_test_experiment()
db = self.get_db_provider()
db.add_experiment(experiment_tuple[0])
experiment = db.get_experiment(experiment_tuple[0].key)
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(experiment.filename, experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
db.delete_experiment(experiment_tuple[1])
def test_add_get_delete_experiment(self):
with self.get_db_provider('test_config.yaml') as fp, \
self.get_db_provider('test_config_http_client.yaml') as hp:
experiment_tuple = get_test_experiment()
hp.add_experiment(experiment_tuple[0])
experiment = fp.get_experiment(experiment_tuple[0].key)
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(
experiment.filename,
experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
fp.delete_experiment(experiment_tuple[1])
try:
thrown = False
hp.get_experiment(experiment_tuple[1])
except BaseException:
thrown = True
self.assertTrue(thrown)
experiment_tuple = get_test_experiment()
fp.add_experiment(experiment_tuple[0])
experiment = hp.get_experiment(experiment_tuple[0].key)
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(
experiment.filename,
experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
hp.delete_experiment(experiment_tuple[1])
try:
thrown = False
fp.get_experiment(experiment_tuple[1])
except BaseException:
thrown = True
self.assertTrue(thrown)
def test_add_get_delete_experiment(self):
with self.get_db_provider('test_config_http_client.yaml') as hp:
experiment_tuple = get_test_experiment()
hp.add_experiment(experiment_tuple[0])
experiment = hp.get_experiment(experiment_tuple[0].key)
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(
experiment.filename,
experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
hp.delete_experiment(experiment_tuple[1])
self.assertTrue(hp.get_experiment(experiment_tuple[1]) is None)
def test_add_experiment(self):
with self.get_provider() as fb:
experiment, experiment_name, _, _ = get_test_experiment()
fb._delete(fb._get_experiments_keybase() + experiment_name)
fb.add_experiment(experiment)
self.assertTrue(experiment.status == 'waiting')
self.assertTrue(experiment.time_added <= time.time())
actual_experiment_dict = fb._get(
fb._get_experiments_keybase() + experiment_name)
self._compare_experiment_data(experiment, actual_experiment_dict)
fb.finish_experiment(experiment)
fb.delete_experiment(experiment)
def test_add_experiment(self):
with self.get_provider() as fb:
experiment, experiment_name, _, _ = get_test_experiment()
fb._delete(fb._get_experiments_keybase() + experiment_name)
fb.add_experiment(experiment)
self.assertTrue(experiment.status == 'waiting')
self.assertTrue(experiment.time_added <= time.time())
actual_experiment_dict = fb._get(fb._get_experiments_keybase() +
experiment_name)
for key, value in six.iteritems(experiment.__dict__):
if value:
self.assertTrue(actual_experiment_dict[key] == value)
fb.finish_experiment(experiment)
fb.delete_experiment(experiment)
def test_start_experiment(self):
with self.get_db_provider('test_config_http_client.yaml') as hp:
experiment_tuple = get_test_experiment()
hp.add_experiment(experiment_tuple[0])
hp.start_experiment(experiment_tuple[0])
experiment = hp.get_experiment(experiment_tuple[1])
self.assertTrue(experiment.status == 'running')
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(experiment.filename,
experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
hp.finish_experiment(experiment_tuple[0])
hp.delete_experiment(experiment_tuple[1])
def test_start_experiment(self):
db = self.get_db_provider()
experiment_tuple = get_test_experiment()
db.add_experiment(experiment_tuple[0])
db.start_experiment(experiment_tuple[0])
experiment = db.get_experiment(experiment_tuple[1])
self.assertTrue(experiment.status == 'running')
self.assertTrue(experiment.time_added <= time.time())
self.assertTrue(experiment.time_started <= time.time())
self.assertEquals(experiment.key, experiment_tuple[0].key)
self.assertEquals(experiment.filename, experiment_tuple[0].filename)
self.assertEquals(experiment.args, experiment_tuple[0].args)
db.finish_experiment(experiment_tuple[0])
db.delete_experiment(experiment_tuple[1])
def test_checkpoint_experiment(self):
with self.get_provider() as fb:
experiment, experiment_name, _, _, = get_test_experiment()
modeldir = experiment.artifacts['modeldir'].local_path
if os.path.exists(modeldir):
shutil.rmtree(modeldir)
os.makedirs(modeldir)
try:
fb.delete_experiment(experiment_name)
except BaseException:
pass
fb.add_experiment(experiment)
fb.start_experiment(experiment)
file_in_modeldir = os.path.join(modeldir, str(uuid.uuid4()))
random_str = str(uuid.uuid4())
with open(file_in_modeldir, 'w') as f:
f.write(random_str)
checkpoint_threads = fb.checkpoint_experiment(experiment)
if checkpoint_threads:
for t in checkpoint_threads:
t.join()
shutil.rmtree(modeldir)
fb.store.get_artifact(
fb.get_experiment(
experiment_name,
getinfo=False).artifacts['modeldir'])
with open(file_in_modeldir, 'r') as f:
line = f.read()
self.assertTrue(line == random_str)
fb.delete_experiment(experiment)
