def test_add_get_experiment_artifacts(self): experiment_tuple = get_test_experiment() e_experiment = experiment_tuple[0] e_artifacts = e_experiment.artifacts a1_filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) a2_filename = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) with open(a1_filename, 'w') as f: f.write('hello world') e_artifacts['a1'] = { 'local': a1_filename, 'mutable': False } e_artifacts['a2'] = { 'local': a2_filename, 'mutable': True } with self.get_db_provider('test_config_http_client.yaml') as db: db.add_experiment(e_experiment) experiment = db.get_experiment(e_experiment.key) self.assertEquals(experiment.key, e_experiment.key) self.assertEquals(experiment.filename, e_experiment.filename) self.assertEquals(experiment.args, e_experiment.args) db.delete_experiment(e_experiment.key) os.remove(a1_filename)
def test_add_get_experiment(self): experiment_tuple = get_test_experiment() db = self.get_db_provider() db.add_experiment(experiment_tuple[0]) experiment = db.get_experiment(experiment_tuple[0].key) self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals(experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) db.delete_experiment(experiment_tuple[1])
def test_add_get_delete_experiment(self): with self.get_db_provider('test_config.yaml') as fp, \ self.get_db_provider('test_config_http_client.yaml') as hp: experiment_tuple = get_test_experiment() hp.add_experiment(experiment_tuple[0]) experiment = fp.get_experiment(experiment_tuple[0].key) self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals( experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) fp.delete_experiment(experiment_tuple[1]) try: thrown = False hp.get_experiment(experiment_tuple[1]) except BaseException: thrown = True self.assertTrue(thrown) experiment_tuple = get_test_experiment() fp.add_experiment(experiment_tuple[0]) experiment = hp.get_experiment(experiment_tuple[0].key) self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals( experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) hp.delete_experiment(experiment_tuple[1]) try: thrown = False fp.get_experiment(experiment_tuple[1]) except BaseException: thrown = True self.assertTrue(thrown)
def test_add_get_delete_experiment(self): with self.get_db_provider('test_config_http_client.yaml') as hp: experiment_tuple = get_test_experiment() hp.add_experiment(experiment_tuple[0]) experiment = hp.get_experiment(experiment_tuple[0].key) self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals( experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) hp.delete_experiment(experiment_tuple[1]) self.assertTrue(hp.get_experiment(experiment_tuple[1]) is None)
def test_add_experiment(self): with self.get_provider() as fb: experiment, experiment_name, _, _ = get_test_experiment() fb._delete(fb._get_experiments_keybase() + experiment_name) fb.add_experiment(experiment) self.assertTrue(experiment.status == 'waiting') self.assertTrue(experiment.time_added <= time.time()) actual_experiment_dict = fb._get( fb._get_experiments_keybase() + experiment_name) self._compare_experiment_data(experiment, actual_experiment_dict) fb.finish_experiment(experiment) fb.delete_experiment(experiment)
def test_add_experiment(self): with self.get_provider() as fb: experiment, experiment_name, _, _ = get_test_experiment() fb._delete(fb._get_experiments_keybase() + experiment_name) fb.add_experiment(experiment) self.assertTrue(experiment.status == 'waiting') self.assertTrue(experiment.time_added <= time.time()) actual_experiment_dict = fb._get(fb._get_experiments_keybase() + experiment_name) for key, value in six.iteritems(experiment.__dict__): if value: self.assertTrue(actual_experiment_dict[key] == value) fb.finish_experiment(experiment) fb.delete_experiment(experiment)
def test_start_experiment(self): with self.get_db_provider('test_config_http_client.yaml') as hp: experiment_tuple = get_test_experiment() hp.add_experiment(experiment_tuple[0]) hp.start_experiment(experiment_tuple[0]) experiment = hp.get_experiment(experiment_tuple[1]) self.assertTrue(experiment.status == 'running') self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals(experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) hp.finish_experiment(experiment_tuple[0]) hp.delete_experiment(experiment_tuple[1])
def test_start_experiment(self): db = self.get_db_provider() experiment_tuple = get_test_experiment() db.add_experiment(experiment_tuple[0]) db.start_experiment(experiment_tuple[0]) experiment = db.get_experiment(experiment_tuple[1]) self.assertTrue(experiment.status == 'running') self.assertTrue(experiment.time_added <= time.time()) self.assertTrue(experiment.time_started <= time.time()) self.assertEquals(experiment.key, experiment_tuple[0].key) self.assertEquals(experiment.filename, experiment_tuple[0].filename) self.assertEquals(experiment.args, experiment_tuple[0].args) db.finish_experiment(experiment_tuple[0]) db.delete_experiment(experiment_tuple[1])
def test_checkpoint_experiment(self): with self.get_provider() as fb: experiment, experiment_name, _, _, = get_test_experiment() modeldir = experiment.artifacts['modeldir'].local_path if os.path.exists(modeldir): shutil.rmtree(modeldir) os.makedirs(modeldir) try: fb.delete_experiment(experiment_name) except BaseException: pass fb.add_experiment(experiment) fb.start_experiment(experiment) file_in_modeldir = os.path.join(modeldir, str(uuid.uuid4())) random_str = str(uuid.uuid4()) with open(file_in_modeldir, 'w') as f: f.write(random_str) checkpoint_threads = fb.checkpoint_experiment(experiment) if checkpoint_threads: for t in checkpoint_threads: t.join() shutil.rmtree(modeldir) fb.store.get_artifact( fb.get_experiment( experiment_name, getinfo=False).artifacts['modeldir']) with open(file_in_modeldir, 'r') as f: line = f.read() self.assertTrue(line == random_str) fb.delete_experiment(experiment)