def test_register_trials(database, random_dt, hacked_exp): """Register a list of newly proposed trials/parameters.""" hacked_exp._id = 'lalala' # white box hack trials = [ Trial(params=[{ 'name': 'a', 'type': 'int', 'value': 5 }]), Trial(params=[{ 'name': 'b', 'type': 'int', 'value': 6 }]), ] hacked_exp.register_trials(trials) yo = list(database.trials.find({'experiment': hacked_exp._id})) assert len(yo) == len(trials) assert yo[0]['params'] == list(map(lambda x: x.to_dict(), trials[0].params)) assert yo[1]['params'] == list(map(lambda x: x.to_dict(), trials[1].params)) assert yo[0]['status'] == 'new' assert yo[1]['status'] == 'new' assert yo[0]['submit_time'] == random_dt assert yo[1]['submit_time'] == random_dt
def test_not_allowed_status(self): """Other than `Trial.allowed_stati` are not allowed in `Trial.status`.""" t = Trial() with pytest.raises(ValueError): t.status = 'asdf' with pytest.raises(ValueError): t = Trial(status='ispi')
def stats(self): """Calculate a stats dictionary for this particular experiment. Returns ------- stats : dict Stats ----- trials_completed : int Number of completed trials best_trials_id : int Unique identifier of the `Trial` object in the database which achieved the best known objective result. best_evaluation : float Evaluation score of the best trial start_time : `datetime.datetime` When Experiment was first dispatched and started running. finish_time : `datetime.datetime` When Experiment reached terminating condition and stopped running. duration : `datetime.timedelta` Elapsed time. """ query = dict(experiment=self._id, status='completed') completed_trials = self._db.read('trials', query, selection={ '_id': 1, 'end_time': 1, 'results': 1 }) stats = dict() stats['trials_completed'] = len(completed_trials) stats['best_trials_id'] = None trial = Trial(**completed_trials[0]) stats['best_evaluation'] = trial.objective.value stats['best_trials_id'] = trial.id stats['start_time'] = self.metadata['datetime'] stats['finish_time'] = stats['start_time'] for trial in completed_trials: trial = Trial(**trial) # All trials are going to finish certainly after the start date # of the experiment they belong to if trial.end_time > stats['finish_time']: # pylint:disable=no-member stats['finish_time'] = trial.end_time objective = trial.objective.value if objective < stats['best_evaluation']: stats['best_evaluation'] = objective stats['best_trials_id'] = trial.id stats['duration'] = stats['finish_time'] - stats['start_time'] return stats
def reserve_trial(self, score_handle=None): """Find *new* trials that exist currently in database and select one of them based on the highest score return from `score_handle` callable. :param score_handle: A way to decide which trial out of the *new* ones to to pick as *reserved*, defaults to a random choice. :type score_handle: callable :return: selected `Trial` object, None if could not find any. """ if score_handle is not None and not callable(score_handle): raise ValueError("Argument `score_handle` must be callable with a `Trial`.") query = dict( experiment=self._id, status={'$in': ['new', 'suspended', 'interrupted']} ) new_trials = Trial.build(self._db.read('trials', query)) if not new_trials: return None if score_handle is not None and self.space: scores = list(map(score_handle, map(lambda x: trial_to_tuple(x, self.space), new_trials))) scored_trials = zip(scores, new_trials) best_trials = filter(lambda st: st[0] == max(scores), scored_trials) new_trials = list(zip(*best_trials))[1] elif score_handle is not None: log.warning("While reserving trial: `score_handle` was provided, but " "parameter space has not been defined yet.") selected_trial = random.sample(new_trials, 1)[0] # Query on status to ensure atomicity. If another process change the # status meanwhile, read_and_write will fail, because query will fail. query = {'_id': selected_trial.id, 'status': selected_trial.status} update = dict(status='reserved') if selected_trial.status == 'new': update["start_time"] = datetime.datetime.utcnow() selected_trial_dict = self._db.read_and_write( 'trials', query=query, data=update) if selected_trial_dict is None: selected_trial = self.reserve_trial(score_handle=score_handle) else: selected_trial = Trial(**selected_trial_dict) return selected_trial
def test_value_not_allowed_type(self): """Other than `Trial.Result.allowed_types` are not allowed in `Trial.Result.type`. Same for `Trial.Param`. """ with pytest.raises(ValueError): v = Trial.Result(name='asfda', type='hoho') v = Trial.Result() with pytest.raises(ValueError): v.type = 'asfda' with pytest.raises(ValueError): v = Trial.Param(name='asfda', type='hoho') v = Trial.Param() with pytest.raises(ValueError): v.type = 'asfda'
def trial(): """Stab trial to match tuple from fixture `fixed_suggestion`.""" params = [ dict(name='yolo', type='categorical', value=('asdfa', 2)), dict(name='yolo2', type='integer', value=0), dict(name='yolo3', type='real', value=3.5) ] return Trial(params=params)
def test_generate_only_with_json_config(self, spacebuilder, json_sample_path, tmpdir, json_converter): """Build a space using only a json config.""" spacebuilder.build_from(['--config=' + json_sample_path]) trial = Trial(params=[{ 'name': '/layers/1/width', 'type': 'integer', 'value': 100 }, { 'name': '/layers/1/type', 'type': 'categorical', 'value': 'relu' }, { 'name': '/layers/2/type', 'type': 'categorical', 'value': 'sigmoid' }, { 'name': '/training/lr0', 'type': 'real', 'value': 0.032 }, { 'name': '/training/mbs', 'type': 'integer', 'value': 64 }, { 'name': '/something-same', 'type': 'categorical', 'value': '3' }]) output_file = str(tmpdir.join("output.json")) cmd_inst = spacebuilder.build_to(output_file, trial) assert cmd_inst == ['--config=' + output_file] output_data = json_converter.parse(output_file) assert output_data == { 'yo': 5, 'training': { 'lr0': 0.032, 'mbs': 64 }, 'layers': [{ 'width': 64, 'type': 'relu' }, { 'width': 100, 'type': 'relu' }, { 'width': 16, 'type': 'sigmoid' }], 'something-same': '3' }
def test_init_empty(self): """Initialize empty trial.""" t = Trial() assert t.experiment is None assert t.status == 'new' assert t.worker is None assert t.submit_time is None assert t.start_time is None assert t.end_time is None assert t.results == [] assert t.params == []
def test_push_completed_trial(hacked_exp, database, random_dt): """Successfully push a completed trial into database.""" trial = hacked_exp.reserve_trial() trial.results = [] res = Trial.Result(name='yolo', type='objective', value='3') trial.results.append(res) hacked_exp.push_completed_trial(trial) yo = database.trials.find_one({'_id': trial.id}) assert len(yo['results']) == len(trial.results) assert yo['results'][0] == trial.results[0].to_dict() assert yo['status'] == 'completed' assert yo['end_time'] == random_dt
def tuple_to_trial(data, space): """Create a `metaopt.core.worker.trial.Trial` object from `data`, filling only parameter information from `data`. :param data: A tuple representing a sample point from `space`. :param space: Definition of problem's domain. :type space: `metaopt.algo.space.Space` """ assert len(data) == len(space) params = [ dict(name=space[order].name, type=space[order].type, value=data[order]) for order in range(len(space)) ] return Trial(params=params)
def test_gradient_property(self, exp_config): """Check property `Trial.gradient`.""" # 1 results in `results` list t = Trial(**exp_config[1][2]) assert isinstance(t.gradient, Trial.Result) assert t.gradient.name == 'naedw_grad' assert t.gradient.type == 'gradient' assert t.gradient.value == [5, 3] # 0 results in `results` list tmp = exp_config[1][2]['results'].pop() t = Trial(**exp_config[1][2]) assert t.gradient is None exp_config[1][2]['results'].append(tmp) # >1 results in `results` list exp_config[1][2]['results'].append( dict(name='yolo2', type='gradient', value=[12, 15])) t = Trial(**exp_config[1][2]) assert isinstance(t.gradient, Trial.Result) assert t.gradient.name == 'naedw_grad' assert t.gradient.type == 'gradient' assert t.gradient.value == [5, 3] tmp = exp_config[1][2]['results'].pop()
def test_objective_property(self, exp_config): """Check property `Trial.objective`.""" # 1 results in `results` list t = Trial(**exp_config[1][2]) assert isinstance(t.objective, Trial.Result) assert t.objective.name == 'yolo' assert t.objective.type == 'objective' assert t.objective.value == 10 # 0 results in `results` list tmp = exp_config[1][2]['results'].pop(0) t = Trial(**exp_config[1][2]) assert t.objective is None exp_config[1][2]['results'].append(tmp) # >1 results in `results` list exp_config[1][2]['results'].append( dict(name='yolo2', type='objective', value=12)) t = Trial(**exp_config[1][2]) assert isinstance(t.objective, Trial.Result) assert t.objective.name == 'yolo' assert t.objective.type == 'objective' assert t.objective.value == 10 tmp = exp_config[1][2]['results'].pop()
def test_init_full(self, exp_config): """Initialize with a dictionary with complete specification.""" t = Trial(**exp_config[1][1]) assert t.experiment == exp_config[1][1]['experiment'] assert t.status == exp_config[1][1]['status'] assert t.worker == exp_config[1][1]['worker'] assert t.submit_time == exp_config[1][1]['submit_time'] assert t.start_time == exp_config[1][1]['start_time'] assert t.end_time == exp_config[1][1]['end_time'] assert list(map(lambda x: x.to_dict(), t.results)) == exp_config[1][1]['results'] assert t.results[0].name == exp_config[1][1]['results'][0]['name'] assert t.results[0].type == exp_config[1][1]['results'][0]['type'] assert t.results[0].value == exp_config[1][1]['results'][0]['value'] assert list(map(lambda x: x.to_dict(), t.params)) == exp_config[1][1]['params']
def fetch_completed_trials(self): """Fetch recent completed trials that this `Experiment` instance has not yet seen. .. note:: It will return only those with `Trial.end_time` after `_last_fetched`, for performance reasons. :return: list of completed `Trial` objects """ query = dict(experiment=self._id, status='completed', end_time={'$gte': self._last_fetched}) completed_trials = Trial.build(self._db.read('trials', query)) self._last_fetched = datetime.datetime.utcnow() return completed_trials
def _consume(self, trial, workdirname): config_file = tempfile.NamedTemporaryFile(mode='w', prefix='trial_', suffix='.conf', dir=workdirname, delete=False) config_file.close() log.debug("## New temp config file: %s", config_file.name) results_file = tempfile.NamedTemporaryFile(mode='w', prefix='results_', suffix='.log', dir=workdirname, delete=False) results_file.close() log.debug("## New temp results file: %s", results_file.name) log.debug( "## Building command line argument and configuration for trial.") cmd_args = self.template_builder.build_to(config_file.name, trial) log.debug( "## Launch user's script as a subprocess and wait for finish.") script_process = self.launch_process(results_file.name, cmd_args) if script_process is None: return None returncode = script_process.wait() if returncode != 0: log.error( "Something went wrong. Check logs. Process " "returned with code %d !", returncode) return None log.debug( "## Parse results from file and fill corresponding Trial object.") results = self.converter.parse(results_file.name) trial.results = [ Trial.Result(name=res['name'], type=res['type'], value=res['value']) for res in results ] return trial
def test_generate_without_config(self, spacebuilder): """Build a space using only args.""" cmd_args = [ "--seed=555", "-yolo~uniform(-3, 1)", "--arch1=choices({'lala': 0.2, 'yolo': 0.8})", "--arch2~choices({'lala': 0.2, 'yolo': 0.8})" ] spacebuilder.build_from(cmd_args) trial = Trial(params=[{ 'name': '/yolo', 'type': 'real', 'value': -2.4 }, { 'name': '/arch2', 'type': 'categorical', 'value': 'yolo' }]) cmd_inst = spacebuilder.build_to(None, trial) assert cmd_inst == [ "--seed=555", "--arch1=choices({'lala': 0.2, 'yolo': 0.8})", "-yolo=-2.4", "--arch2=yolo" ]
def reserve_trial(self, score_handle=None): """Find *new* trials that exist currently in database and select one of them based on the highest score return from `score_handle` callable. :param score_handle: A way to decide which trial out of the *new* ones to to pick as *reserved*, defaults to a random choice. :type score_handle: callable :return: selected `Trial` object, None if could not find any. """ if score_handle is not None and not callable(score_handle): raise ValueError( "Argument `score_handle` must be callable with a `Trial`.") query = dict(experiment=self._id, status={'$in': ['new', 'suspended', 'interrupted']}) new_trials = Trial.build(self._db.read('trials', query)) if not new_trials: return None if score_handle is None: selected_trial = random.sample(new_trials, 1)[0] else: raise NotImplementedError( "scoring will be supported in the next iteration.") if selected_trial.status == 'new': selected_trial.start_time = datetime.datetime.utcnow() selected_trial.status = 'reserved' self._db.write('trials', selected_trial.to_dict(), query={'_id': selected_trial.id}) return selected_trial
def test_bad_access(self): """Other than `Trial.__slots__` are not allowed.""" t = Trial() with pytest.raises(AttributeError): t.asdfa = 3
def test_generate_from_args_and_config(self, spacebuilder, json_sample_path, tmpdir, json_converter): """Build a space using only a json config.""" cmd_args = [ "--seed=555", "-yolo~uniform(-3, 1)", '--config=' + json_sample_path, "--arch1=choices({'lala': 0.2, 'yolo': 0.8})", "--arch2~choices({'lala': 0.2, 'yolo': 0.8})" ] spacebuilder.build_from(cmd_args) trial = Trial(params=[{ 'name': '/yolo', 'type': 'real', 'value': -2.4 }, { 'name': '/arch2', 'type': 'categorical', 'value': 'yolo' }, { 'name': '/layers/1/width', 'type': 'integer', 'value': 100 }, { 'name': '/layers/1/type', 'type': 'categorical', 'value': 'relu' }, { 'name': '/layers/2/type', 'type': 'categorical', 'value': 'sigmoid' }, { 'name': '/training/lr0', 'type': 'real', 'value': 0.032 }, { 'name': '/training/mbs', 'type': 'integer', 'value': 64 }, { 'name': '/something-same', 'type': 'categorical', 'value': '3' }]) output_file = str(tmpdir.join("output.json")) cmd_inst = spacebuilder.build_to(output_file, trial) assert cmd_inst == ['--config=' + output_file] + [ "--seed=555", "--arch1=choices({'lala': 0.2, 'yolo': 0.8})", "-yolo=-2.4", "--arch2=yolo" ] output_data = json_converter.parse(output_file) assert output_data == { 'yo': 5, 'training': { 'lr0': 0.032, 'mbs': 64 }, 'layers': [{ 'width': 64, 'type': 'relu' }, { 'width': 100, 'type': 'relu' }, { 'width': 16, 'type': 'sigmoid' }], 'something-same': '3' }
def test_value_bad_init(self): """Other than `Trial.Value.__slots__` are not allowed in __init__ too.""" with pytest.raises(AttributeError): Trial.Value(ispii='iela')
def test_convertion_to_dict(self, exp_config): """Convert to dictionary form for database using ``dict``.""" t = Trial(**exp_config[1][1]) assert t.to_dict() == exp_config[1][1]
def test_build_trials(self, exp_config): """Convert to objects form using `Trial.build`.""" trials = Trial.build(exp_config[1]) assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]
def test_str_trial(self, exp_config): """Test representation of `Trial`.""" t = Trial(**exp_config[1][2]) assert str(t) == "Trial(experiment='supernaedo2', "\ "status='completed', params.value=['gru', 'lstm_with_attention'])"
def test_str_value(self, exp_config): """Test representation of `Trial.Value`.""" t = Trial(**exp_config[1][2]) assert str(t.params[0]) == "Param(name='/encoding_layer', "\ "type='categorical', value='gru')"