def __to_frozen_trial(self, trial): number = trial["number"] state = trial["state"] datetime_start = trial["datetime_start"] datetime_complete = trial["datetime_complete"] params = trial["params"] distributions = {} for p in params: distributions[p] = json_to_distribution(trial["distributions"][p]) user_attrs = trial["user_attrs"] system_attrs = trial["system_attrs"] trial["intermediate_values"] intkey_val = {} for k in trial["intermediate_values"]: intk = int(k) intkey_val[k] = trial["intermediate_values"][k] intermediate_values = intkey_val trial_id = trial["trial_id"] value = trial["value"] ret = structs.FrozenTrial(number=number, state=structs.TrialState(state), datetime_start=datetime_start, datetime_complete=datetime_complete, params=params, distributions=distributions, user_attrs=user_attrs, system_attrs=system_attrs, intermediate_values=intermediate_values, trial_id=trial_id, value=value) return ret
def _create_running_trial(): # type: () -> structs.FrozenTrial return structs.FrozenTrial( trial_id=-1, # dummy value. number=-1, # dummy value. state=structs.TrialState.RUNNING, params={}, distributions={}, user_attrs={}, system_attrs={}, value=None, intermediate_values={}, datetime_start=datetime.now(), datetime_complete=None)
def _trial(self, trial_id): if trial_id in self.trials: return self.trials[trial_id] trial = structs.FrozenTrial(trial_id=trial_id, state=structs.TrialState.RUNNING, params={}, user_attrs={}, system_attrs={}, value=None, intermediate_values={}, params_in_internal_repr={}, datetime_start=None, datetime_complete=None) self.trials[trial_id] = trial return trial
def create_new_trial_id(self, study_id): # type: (int) -> int self._check_study_id(study_id) with self._lock: trial_id = len(self.trials) self.trials.append( structs.FrozenTrial(trial_id=trial_id, state=structs.TrialState.RUNNING, params={}, user_attrs={}, system_attrs={}, value=None, intermediate_values={}, params_in_internal_repr={}, datetime_start=datetime.now(), datetime_complete=None, timedelta_sampling_elapsed=None)) return trial_id
def create_new_trial(self, study_id): # type: (int) -> int self._check_study_id(study_id) with self._lock: trial_id = len(self.trials) self.trials.append( structs.FrozenTrial(number=trial_id, state=structs.TrialState.RUNNING, params={}, distributions={}, user_attrs={}, system_attrs={'_number': trial_id}, value=None, intermediate_values={}, datetime_start=datetime.now(), datetime_complete=None, trial_id=trial_id)) return trial_id
def _append_trial( self, value=None, # type: Optional[float] params=None, # type: Optional[Dict[str, Any]] distributions=None, # type: Optional[Dict[str, BaseDistribution]] user_attrs=None, # type: Optional[Dict[str, Any]] system_attrs=None, # type: Optional[Dict[str, Any]] intermediate_values=None, # type: Optional[Dict[int, float]] state=structs.TrialState.COMPLETE, # type: structs.TrialState datetime_start=None, # type: Optional[datetime.datetime] datetime_complete=None, # type: Optional[datetime.datetime] ): # type: (...) -> None params = params or {} distributions = distributions or {} user_attrs = user_attrs or {} system_attrs = system_attrs or {} intermediate_values = intermediate_values or {} datetime_start = datetime_start or datetime.datetime.now() if state.is_finished(): datetime_complete = datetime_complete or datetime.datetime.now() trial = structs.FrozenTrial( number=-1, # dummy value. trial_id=-1, # dummy value. state=state, value=value, datetime_start=datetime_start, datetime_complete=datetime_complete, params=params, distributions=distributions, user_attrs=user_attrs, system_attrs=system_attrs, intermediate_values=intermediate_values, ) trial._validate() self._storage.create_new_trial(self._study_id, template_trial=trial)
def dict_to_trial(d): params = {} params_in_internal_repr = {} for k, v in d['params'].items(): distribution = distributions.json_to_distribution(v['distribution']) params[k] = distribution.to_external_repr(v['value']) params_in_internal_repr[k] = v['value'] return structs.FrozenTrial( trial_id=d['trial_id'], state=str_to_trial_state(d['state']), params=params, user_attrs=d['user_attrs'], system_attrs=d['system_attrs'], value=d['value'], intermediate_values=dict( (int(k), v) for k, v in d['intermediate_values'].items()), params_in_internal_repr=params_in_internal_repr, datetime_start=datetime.fromtimestamp(d['datetime_start']), datetime_complete=datetime.fromtimestamp(d['datetime_end']) if d['datetime_end'] else None, )
def _merge_trials_orm( self, trials, # type: List[models.TrialModel] trial_params, # type: List[models.TrialParamModel] trial_intermediate_values, # type: List[models.TrialValueModel] trial_user_attrs, # type: List[models.TrialUserAttributeModel] trial_system_attrs # type: List[models.TrialSystemAttributeModel] ): # type: (...) -> List[structs.FrozenTrial] id_to_trial = {} for trial in trials: id_to_trial[trial.trial_id] = trial id_to_params = defaultdict( list) # type: Dict[int, List[models.TrialParamModel]] for param in trial_params: id_to_params[param.trial_id].append(param) id_to_values = defaultdict( list) # type: Dict[int, List[models.TrialValueModel]] for value in trial_intermediate_values: id_to_values[value.trial_id].append(value) id_to_user_attrs = \ defaultdict(list) # type: Dict[int, List[models.TrialUserAttributeModel]] for user_attr in trial_user_attrs: id_to_user_attrs[user_attr.trial_id].append(user_attr) id_to_system_attrs = \ defaultdict(list) # type: Dict[int, List[models.TrialSystemAttributeModel]] for system_attr in trial_system_attrs: id_to_system_attrs[system_attr.trial_id].append(system_attr) result = [] for trial_id, trial in id_to_trial.items(): params = {} param_distributions = {} for param in id_to_params[trial_id]: distribution = distributions.json_to_distribution( param.distribution_json) params[param.param_name] = distribution.to_external_repr( param.param_value) param_distributions[param.param_name] = distribution intermediate_values = {} for value in id_to_values[trial_id]: intermediate_values[value.step] = value.value user_attrs = {} for user_attr in id_to_user_attrs[trial_id]: user_attrs[user_attr.key] = json.loads(user_attr.value_json) system_attrs = {} for system_attr in id_to_system_attrs[trial_id]: system_attrs[system_attr.key] = json.loads( system_attr.value_json) result.append( structs.FrozenTrial(number=trial.number, state=trial.state, params=params, distributions=param_distributions, user_attrs=user_attrs, system_attrs=system_attrs, value=trial.value, intermediate_values=intermediate_values, datetime_start=trial.datetime_start, datetime_complete=trial.datetime_complete, trial_id=trial_id)) return result
def _merge_trials_orm( self, trials, # type: List[models.TrialModel] trial_params, # type: List[models.TrialParamModel] trial_intermediate_values, # type: List[models.TrialValueModel] trial_user_attrs, # type: List[models.TrialUserAttributeModel] trial_system_attrs # type: List[models.TrialSystemAttributeModel] ): # type: (...) -> List[structs.FrozenTrial] id_to_trial = {} for trial in trials: id_to_trial[trial.trial_id] = trial id_to_params = defaultdict( list) # type: Dict[int, List[models.TrialParamModel]] for param in trial_params: id_to_params[param.trial_id].append(param) id_to_values = defaultdict( list) # type: Dict[int, List[models.TrialValueModel]] for value in trial_intermediate_values: id_to_values[value.trial_id].append(value) id_to_user_attrs = \ defaultdict(list) # type: Dict[int, List[models.TrialUserAttributeModel]] for user_attr in trial_user_attrs: id_to_user_attrs[user_attr.trial_id].append(user_attr) id_to_system_attrs = \ defaultdict(list) # type: Dict[int, List[models.TrialSystemAttributeModel]] for system_attr in trial_system_attrs: id_to_system_attrs[system_attr.trial_id].append(system_attr) temp_trials = [] for trial_id, trial in id_to_trial.items(): params = {} param_distributions = {} for param in id_to_params[trial_id]: distribution = distributions.json_to_distribution( param.distribution_json) params[param.param_name] = distribution.to_external_repr( param.param_value) param_distributions[param.param_name] = distribution intermediate_values = {} for value in id_to_values[trial_id]: intermediate_values[value.step] = value.value user_attrs = {} for user_attr in id_to_user_attrs[trial_id]: user_attrs[user_attr.key] = json.loads(user_attr.value_json) system_attrs = {} for system_attr in id_to_system_attrs[trial_id]: system_attrs[system_attr.key] = json.loads( system_attr.value_json) # `-1` is a dummy value. # It will be replaced by a proper value before returned to the caller. # # TODO(ohta): Use trial.number after TrialModel.number is added. trial_number = -1 temp_trials.append( structs.FrozenTrial(number=trial_number, state=trial.state, params=params, distributions=param_distributions, user_attrs=user_attrs, system_attrs=system_attrs, value=trial.value, intermediate_values=intermediate_values, datetime_start=trial.datetime_start, datetime_complete=trial.datetime_complete, trial_id=trial_id)) result = [] for temp_trial in temp_trials: # [NOTE] # We set actual trial numbers here to avoid calling `self.get_trial_number_from_id()` # within the above loop. # # This is because `self.get_trial_number_from_id()` may call `session.commit()` # internally, which causes unintended changes of the states of `trials`. # (see https://github.com/pfnet/optuna/pull/349#issuecomment-475086642 for details) trial_number = self.get_trial_number_from_id(temp_trial.trial_id) result.append(temp_trial._replace(number=trial_number)) return result