def _create_child_runs_for_parameter_search(parent_estimator, parent_model, parent_run, child_tags): from itertools import zip_longest client = MlflowClient() # Use the start time of the parent parameter search run as a rough estimate for the # start time of child runs, since we cannot precisely determine when each point # in the parameter search space was explored child_run_start_time = parent_run.info.start_time child_run_end_time = int(time.time() * 1000) estimator_param_maps = parent_estimator.getEstimatorParamMaps() tuned_estimator = parent_estimator.getEstimator() metrics_dict, _ = _get_param_search_metrics_and_best_index( parent_estimator, parent_model) for i in range(len(estimator_param_maps)): child_estimator = tuned_estimator.copy(estimator_param_maps[i]) tags_to_log = dict(child_tags) if child_tags else {} tags_to_log.update({MLFLOW_PARENT_RUN_ID: parent_run.info.run_id}) tags_to_log.update(_get_estimator_info_tags(child_estimator)) child_run = client.create_run( experiment_id=parent_run.info.experiment_id, start_time=child_run_start_time, tags=tags_to_log, ) params_to_log = _get_instance_param_map( child_estimator, parent_estimator._autologging_metadata.uid_to_indexed_name_map) param_batches_to_log = _chunk_dict( params_to_log, chunk_size=MAX_PARAMS_TAGS_PER_BATCH) metrics_to_log = {k: v[i] for k, v in metrics_dict.items()} for params_batch, metrics_batch in zip_longest(param_batches_to_log, [metrics_to_log], fillvalue={}): # Trim any parameter keys / values and metric keys that exceed the limits # imposed by corresponding MLflow Tracking APIs (e.g., LogParam, LogMetric) truncated_params_batch = _truncate_dict(params_batch, MAX_ENTITY_KEY_LENGTH, MAX_PARAM_VAL_LENGTH) truncated_metrics_batch = _truncate_dict( metrics_batch, max_key_length=MAX_ENTITY_KEY_LENGTH) client.log_batch( run_id=child_run.info.run_id, params=[ Param(str(key), str(value)) for key, value in truncated_params_batch.items() ], metrics=[ Metric(key=str(key), value=value, timestamp=child_run_end_time, step=0) for key, value in truncated_metrics_batch.items() ], ) client.set_terminated(run_id=child_run.info.run_id, end_time=child_run_end_time)
def _create_child_runs_for_parameter_search(cv_estimator, parent_run, child_tags=None): """ Creates a collection of child runs for a parameter search training session. Runs are reconstructed from the `cv_results_` attribute of the specified trained parameter search estimator - `cv_estimator`, which provides relevant performance metrics for each point in the parameter search space. One child run is created for each point in the parameter search space. For additional information, see `https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.GridSearchCV.html`_. # noqa: E501 :param cv_estimator: The trained parameter search estimator for which to create child runs. :param parent_run: A py:class:`mlflow.entities.Run` object referring to the parent parameter search run for which child runs should be created. :param child_tags: An optional dictionary of MLflow tag keys and values to log for each child run. """ import pandas as pd client = MlflowClient() # Use the start time of the parent parameter search run as a rough estimate for the # start time of child runs, since we cannot precisely determine when each point # in the parameter search space was explored child_run_start_time = parent_run.info.start_time child_run_end_time = int(time.time() * 1000) seed_estimator = cv_estimator.estimator # In the unlikely case that a seed of a parameter search estimator is, # itself, a parameter search estimator, we should avoid logging the untuned # parameters of the seeds's seed estimator should_log_params_deeply = not _is_parameter_search_estimator( seed_estimator) # Each row of `cv_results_` only provides parameters that vary across # the user-specified parameter grid. In order to log the complete set # of parameters for each child run, we fetch the parameters defined by # the seed estimator and update them with parameter subset specified # in the result row base_params = seed_estimator.get_params(deep=should_log_params_deeply) cv_results_df = pd.DataFrame.from_dict(cv_estimator.cv_results_) for _, result_row in cv_results_df.iterrows(): tags_to_log = dict(child_tags) if child_tags else {} tags_to_log.update({MLFLOW_PARENT_RUN_ID: parent_run.info.run_id}) tags_to_log.update(_get_estimator_info_tags(seed_estimator)) child_run = client.create_run( experiment_id=parent_run.info.experiment_id, start_time=child_run_start_time, tags=tags_to_log, ) from itertools import zip_longest params_to_log = dict(base_params) params_to_log.update(result_row.get("params", {})) param_batches_to_log = _chunk_dict( params_to_log, chunk_size=MAX_PARAMS_TAGS_PER_BATCH) # Parameters values are recorded twice in the set of search `cv_results_`: # once within a `params` column with dictionary values and once within # a separate dataframe column that is created for each parameter. To prevent # duplication of parameters, we log the consolidated values from the parameter # dictionary column and filter out the other parameter-specific columns with # names of the form `param_{param_name}`. Additionally, `cv_results_` produces # metrics for each training split, which is fairly verbose; accordingly, we filter # out per-split metrics in favor of aggregate metrics (mean, std, etc.) excluded_metric_prefixes = ["param", "split"] metric_batches_to_log = _chunk_dict( { key: value for key, value in result_row.iteritems() if not any([ key.startswith(prefix) for prefix in excluded_metric_prefixes ]) and isinstance(value, Number) }, chunk_size=min(MAX_ENTITIES_PER_BATCH - MAX_PARAMS_TAGS_PER_BATCH, MAX_METRICS_PER_BATCH), ) for params_batch, metrics_batch in zip_longest(param_batches_to_log, metric_batches_to_log, fillvalue={}): # Trim any parameter keys / values and metric keys that exceed the limits # imposed by corresponding MLflow Tracking APIs (e.g., LogParam, LogMetric) truncated_params_batch = _truncate_dict(params_batch, MAX_ENTITY_KEY_LENGTH, MAX_PARAM_VAL_LENGTH) truncated_metrics_batch = _truncate_dict( metrics_batch, max_key_length=MAX_ENTITY_KEY_LENGTH) client.log_batch( run_id=child_run.info.run_id, params=[ Param(str(key), str(value)) for key, value in truncated_params_batch.items() ], metrics=[ Metric(key=str(key), value=value, timestamp=child_run_end_time, step=0) for key, value in truncated_metrics_batch.items() ], ) client.set_terminated(run_id=child_run.info.run_id, end_time=child_run_end_time)
class BatchMetricsLogger: """ The BatchMetricsLogger will log metrics in batch against an mlflow run. If run_id is passed to to constructor then all recording and logging will happen against that run_id. If no run_id is passed into constructor, then the run ID will be fetched from `mlflow.active_run()` each time `record_metrics()` or `flush()` is called; in this case, callers must ensure that an active run is present before invoking `record_metrics()` or `flush()`. """ def __init__(self, run_id=None, tracking_uri=None): self.run_id = run_id self.client = MlflowClient(tracking_uri) # data is an array of Metric objects self.data = [] self.total_training_time = 0 self.total_log_batch_time = 0 self.previous_training_timestamp = None def flush(self): """ The metrics accumulated by BatchMetricsLogger will be batch logged to an MLFlow run. """ self._timed_log_batch() self.data = [] def _timed_log_batch(self): if self.run_id is None: # Retrieving run_id from active mlflow run. current_run_id = mlflow.active_run().info.run_id else: current_run_id = self.run_id start = time.time() metrics_slices = [ self.data[i:i + MAX_METRICS_PER_BATCH] for i in range(0, len(self.data), MAX_METRICS_PER_BATCH) ] for metrics_slice in metrics_slices: self.client.log_batch(run_id=current_run_id, metrics=metrics_slice) end = time.time() self.total_log_batch_time += end - start def _should_flush(self): target_training_to_logging_time_ratio = 10 if (self.total_training_time >= self.total_log_batch_time * target_training_to_logging_time_ratio): return True return False def record_metrics(self, metrics, step=None): """ Submit a set of metrics to be logged. The metrics may not be immediately logged, as this class will batch them in order to not increase execution time too much by logging frequently. :param metrics: dictionary containing key, value pairs of metrics to be logged. :param step: the training step that the metrics correspond to. """ current_timestamp = time.time() if self.previous_training_timestamp is None: self.previous_training_timestamp = current_timestamp training_time = current_timestamp - self.previous_training_timestamp self.total_training_time += training_time # log_batch() requires step to be defined. Therefore will set step to 0 if not defined. if step is None: step = 0 for key, value in metrics.items(): self.data.append( Metric(key, value, int(current_timestamp * 1000), step)) if self._should_flush(): self.flush() self.previous_training_timestamp = current_timestamp