Beispiel #1
0
 def testTransform(self):
     obsd1_0 = ObservationData(
         metric_names=["m1", "m2", "m2"],
         means=np.array([1.0, 2.0, 1.0]),
         covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8],
                              [0.4, 0.8, 3.0]]),
     )
     obsd1_1 = ObservationData(
         metric_names=["m1", "m1", "m2", "m2"],
         means=np.array([1.0, 1.0, 2.0, 1.0]),
         covariance=np.array([
             [1.0, 0.0, 0.0, 0.0],
             [0.0, 1.0, 0.2, 0.4],
             [0.0, 0.2, 2.0, 0.8],
             [0.0, 0.4, 0.8, 3.0],
         ]),
     )
     obsd2_0 = ObservationData(
         metric_names=["m1", "m2"],
         means=np.array([1.0, 1.6]),
         covariance=np.array([[1.0, 0.28], [0.28, 1.584]]),
     )
     obsd2_1 = ObservationData(
         metric_names=["m1", "m2"],
         means=np.array([1.0, 1.6]),
         covariance=np.array([[0.5, 0.14], [0.14, 1.584]]),
     )
     observation_data = [obsd1_0, obsd1_1]
     t = IVW(None, None, None)
     observation_data2 = t.transform_observation_data(observation_data, [])
     observation_data2_true = [obsd2_0, obsd2_1]
     for i, obsd in enumerate(observation_data2_true):
         self.assertEqual(observation_data2[i].metric_names,
                          obsd.metric_names)
         self.assertTrue(
             np.array_equal(observation_data2[i].means, obsd.means))
         discrep = np.max(
             np.abs(observation_data2[i].covariance - obsd.covariance))
         self.assertTrue(discrep < 1e-8)
Beispiel #2
0
def _get_in_sample_arms(
    model: ModelBridge,
    metric_names: Set[str],
    fixed_features: Optional[ObservationFeatures] = None,
) -> Tuple[Dict[str, PlotInSampleArm], RawData, Dict[str, TParameterization]]:
    """Get in-sample arms from a model with observed and predicted values
    for specified metrics.

    Returns a PlotInSampleArm object in which repeated observations are merged
    with IVW, and a RawData object in which every observation is listed.

    Fixed features input can be used to override fields of the insample arms
    when making model predictions.

    Args:
        model: An instance of the model bridge.
        metric_names: Restrict predictions to these metrics. If None, uses all
            metrics in the model.
        fixed_features: Features that should be fixed in the arms this function
            will obtain predictions for.

    Returns:
        A tuple containing

        - Map from arm name to PlotInSampleArm.
        - List of the data for each observation like::

            {'metric_name': 'likes', 'arm_name': '0_0', 'mean': 1., 'sem': 0.1}

        - Map from arm name to parameters
    """
    observations = model.get_training_data()
    # Calculate raw data
    raw_data = []
    arm_name_to_parameters = {}
    for obs in observations:
        arm_name_to_parameters[obs.arm_name] = obs.features.parameters
        for j, metric_name in enumerate(obs.data.metric_names):
            if metric_name in metric_names:
                raw_data.append({
                    "metric_name": metric_name,
                    "arm_name": obs.arm_name,
                    "mean": obs.data.means[j],
                    "sem": np.sqrt(obs.data.covariance[j, j]),
                })

    # Check that we have one ObservationFeatures per arm name since we
    # key by arm name and the model is not Multi-task.
    # If "TrialAsTask" is present, one of the arms is also chosen.
    if ("TrialAsTask" not in model.transforms.keys()) and (
            len(arm_name_to_parameters) != len(observations)):
        logger.error(
            "Have observations of arms with different features but same"
            " name. Arbitrary one will be plotted.")

    # Merge multiple measurements within each Observation with IVW to get
    # un-modeled prediction
    t = IVW(None, [], [])
    obs_data = t.transform_observation_data([obs.data for obs in observations],
                                            [])
    # Start filling in plot data
    in_sample_plot: Dict[str, PlotInSampleArm] = {}
    for i, obs in enumerate(observations):
        if obs.arm_name is None:
            raise ValueError("Observation must have arm name for plotting.")

        # Extract raw measurement
        obs_y = {}  # Observed metric means.
        obs_se = {}  # Observed metric standard errors.
        # Use the IVW data, not obs.data
        for j, metric_name in enumerate(obs_data[i].metric_names):
            if metric_name in metric_names:
                obs_y[metric_name] = obs_data[i].means[j]
                obs_se[metric_name] = np.sqrt(obs_data[i].covariance[j, j])
        # Make a prediction.
        if model.training_in_design[i]:
            features = obs.features
            if fixed_features is not None:
                features.update_features(fixed_features)
            pred_y, pred_se = _predict_at_point(model, features, metric_names)
        else:
            # Use raw data for out-of-design points
            pred_y = obs_y
            pred_se = obs_se
        in_sample_plot[not_none(obs.arm_name)] = PlotInSampleArm(
            name=not_none(obs.arm_name),
            y=obs_y,
            se=obs_se,
            parameters=obs.features.parameters,
            y_hat=pred_y,
            se_hat=pred_se,
            context_stratum=None,
        )
    return in_sample_plot, raw_data, arm_name_to_parameters
Beispiel #3
0
def _get_in_sample_arms(
    model: ModelBridge, metric_names: Set[str]
) -> Tuple[Dict[str, PlotInSampleArm], RawData, Dict[str, TParameterization]]:
    """Get in-sample arms from a model with observed and predicted values
    for specified metrics.

    Returns a PlotInSampleArm object in which repeated observations are merged
    with IVW, and a RawData object in which every observation is listed.

    Args:
        model: An instance of the model bridge.
        metric_names: Restrict predictions to these metrics. If None, uses all
            metrics in the model.

    Returns:
        A tuple containing

        - Map from arm name to PlotInSampleArm.
        - List of the data for each observation like::

            {'metric_name': 'likes', 'arm_name': '0_0', 'mean': 1., 'sem': 0.1}

        - Map from arm name to parameters
    """
    observations = model.get_training_data()
    # Calculate raw data
    raw_data = []
    cond_name_to_parameters = {}
    for obs in observations:
        cond_name_to_parameters[obs.arm_name] = obs.features.parameters
        for j, metric_name in enumerate(obs.data.metric_names):
            if metric_name in metric_names:
                raw_data.append({
                    "metric_name": metric_name,
                    "arm_name": obs.arm_name,
                    "mean": obs.data.means[j],
                    "sem": np.sqrt(obs.data.covariance[j, j]),
                })
    # Check that we have one ObservationFeatures per arm name since we
    # key by arm name.
    if len(cond_name_to_parameters) != len(observations):
        logger.error(
            "Have observations of arms with different features but same"
            " name. Arbitrary one will be plotted.")
    # Merge multiple measurements within each Observation with IVW to get
    # un-modeled prediction
    t = IVW(None, [], [])
    obs_data = t.transform_observation_data([obs.data for obs in observations],
                                            [])
    # Start filling in plot data
    in_sample_plot: Dict[str, PlotInSampleArm] = {}
    for i, obs in enumerate(observations):
        if obs.arm_name is None:
            raise ValueError("Observation must have arm name for plotting.")

        # Extract raw measurement
        obs_y = {}
        obs_se = {}
        # Use the IVW data, not obs.data
        for j, metric_name in enumerate(obs_data[i].metric_names):
            if metric_name in metric_names:
                obs_y[metric_name] = obs_data[i].means[j]
                obs_se[metric_name] = np.sqrt(obs_data[i].covariance[j, j])
        # Make a prediction.
        if model.training_in_design[i]:
            pred_y, pred_se = _predict_at_point(model, obs.features,
                                                metric_names)
        else:
            # Use raw data for out-of-design points
            pred_y = obs_y
            pred_se = obs_se
        in_sample_plot[obs.arm_name] = PlotInSampleArm(
            name=obs.arm_name,
            y=obs_y,
            se=obs_se,
            parameters=obs.features.parameters,
            y_hat=pred_y,
            se_hat=pred_se,
            context_stratum=None,
        )
    return in_sample_plot, raw_data, cond_name_to_parameters