def testNoiselessMerge(self): # One noiseless obsd = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 1.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 0.0]]), ) obsd2 = ivw_metric_merge(obsd) np.array_equal(obsd2.means, np.array([1.0, 1.0])) cov_true = np.array([[1.0, 0.4], [0.4, 0.0]]) self.assertTrue(np.array_equal(obsd2.covariance, cov_true)) # Conflicting noiseless, default (warn) obsd = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 1.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 0.0, 0.8], [0.4, 0.8, 0.0]]), ) with self.assertRaises(ValueError): obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="wrong") obsd2 = ivw_metric_merge(obsd) self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 2.0]))) cov_true = np.array([[1.0, 0.2], [0.2, 0.0]]) self.assertTrue(np.array_equal(obsd2.covariance, cov_true)) # Conflicting noiseless, raise with self.assertRaises(ValueError): obsd2 = ivw_metric_merge(obsd, conflicting_noiseless="raise")
def testNoRepeats(self): obsd = ObservationData( metric_names=["m1", "m2"], means=np.array([1.0, 2.0]), covariance=np.array([[1.0, 0.2], [0.2, 2.0]]), ) obsd2 = ivw_metric_merge(obsd) self.assertEqual(obsd2, obsd)
def transform_optimization_config( self, optimization_config: OptimizationConfig, modelbridge: Optional["modelbridge_module.base.ModelBridge"], fixed_features: ObservationFeatures, ) -> OptimizationConfig: use_raw_sq = self.config.get("use_raw_status_quo", False) has_relative_constraint = any( c.relative for c in optimization_config.all_constraints ) if not has_relative_constraint: return optimization_config # Else, we have at least one relative constraint. # Estimate the value at the status quo. if modelbridge is None: raise ValueError("ModelBridge not supplied to transform.") if modelbridge.status_quo is None: raise ValueError( "Optimization config has relative constraint, but model was " "not fit with status quo." ) try: # pyre-fixme[16]: `Optional` has no attribute `features`. f, _ = modelbridge.predict([modelbridge.status_quo.features]) except Exception: # Check if it is out-of-design. if use_raw_sq or not modelbridge.model_space.check_membership( modelbridge.status_quo.features.parameters ): # Out-of-design: use the raw observation sq_data = ivw_metric_merge( # pyre-fixme[16]: `Optional` has no attribute `data`. obsd=modelbridge.status_quo.data, conflicting_noiseless="raise", ) f, _ = unwrap_observation_data([sq_data]) else: # Should have worked. raise # pragma: no cover # Plug in the status quo value to each relative constraint. for c in optimization_config.all_constraints: if c.relative: if isinstance(c, ScalarizedOutcomeConstraint): # The sq_val of scalarized outcome is the weighted # sum of its component metrics sq_val = np.sum( [ c.weights[i] * f[metric.name][0] for i, metric in enumerate(c.metrics) ] ) else: sq_val = f[c.metric.name][0] c.bound = (1 + c.bound / 100.0) * sq_val c.relative = False return optimization_config
def testMerge(self): obsd = ObservationData( metric_names=["m1", "m2", "m2"], means=np.array([1.0, 2.0, 1.0]), covariance=np.array([[1.0, 0.2, 0.4], [0.2, 2.0, 0.8], [0.4, 0.8, 3.0]]), ) obsd2 = ivw_metric_merge(obsd) self.assertEqual(obsd2.metric_names, ["m1", "m2"]) self.assertTrue(np.array_equal(obsd2.means, np.array([1.0, 0.6 * 2 + 0.4]))) cov12 = 0.2 * 0.6 + 0.4 * 0.4 # var(w1*y1 + w2*y2) = # w1 ** 2 * var(y1) + w2 ** 2 * var(y2) + 2 * w1 * w2 * cov(y1, y2) cov22 = 0.6 ** 2 * 2.0 + 0.4 ** 2 * 3 + 2 * 0.6 * 0.4 * 0.8 cov_true = np.array([[1.0, cov12], [cov12, cov22]]) discrep = np.max(np.abs(obsd2.covariance - cov_true)) self.assertTrue(discrep < 1e-8)
def transform_optimization_config( self, optimization_config: OptimizationConfig, modelbridge: Optional["modelbridge_module.base.ModelBridge"], fixed_features: ObservationFeatures, ) -> OptimizationConfig: has_relative_constraint = any( c.relative for c in optimization_config.outcome_constraints ) if not has_relative_constraint: return optimization_config # Else, we have at least one relative constraint. # Estimate the value at the status quo. if modelbridge is None: raise ValueError("ModelBridge not supplied to transform.") if modelbridge.status_quo is None: raise ValueError( "Optimization config has relative constraint, but model was " "not fit with status quo." ) try: # pyre-fixme[16]: `Optional` has no attribute `features`. f, _ = modelbridge.predict([modelbridge.status_quo.features]) except Exception: # Check if it is out-of-design. if not modelbridge.model_space.check_membership( modelbridge.status_quo.features.parameters ): # Out-of-design: use the raw observation sq_data = ivw_metric_merge( # pyre-fixme[16]: `Optional` has no attribute `data`. obsd=modelbridge.status_quo.data, conflicting_noiseless="raise", ) f, _ = unwrap_observation_data([sq_data]) else: # Should have worked. raise # pragma: no cover # Plug in the status quo value to each relative constraint. for c in optimization_config.outcome_constraints: if c.relative: # Compute new bound. c.bound = (1 + c.bound / 100.0) * f[c.metric.name][0] c.relative = False return optimization_config