def test_metric_called_n_signals_times(self): metric = Mock(return_value=1.0) # noinspection PyTypeChecker hyper_score_ar(MockRegressor, self.dataset, metric, n_models=2, n_features=2) self.assertEqual(metric.call_count, self.n_signals)
def test_monitor_sequence_passed_to_transform_when_step_is_default(self): monitor = ["a", "b"] hyper_score_ar(self.regressor_class, self.dataset, self.metric, monitor=monitor) self.regressors[0].transform.assert_called() call = self.regressors[0].transform.call_args_list[0] self.assertIn("monitor", call[1]) self.assertEqual(call[1]["monitor"], monitor)
def test_progress_called(self): mock_progress = mock.MagicMock() hyper_score_ar( MockRegressor, [SimpleNamespace(y=[1, 2], usage_seq=[0, 0])], lambda x, y: 1.0, n_models=2, n_features=3, progress=mock_progress, ) mock_progress.assert_called()
def test_returns_tuple(self): res = hyper_score_ar(MockRegressor, self.dataset, lambda x, y: 1.0, n_models=2, n_features=2) self.assertEqual(len(res), 2)
def setUp(self): self.rng = np.random.default_rng(0) self.n_signals = 4 self.n_samples = 35 self.n_models = 3 self.dataset = [ SimpleNamespace( y=self.rng.normal(size=self.n_samples), usage_seq=self.rng.integers(0, self.n_models, size=self.n_samples), ) for _ in range(self.n_signals) ] self.metric = lambda x, y: 1.0 self.regressors = [Mock(n_features=4) for _ in range(self.n_signals)] for regressor in self.regressors: regressor.transform.return_value = ( self.rng.uniform(size=(self.n_samples, self.n_models)), SimpleNamespace(), ) self.regressor_class = Mock(side_effect=self.regressors) self.res = hyper_score_ar(self.regressor_class, self.dataset, self.metric)
def setUp(self): self.rng = np.random.default_rng(0) self.n_signals = 5 self.n_samples = 34 self.n_models = 4 self.ar_order = 3 self.models = self.rng.normal(size=(self.n_signals, self.n_models, self.ar_order)) self.dataset = [ SimpleNamespace( y=self.rng.normal(size=self.n_samples), usage_seq=self.rng.integers(0, self.n_models, size=self.n_samples), armas=[SimpleNamespace(a=model) for model in sig_models], ) for sig_models in self.models ] self.regressor_class = Mock(side_effect=lambda *args, **kwargs: Mock( n_features=self.ar_order, transform=Mock(return_value=( self.rng.uniform(size=(self.n_samples, self.n_models)), SimpleNamespace(), ), ), **kwargs, )) self.metric = lambda x, y: 1.0 self.res = hyper_score_ar(self.regressor_class, self.dataset, self.metric, initial_weights="oracle_ar")
def setUp(self): self.rng = np.random.default_rng(3) self.n_signals = 4 self.n_samples = 35 self.n_models = 3 self.dataset = [ SimpleNamespace( y=self.rng.normal(size=self.n_samples), usage_seq=self.rng.integers(0, self.n_models, size=self.n_samples), ) for _ in range(self.n_signals) ] self.regressors = [] for i in range(self.n_signals): regressor = Mock(n_features=4) regressor.transform.return_value = ( self.rng.uniform(size=(self.n_samples, self.n_models)), SimpleNamespace(), ) self.regressors.append(regressor) self.regressor_class = Mock(side_effect=self.regressors) self.metric_outputs = np.linspace(0, 1, self.n_signals) self.metric = Mock(side_effect=self.metric_outputs) # noinspection PyTypeChecker self.summary, self.details = hyper_score_ar(self.regressor_class, self.dataset, self.metric, test_fraction=1.0)
def test_transform_second_output_is_returned_in_details_history(self): _, details = hyper_score_ar(self.regressor_class, self.dataset, self.metric) for i, history in enumerate(details.history): for key, value in self.history_out[i].items(): self.assertTrue(hasattr(history, key)) self.assertEqual(getattr(history, key), value)
def setUp(self): self.rng = np.random.default_rng(2) self.n_signals = 4 self.n_samples = 41 self.n_models = 3 self.dataset = [ SimpleNamespace( y=self.rng.normal(size=self.n_samples), usage_seq=self.rng.integers(0, self.n_models, size=self.n_samples), ) for _ in range(self.n_signals) ] self.regressors = [] for i in range(self.n_signals): regressor = Mock(n_features=4) regressor.transform.return_value = ( np.zeros((self.n_samples, self.n_models)), SimpleNamespace(), ) self.regressors.append(regressor) self.regressor_class = Mock(side_effect=self.regressors) self.metric = lambda x, y: 1.0 self.kwargs = {"foo": "bar"} self.fit_kws = {"bar": 3} with mock.patch("bioslds.batch.transform_ar") as mock_transform_ar: mock_transform_ar.side_effect = [ _.transform.return_value for _ in self.regressors ] hyper_score_ar( self.regressor_class, self.dataset, self.metric, fit_kws=self.fit_kws, **self.kwargs, ) self.mock_transform_ar = mock_transform_ar for call in self.regressor_class.call_args_list: for key, value in self.kwargs.items(): self.assertIn(key, call[1]) self.assertEqual(call[1][key], value)
def test_appropriate_test_samples_are_passed_to_metric(self): samples = 23 # noinspection PyTypeChecker hyper_score_ar(self.regressor_class, self.dataset, self.metric, test_samples=samples) call = self.metric.call_args_list[1] usage1 = self.dataset[1].usage_seq r1 = self.regressors[1].transform.return_value[0] inferred1 = r1.argmax(axis=1) labels_true_exp = usage1[-samples:] labels_pred_exp = inferred1[-samples:] np.testing.assert_equal(call[0][0], labels_true_exp) np.testing.assert_equal(call[0][1], labels_pred_exp)
def test_r_is_special_value_in_monitor(self): _, details = hyper_score_ar( self.regressor_class, self.dataset, self.metric, monitor="r", ) for i, history in enumerate(details.history): self.assertTrue(hasattr(history, "r")) np.testing.assert_allclose(history.r, self.r[i])
def test_appropriate_test_fraction_is_passed_to_metric(self): fraction = 0.6 # noinspection PyTypeChecker hyper_score_ar(self.regressor_class, self.dataset, self.metric, test_fraction=fraction) call = self.metric.call_args_list[0] usage0 = self.dataset[0].usage_seq r0 = self.regressors[0].transform.return_value[0] inferred0 = r0.argmax(axis=1) n = int(fraction * len(r0)) labels_true_exp = usage0[-n:] labels_pred_exp = inferred0[-n:] np.testing.assert_equal(call[0][0], labels_true_exp) np.testing.assert_equal(call[0][1], labels_pred_exp)
def test_progress_trial_passed_to_transform_ra(self): regressor = Mock(n_features=4) regressor.transform.return_value = (np.zeros( (10, 2)), SimpleNamespace()) regressor_class = Mock(return_value=regressor) mock_progress = Mock() hyper_score_ar( regressor_class, [SimpleNamespace(y=[1, 2], usage_seq=[0, 0])], lambda x, y: 1.0, n_models=2, n_features=3, progress_trial=mock_progress, ) regressor.transform.assert_called() call = regressor.transform.call_args_list[0] self.assertIn("progress", call[1]) self.assertIs(call[1]["progress"], mock_progress)
def test_default_test_fraction_is_twenty_percent(self): # noinspection PyTypeChecker res_def = hyper_score_ar(self.regressor_class, self.dataset, self.metric)[0] regressor_class = Mock(side_effect=self.regressors) # noinspection PyTypeChecker res_20 = hyper_score_ar(regressor_class, self.dataset, self.metric, test_fraction=0.2)[0] regressor_class = Mock(side_effect=self.regressors) # noinspection PyTypeChecker res_50 = hyper_score_ar(regressor_class, self.dataset, self.metric, test_fraction=0.5)[0] self.assertNotEqual(res_20, res_50) self.assertEqual(res_def, res_20)
def test_r_monitor_with_nontrivial_step(self): step = 3 _, details = hyper_score_ar( self.regressor_class, self.dataset, self.metric, monitor="r", monitor_step=step, ) for i, history in enumerate(details.history): self.assertTrue(hasattr(history, "r")) np.testing.assert_allclose(history.r, self.r[i][::step])
def fct(**kwargs): crt_res = hyper_score_ar( CrosscorrelationRegressor, *common_hyper_args, nsm_rate=kwargs["rate"], xcorr_rate=1 / kwargs["exp_streak"], **common_hyper_kws, ) if economy: del crt_res[1].regressors if len(monitor) == 0: del crt_res[1].history return crt_res
def fct(**kwargs): crt_res = hyper_score_ar( CepstralRegressor, *common_hyper_args, initial_weights="oracle_ar", cepstral_order=kwargs["cepstral_order"], cepstral_kws={"rate": kwargs["rate"]}, **common_hyper_kws, ) if economy: del crt_res[1].regressors if len(monitor) == 0: del crt_res[1].history return crt_res
def test_when_monitor_step_is_not_one_an_attribute_monitor_is_created( self): monitor = ["a", "b"] step = 3 with mock.patch( "bioslds.batch.AttributeMonitor") as MockAttributeMonitor: MockAttributeMonitor.history = SimpleNamespace() hyper_score_ar( self.regressor_class, self.dataset, self.metric, monitor=monitor, monitor_step=step, ) MockAttributeMonitor.assert_called() call = MockAttributeMonitor.call_args_list[0] self.assertEqual(len(call[0]), 1) self.assertEqual(call[0][0], monitor) self.assertIn("step", call[1]) self.assertEqual(call[1]["step"], step)
def fct(**kwargs): crt_res = hyper_score_ar( make_bio_wta_with_stable_initial, *common_hyper_args, rate=kwargs["rate"], trans_mat=1 - 1 / kwargs["exp_streak"], temperature=kwargs["temperature"], error_timescale=kwargs["timescale"], **common_hyper_kws, ) if economy: del crt_res[1].regressors if len(monitor) == 0: del crt_res[1].history return crt_res
def check_regressor_calls( self, rng: Union[None, int, np.random.RandomState, np.random.Generator] ) -> Tuple[Mock, Tuple]: regressor = Mock(n_features=4) regressor.transform.return_value = ( self.rng.uniform(size=(self.n_samples, self.n_models)), SimpleNamespace(), ) regressor_class = Mock(return_value=regressor) kwargs = {} if rng is not None: kwargs["rng"] = rng res = hyper_score_ar(regressor_class, self.dataset, self.metric, **kwargs) return regressor_class, res
two_ar3.orders, dwell_times=two_ar3.dwell_times, min_dwell=two_ar3.min_dwell, fix_scale=two_ar3.fix_scale, rng=two_ar3.seed, arma_kws={"max_pole_radius": two_ar3.max_pole_radius}, ) # %% t0 = time.time() two_ar3.result_biowta = hyper_score_ar( BioWTARegressor, two_ar3.dataset, two_ar3.metric, n_models=two_ar3.n_models, n_features=two_ar3.n_features, rate=two_ar3.rate_biowta, # trans_mat=1 - 1 / two_ar3.streak_biowta, progress=tqdm, monitor=["r", "weights_", "prediction_"], ) t1 = time.time() print(f"Median accuracy score BioWTA: {two_ar3.result_biowta[0]:.2}. " f"(Took {t1 - t0:.2f} seconds.)") # %% t0 = time.time() two_ar3.result_xcorr = hyper_score_ar( CrosscorrelationRegressor, two_ar3.dataset, two_ar3.metric,
def test_transform_details_has_history_for_each_signal(self): _, details = hyper_score_ar(self.regressor_class, self.dataset, self.metric) self.assertTrue(hasattr(details, "history")) self.assertEqual(len(details.history), self.n_signals)