def test_passes_additional_kwargs_to_regressor_transform(self): regressor = mock.Mock(n_features=3) kwargs = {"foo": 3, "bar": 5} transform_ar(regressor, [], **kwargs) call_kwargs = regressor.transform.call_args[1] for key in kwargs: self.assertIn(key, call_kwargs) self.assertEqual(kwargs[key], call_kwargs[key])
def test_size_of_predictor_variable_correct_when_bias_is_true(self): p = 5 regressor = mock.Mock(n_features=p + 1) n = 20 a = n * [0] transform_ar(regressor, a, bias=True) call_args = regressor.transform.call_args[0] self.assertEqual(np.shape(call_args[0])[1], p + 1)
def test_args_sent_to_regressor_transform_have_the_correct_length(self): p = 4 regressor = mock.Mock(n_features=p) n = 52 a = n * [0] transform_ar(regressor, a) call_args = regressor.transform.call_args[0] self.assertEqual(len(call_args[0]), n - p) self.assertEqual(len(call_args[1]), n - p)
def test_y_values_passed_to_regressor_infer_fit_correct_when_bias_is_false(self): rng = np.random.default_rng(1) n = 57 y = rng.normal(size=n) p = 4 regressor = mock.Mock(n_features=p) transform_ar(regressor, y, bias=False) call_args = regressor.transform.call_args[0] np.testing.assert_allclose(call_args[1], y[p:])
def test_bias_false_by_default(self): p = 4 regressor1 = mock.Mock(n_features=p) regressor2 = mock.Mock(n_features=p) n = 10 a = n * [0] transform_ar(regressor1, a, bias=False) transform_ar(regressor2, a) call_args1 = regressor1.transform.call_args[0] call_args2 = regressor2.transform.call_args[0] self.assertEqual(np.shape(call_args1[0]), np.shape(call_args2[0]))
def test_returns_regressor_transform_output(self): regressor = mock.Mock(n_features=3) ret = (1, {"a": 3}) regressor.transform.return_value = ret actual_ret = transform_ar(regressor, []) self.assertEqual(actual_ret, ret)
def test_X_values_passed_to_regressor_infer_fit_correct_when_bias_is_false(self): rng = np.random.default_rng(1) n = 57 y = rng.normal(size=n) p = 4 regressor = mock.Mock(n_features=p) transform_ar(regressor, y, bias=False) call_args = regressor.transform.call_args[0] expected_X = np.zeros((len(y) - p, p)) for i in range(len(y) - p): for j in range(p): expected_X[i, j] = y[i - j + p - 1] np.testing.assert_allclose(call_args[0], expected_X)
def test_X_values_augmented_by_constant_one_when_bias_is_true(self): rng = np.random.default_rng(2) n = 23 y = rng.normal(size=n) p = 5 regressor1 = mock.Mock(n_features=p) regressor2 = mock.Mock(n_features=p + 1) transform_ar(regressor1, y, bias=False) transform_ar(regressor2, y, bias=True) call_args1 = regressor1.transform.call_args[0] call_args2 = regressor2.transform.call_args[0] X = call_args1[0] X_bias = call_args2[0] X_bias_exp = np.hstack((X, np.ones((len(X), 1)))) np.testing.assert_allclose(X_bias, X_bias_exp)
def hyper_score_ar( regressor_class: Callable, dataset: Sequence, metric: Callable[[np.ndarray, np.ndarray], float], rng: Union[int, np.random.RandomState, np.random.Generator] = 0, fit_kws: Optional[dict] = None, initial_weights: str = "default", monitor: Optional[Sequence] = None, monitor_step: int = 1, progress: Optional[Callable] = None, progress_trial: Optional[Callable] = None, test_fraction: float = 0.2, test_samples: Optional[int] = None, **kwargs, ) -> Tuple[float, SimpleNamespace]: """ Score hyperparameter choice for AR clustering. This runs a time series clustering algorithm on a set of signals and assesses the accuracy of the output using a given metric. This is done using a given set of hyperparameter values for the algorithm. The function then returns a summary score. Parameters ---------- regressor_class Class to use to create regressors. One regressor is created for each signal in the dataset, and `transform_ar` is used to fit the regressor on each signal. Additional keyword arguments passed to `hyper_score_ar` are transferred to the `regressor_class` constructor -- you'll probably need at least `n_models` and `n_features`. A random number generator is chosen automatically by `hyper_score_ar` and passed as the `rng` argument to the constructor -- note that this will not pass `hyper_score_ar`s `rng` directly! dataset Sequence of signals to use for testing the regressors. Each entry in `dataset` should be an object containing at least fields `y` and `usage_seq`, which should be arrays of equal length. `y` is the signal, `usage_seq` gives the ground truth of which generating model was used at each time step. metric The function to use to assess the accuracy of the clustering. This should take a vector `labels_true` of ground-truth latent states at each time step and a vector `labels_pred` of predicted (inferred) latent states, and return a scalar accuracy score. rng Random number generator or seed to use for generating initial weight values. If seed, a random number generator is created using `np.random.default_rng`. fit_kws Additional argument to pass to `transform_ar`. initial_weights How to set the initial weights (for regressors that have that option). This can be "default": do the default, i.e., no `weights` attribute passed to __init__ "oracle_ar": pass the `a` coefficients from the signal's `armas` member as `weights`. monitor Sequence of strings denoting values to monitor during the inference procedure for each regressor and signal pair. The special value "r" can be used to also store continuous latent-state assignments that are returned as the first output of `transform` / `transform_ar`. monitor_step How often to record the values from `monitor`. progress Progress callable for monitoring how far the function is in running the regressor on each signal from the `dataset`. This is used to wrap an iterator, `progress(iterator) -> iterator`. progress_trial Callable for monitoring the progress during each trial. This is directly passed to `transform_ar` (and thus to `regressor_class().transform`). test_fraction Fraction of samples to use when estimating the clustering score. test_samples Number of samples to use when estimating the clustering score. If this is provided, it overrides `test_fraction` Additional keyword arguments are directly passed to the `regressor_class` constructor. Returns a tuple `(summary_score, details)`, where `summary_score` is the median of the scores assigned for the regressor output on each signal in the `dataset`, and `details` is a `SimpleNamespace` containing more details from the run: trial_scores (np.ndarray) -- The scores for each trial. regressor_seeds (np.ndarray) -- Random seeds used to initialize regressors. regressors (Sequence) -- Regressors used for each trial. history (Sequence) -- Monitoring data for each trial. """ # handle seed form of rng if not hasattr(rng, "normal"): rng = np.random.default_rng(rng) # handle optional fit_kws if fit_kws is None: fit_kws = {} else: fit_kws = copy.copy(fit_kws) fit_kws.setdefault("progress", progress_trial) # handle optional monitor if monitor is None: monitor = [] # handle monitoring of transform output if "r" in monitor: monitor = [_ for _ in monitor if _ != "r"] store_r = True else: store_r = False # ensure monitor_step makes sense monitor_step = max(monitor_step, 1) # set up trial scores n_trials = len(dataset) trial_scores = np.zeros(n_trials) # set up regressor random seeds if hasattr(rng, "randint"): gen_integers = rng.randint else: gen_integers = rng.integers regressor_seeds = gen_integers(0, sys.maxsize, size=n_trials) # run the simulations it = dataset if progress is not None: it = progress(it) history = [] regressors = [] for i, signal in enumerate(it): # create the regressor crt_args = copy.copy(kwargs) crt_seed = regressor_seeds[i] crt_rng = np.random.default_rng(crt_seed) crt_args["rng"] = crt_rng if initial_weights == "oracle_ar" and not hasattr(crt_args, "weights"): crt_args["weights"] = np.asarray([_.a for _ in signal.armas]) regressor = regressor_class(**crt_args) # run transform_ar with this regressor crt_monitor = monitor if len(crt_monitor) > 0 and monitor_step > 1: crt_monitor = AttributeMonitor(monitor, step=monitor_step) crt_r, crt_history = transform_ar(regressor, signal.y, monitor=crt_monitor, **fit_kws) if store_r: crt_history.r = crt_r[::monitor_step] history.append(crt_history) crt_inferred_usage = np.argmax(crt_r, axis=1) # score only the last test_fraction / test_samples samples if test_samples is None: crt_n = int(test_fraction * len(crt_r)) else: crt_n = test_samples crt_score = metric(signal.usage_seq[-crt_n:], crt_inferred_usage[-crt_n:]) regressors.append(regressor) trial_scores[i] = crt_score # noinspection PyTypeChecker summary_score: float = np.median(trial_scores) details = SimpleNamespace( trial_scores=trial_scores, regressor_seeds=regressor_seeds, regressors=regressors, history=history, ) return summary_score, details
def test_calls_regressor_transform_once(self): regressor = mock.Mock(n_features=3) transform_ar(regressor, []) regressor.transform.assert_called_once()
def test_calls_regressor_transform_with_two_positional_args(self): regressor = mock.Mock(n_features=3) transform_ar(regressor, []) self.assertEqual(len(regressor.transform.call_args[0]), 2)
def test_does_not_pass_bias_to_regressor_transform(self): regressor = mock.Mock(n_features=3) transform_ar(regressor, [], bias=True) call_args = regressor.transform.call_args self.assertNotIn("bias", call_args[1])