def run(self, control_samples, variation_samples, alpha=DEFAULT_ALPHA, inference_kwargs=None): """ Run the inference procedure over the samples with a selected alpha value. alpha : float in [0, 1] the assumed Type I error rate """ if isinstance(control_samples, (list, ndarray)): control_samples = Samples(control_samples) if isinstance(variation_samples, (list, ndarray)): variation_samples = Samples(variation_samples) self.alpha = alpha nobs = min(control_samples.nobs, variation_samples.nobs) test_statistic = 'z' if nobs > MIN_OBS_FOR_Z else 't' self.comparison = MeanComparison(samples_a=variation_samples, samples_b=control_samples, alpha=self.alpha, test_statistic=test_statistic, hypothesis=self.hypothesis)
def run(self, control_samples, variation_samples, alpha=DEFAULT_ALPHA, inference_kwargs=None): """ Run the inference procedure over the samples with a selected alpha value. alpha : float in [0, 1] the assumed Type I error rate """ if not isinstance(control_samples, Samples): control_samples = Samples(control_samples) if not isinstance(variation_samples, Samples): variation_samples = Samples(variation_samples) self.alpha = alpha self.comparison = BootstrapStatisticComparison( samples_a=variation_samples, samples_b=control_samples, alpha=self.alpha, hypothesis=self.hypothesis, statistic_function=self.statistic_function)
def run_test( self, test, alpha=DEFAULT_ALPHA, correction_method=None, display_results=False, visualize_results=False, inference_kwargs=None ): """ Given a HypothesisTest, run the test and return the results. Parameters ---------- test : HypothesisTest A hypothesis test object alpha : float The Type I error assumed by the experimenter when running the test correction_method: str Correction method used when running test (if any) display_results : boolean Whether to print the test results to stdout visualize_results : boolean Whether to render a visual representation of the results Returns ------- test_results: an instance of a HypothesisTestResults or sublass The results of the statistical test. """ control_obs = test.filter_variations(self.dataset, test.control, self.ds.treatment) control_obs = test.filter_segments(control_obs) control_obs = test.filter_metrics(control_obs) control_samples = Samples(control_obs, name=test.control) variation_obs = test.filter_variations(self.dataset, test.variation, self.ds.treatment) variation_obs = test.filter_segments(variation_obs) variation_obs = test.filter_metrics(variation_obs) variation_samples = Samples(variation_obs, name=test.variation) test_results = test.run( control_samples, variation_samples, alpha=alpha, inference_kwargs=inference_kwargs ) test_results.correction_method = correction_method if display_results: test_results.display() if visualize_results: test_results.visualize() return test_results
def __init__(self, traces): self.variables = [] for k, v in list(traces.items()): if k != "lp__": self.variables.append(k) setattr(self, k, Samples(v)) self.summarize()
def run(self, control_samples, variation_samples, alpha=DEFAULT_ALPHA, inference_kwargs=None): """ Run the inference procedure over the samples with a selected alpha value. alpha : float in [0, 1] the assumed Type I error rate """ if isinstance(control_samples, (list, np.ndarray)): control_samples = Samples(control_samples) if isinstance(variation_samples, (list, np.ndarray)): variation_samples = Samples(variation_samples) self.alpha = alpha self.comparison = RateComparison(samples_a=variation_samples, samples_b=control_samples, alpha=self.alpha, hypothesis=self.hypothesis)
def run(self, control_samples, variation_samples, alpha=DEFAULT_ALPHA, inference_kwargs=None): """ Run the inference procedure over the samples using a particular inference method. control_samples: instance of Samples the samples for the control condition variation_smaples: instance of Samples the samples for the varitation condition alpha: float One minus the credible mass under the posterior, used to calculate the credible interval. Roughly analogous to the alpha in frequentist tests. inference_kwargs : dict Additional kwargs that used by the Stan sampling or variational inference methods. For details, see: - http://mc-stan.org/rstan/reference/stanmodel-method-sampling.html - http://mc-stan.org/rstan/reference/stanmodel-method-vb.html """ if isinstance(control_samples, (list, np.ndarray)): control_samples = Samples(control_samples) if isinstance(variation_samples, (list, np.ndarray)): variation_samples = Samples(variation_samples) self.alpha = alpha self.credible_mass = 1 - alpha self.control = control_samples self.variation = variation_samples inference_data, hyperparameters = self.inference_inputs( control_samples, variation_samples) self.hyperparameters = { k: v for k, v in list(hyperparameters.items()) if k != 'recompile' } # check for `inference_method` in inference_kwargs if inference_kwargs is not None: if 'inference_method' in inference_kwargs: inference_method = inference_kwargs['inference_method'] del inference_kwargs['inference_method'] else: inference_method = DEFAULT_BAYESIAN_INFERENCE_METHOD else: inference_method = DEFAULT_BAYESIAN_INFERENCE_METHOD self.inference_method = inference_method # run inference if inference_method[0] == "s": # Gibs sampling if inference_kwargs is None: inference_kwargs = SAMPLING_DEFAULT_INFERENCE_KWARGS inference_results = self.bm.sampling(data=inference_data, **inference_kwargs) elif inference_method[0] == "v": # variational bayes if inference_kwargs is None: inference_kwargs = VB_DEFAULT_INFERENCE_KWARGS inference_results = self.bm.vb(data=inference_data, **inference_kwargs) else: raise ValueError( 'Unknown inference procedure {!r}'.format(inference_method)) self.traces = Traces(self.extract_traces(inference_results)) return self.traces