def run(self,
control_samples,
variation_samples,
alpha=DEFAULT_ALPHA,
inference_kwargs=None):
"""
Run the inference procedure over the samples with a selected alpha
value.
alpha : float in [0, 1]
the assumed Type I error rate
"""
if isinstance(control_samples, (list, ndarray)):
control_samples = Samples(control_samples)
if isinstance(variation_samples, (list, ndarray)):
variation_samples = Samples(variation_samples)
self.alpha = alpha
nobs = min(control_samples.nobs, variation_samples.nobs)
test_statistic = 'z' if nobs > MIN_OBS_FOR_Z else 't'
self.comparison = MeanComparison(samples_a=variation_samples,
samples_b=control_samples,
alpha=self.alpha,
test_statistic=test_statistic,
hypothesis=self.hypothesis)
def run(self,
control_samples,
variation_samples,
alpha=DEFAULT_ALPHA,
inference_kwargs=None):
"""
Run the inference procedure over the samples with a selected alpha
value.
alpha : float in [0, 1]
the assumed Type I error rate
"""
if not isinstance(control_samples, Samples):
control_samples = Samples(control_samples)
if not isinstance(variation_samples, Samples):
variation_samples = Samples(variation_samples)
self.alpha = alpha
self.comparison = BootstrapStatisticComparison(
samples_a=variation_samples,
samples_b=control_samples,
alpha=self.alpha,
hypothesis=self.hypothesis,
statistic_function=self.statistic_function)
def run_test(
self,
test,
alpha=DEFAULT_ALPHA,
correction_method=None,
display_results=False,
visualize_results=False,
inference_kwargs=None
):
"""
Given a HypothesisTest, run the test and return the results.
Parameters
----------
test : HypothesisTest
A hypothesis test object
alpha : float
The Type I error assumed by the experimenter when running the test
correction_method: str
Correction method used when running test (if any)
display_results : boolean
Whether to print the test results to stdout
visualize_results : boolean
Whether to render a visual representation of the results
Returns
-------
test_results: an instance of a HypothesisTestResults or sublass
The results of the statistical test.
"""
control_obs = test.filter_variations(self.dataset, test.control, self.ds.treatment)
control_obs = test.filter_segments(control_obs)
control_obs = test.filter_metrics(control_obs)
control_samples = Samples(control_obs, name=test.control)
variation_obs = test.filter_variations(self.dataset, test.variation, self.ds.treatment)
variation_obs = test.filter_segments(variation_obs)
variation_obs = test.filter_metrics(variation_obs)
variation_samples = Samples(variation_obs, name=test.variation)
test_results = test.run(
control_samples,
variation_samples,
alpha=alpha, inference_kwargs=inference_kwargs
)
test_results.correction_method = correction_method
if display_results:
test_results.display()
if visualize_results:
test_results.visualize()
return test_results
def __init__(self, traces):
self.variables = []
for k, v in list(traces.items()):
if k != "lp__":
self.variables.append(k)
setattr(self, k, Samples(v))
self.summarize()
def run(self,
control_samples,
variation_samples,
alpha=DEFAULT_ALPHA,
inference_kwargs=None):
"""
Run the inference procedure over the samples with a selected alpha
value.
alpha : float in [0, 1]
the assumed Type I error rate
"""
if isinstance(control_samples, (list, np.ndarray)):
control_samples = Samples(control_samples)
if isinstance(variation_samples, (list, np.ndarray)):
variation_samples = Samples(variation_samples)
self.alpha = alpha
self.comparison = RateComparison(samples_a=variation_samples,
samples_b=control_samples,
alpha=self.alpha,
hypothesis=self.hypothesis)
def run(self,
control_samples,
variation_samples,
alpha=DEFAULT_ALPHA,
inference_kwargs=None):
"""
Run the inference procedure over the samples using a particular
inference method.
control_samples: instance of Samples
the samples for the control condition
variation_smaples: instance of Samples
the samples for the varitation condition
alpha: float
One minus the credible mass under the posterior, used to calculate
the credible interval. Roughly analogous to the alpha in frequentist
tests.
inference_kwargs : dict
Additional kwargs that used by the Stan sampling or variational
inference methods. For details, see:
- http://mc-stan.org/rstan/reference/stanmodel-method-sampling.html
- http://mc-stan.org/rstan/reference/stanmodel-method-vb.html
"""
if isinstance(control_samples, (list, np.ndarray)):
control_samples = Samples(control_samples)
if isinstance(variation_samples, (list, np.ndarray)):
variation_samples = Samples(variation_samples)
self.alpha = alpha
self.credible_mass = 1 - alpha
self.control = control_samples
self.variation = variation_samples
inference_data, hyperparameters = self.inference_inputs(
control_samples, variation_samples)
self.hyperparameters = {
k: v
for k, v in list(hyperparameters.items()) if k != 'recompile'
}
# check for `inference_method` in inference_kwargs
if inference_kwargs is not None:
if 'inference_method' in inference_kwargs:
inference_method = inference_kwargs['inference_method']
del inference_kwargs['inference_method']
else:
inference_method = DEFAULT_BAYESIAN_INFERENCE_METHOD
else:
inference_method = DEFAULT_BAYESIAN_INFERENCE_METHOD
self.inference_method = inference_method
# run inference
if inference_method[0] == "s": # Gibs sampling
if inference_kwargs is None:
inference_kwargs = SAMPLING_DEFAULT_INFERENCE_KWARGS
inference_results = self.bm.sampling(data=inference_data,
**inference_kwargs)
elif inference_method[0] == "v": # variational bayes
if inference_kwargs is None:
inference_kwargs = VB_DEFAULT_INFERENCE_KWARGS
inference_results = self.bm.vb(data=inference_data,
**inference_kwargs)
else:
raise ValueError(
'Unknown inference procedure {!r}'.format(inference_method))
self.traces = Traces(self.extract_traces(inference_results))
return self.traces