def test_render_experiment(self, experiment_manager, participant):
request = mock.MagicMock()
user = mock.MagicMock(is_enrolled=mock.MagicMock(return_value=True), )
experiment_name = 'some_experiment'
alternative = 'some_alternative'
weight = 1
user_variable = None
context = {
'request': request,
}
caller = mock.MagicMock()
experiment = mock.MagicMock()
experiment_manager.get_experiment.return_value = experiment
participant.return_value = user
retval = self.extension.render_experiment(experiment_name, alternative,
weight, user_variable,
context, caller)
experiment_manager.get_experiment.assert_called_once_with(
experiment_name)
experiment.ensure_alternative_exists.assert_called_once_with(
alternative, weight)
participant.assert_called_once_with(request)
user.is_enrolled.assert_called_once_with(experiment_name, alternative)
self.assertEqual(retval, caller.return_value)
def test_init_set_required_fields_not_new(self):
mock_fields = mock.MagicMock()
instance = mock.MagicMock()
instance.pk = 123
form = AdminConditionalForm(instance=instance, fields=mock_fields)
self.assertTrue(form.fields['description'].required)
self.assertTrue(form.fields['template'].required)
def test_render_experiment_specific_user(self, experiment_manager,
participant):
user = mock.MagicMock(is_enrolled=mock.MagicMock(return_value=False), )
experiment_name = 'some_experiment'
alternative = 'some_alternative'
weight = 1
user_variable = 'user'
context = {
'user': user,
}
caller = mock.MagicMock()
experiment = mock.MagicMock()
experiment_manager.get_experiment.return_value = experiment
participant.return_value = user
retval = self.extension.render_experiment(experiment_name, alternative,
weight, user_variable,
context, caller)
experiment_manager.get_experiment.assert_called_once_with(
experiment_name)
experiment.ensure_alternative_exists.assert_called_once_with(
alternative, weight)
participant.assert_called_once_with(user=user)
user.is_enrolled.assert_called_once_with(experiment_name, alternative)
caller.assert_not_called()
self.assertEqual(str(retval), '')
def test_update_remotes_w_remotes_lock_stuk(
self, patched_conf, logger, update_or_create,
_fetch_remote_instances):
mock_server_config = {
'url': 'some_url',
'token': 'some_token',
}
patched_conf.API = {'remotes': [mock_server_config]}
mock_lock = mock.MagicMock()
mock_lock.extend.return_value = False
_fetch_remote_instances.return_value = []
local_mock_instance = mock.MagicMock()
created = True
update_or_create.return_value = (local_mock_instance, created)
value = RemoteExperiment._update_remotes(mock_lock)
self.assertEqual(list(value), [])
self.assertEqual(1, logger.warning.call_count)
self.assertIn(
"Server too slow or lock to short!",
logger.warning.call_args_list[0][0][0]
)
logger.exception.assert_not_called()
mock_lock.extend.assert_called_once_with(
timeout=RemoteExperiment.MAX_WAIT_REMOTE_SYNC)
update_or_create.assert_not_called()
def test_evaluate_conditionals_w_instances(self, experiment_manager):
i1, i2 = mock.MagicMock(), mock.MagicMock()
i1.name = 'mock_exp_1'
i1.default_alternative = 'variate_for_exp_1'
i2.name = 'mock_exp_2'
i1.is_enabled_by_conditionals.return_value = False
i2.is_enabled_by_conditionals.return_value = True
self.experiments.experiment_names = [i1.name, i2.name]
self.experiments.report = {'conditional': {}}
experiment_manager.get_experiment.side_effect = [i1, i2]
self.experiments._evaluate_conditionals()
expected_report = {
'conditional': {
'mock_exp_1': {
'auto-enrolling': False,
'enrolled_alternative': 'variate_for_exp_1',
},
'mock_exp_2': {
'auto-enrolling': True,
'enrolled_alternative': None,
},
},
}
self.assertEqual(expected_report, self.experiments.report)
expected_list = [
'mock_exp_1',
]
self.assertEqual(expected_list, self.experiments.disabled_experiments)
def test_clean_context_code_good_code(self):
instance = mock.MagicMock()
mock_fields = mock.MagicMock()
cleaned_data = {'context_code': 'a = 42\n'}
form = AdminConditionalForm(instance=instance, fields=mock_fields)
form.cleaned_data = cleaned_data
value = form.clean_context_code()
self.assertEqual(value, 'a = 42\n')
def test_clean_context_code_bad_code(self):
instance = mock.MagicMock()
mock_fields = mock.MagicMock()
cleaned_data = {'context_code': 'MAIN:\n\tGOTO MAIN\n'}
form = AdminConditionalForm(instance=instance, fields=mock_fields)
form.cleaned_data = cleaned_data
with self.assertRaises(ValidationError):
form.clean_context_code()
def setUp(self):
self.experiment = Experiment(name='mock_experiment')
self.instance = AdminConditional(
description='mock template',
experiment=self.experiment,
)
self.request = mock.MagicMock()
self.context = {'request': self.request}
self.experiments = mock.MagicMock()
self.experiments.context = self.context
self.request.experiments = self.experiments
def test_evaluate_conditionals_wo_experiments(self, experiment_manager):
i1, i2 = mock.MagicMock(), mock.MagicMock()
i1.name = 'mock_exp_1'
i1.default_alternative = 'variate_for_exp_1'
i2.name = 'mock_exp_2'
i1.is_enabled_by_conditionals.return_value = False
i2.is_enabled_by_conditionals.return_value = True
self.experiments.experiment_names = [i1.name, i2.name]
self.experiments.report = {'conditional': {}}
experiment_manager.get_experiment.side_effect = [None, None]
self.experiments._evaluate_conditionals()
self.assertEqual({'conditional': {}}, self.experiments.report)
def test_set_default_alternative_called_when_not_changed(self):
obj = mock.MagicMock()
request = mock.MagicMock()
form = mock.MagicMock()
changed = False
model = Experiment
admin_site = mock.MagicMock()
admin_site = ExperimentAdmin(model, admin_site)
admin_site.save_model(request, obj, form, changed)
obj.set_default_alternative.assert_not_called()
obj.save()
def test_set_default_alternative_called_when_changed(self):
obj = mock.MagicMock()
request = mock.MagicMock()
form = mock.MagicMock()
changed = True
model = Experiment
admin_site = mock.MagicMock()
admin_site = ExperimentAdmin(model, admin_site)
admin_site.save_model(request, obj, form, changed)
obj.set_default_alternative.assert_called_once_with(
form.cleaned_data['default_alternative'])
obj.save()
def test_update_remotes_w_remotes(
self, patched_conf, logger, update_or_create,
_fetch_remote_instances):
mock_server_config = {
'url': 'some_url',
'token': 'some_token',
}
patched_conf.API = {'remotes': [mock_server_config]}
mock_lock = mock.MagicMock()
mock_instance = {
'name': 'mock remote name',
'url': 'mock remote url',
'admin_url': 'mock remote admin_url',
'start_date': 'mock remote start_date',
'end_date': 'mock remote end_date',
'state': 'mock remote state',
'statistics': 'mock remote statistics',
'alternatives_list': 'mock remote alternatives_list',
}
mock_site = {
'name': 'mock site name',
}
_fetch_remote_instances.return_value = [(mock_instance, mock_site),]
local_mock_instance = mock.MagicMock()
created = True
update_or_create.return_value = (local_mock_instance, created)
value = RemoteExperiment._update_remotes(mock_lock)
self.assertEqual(list(value), [])
logger.warning.assert_not_called()
logger.exception.assert_not_called()
mock_lock.extend.assert_called_once_with(
timeout=RemoteExperiment.MAX_WAIT_REMOTE_SYNC)
update_or_create.assert_called_once_with(
site='mock site name',
name='mock remote name',
defaults={
'url': 'mock remote url',
'admin_url': 'mock remote admin_url',
'start_date': 'mock remote start_date',
'end_date': 'mock remote end_date',
'state': 'mock remote state',
'statistics': 'mock remote statistics',
'alternatives_list': 'mock remote alternatives_list',
'batch': self.batch + 1,
},
)
def test_name_or_const_with_string(self, nodes):
token = mock.MagicMock(type='string', value='some_value')
retval = self.extension._name_or_const(token)
self.assertEqual(retval, nodes.Const.return_value)
nodes.Const.assert_called_once_with('some_value')
def test_save_not_new_instance(self, super_save):
template_instance = AdminConditionalTemplate(
description='some conditional',
template='da template',
context_code='import this')
instance = AdminConditional(
copy_from=template_instance,
description='some other conditional',
template='da edited template',
context_code='from __future__ import braces',
pk=123,
)
mock_fields = mock.MagicMock()
cleaned_data = {
'copy_from': template_instance,
}
form = AdminConditionalForm(instance=instance, fields=mock_fields)
form.cleaned_data = cleaned_data
value = form.save(commit=True)
self.assertEqual(instance.description, 'some other conditional')
self.assertEqual(instance.template, 'da edited template')
self.assertEqual(instance.context_code,
'from __future__ import braces')
super_save.assert_called_once_with(commit=True)
self.assertEqual(value, super_save.return_value)
def test_name_or_const_with_name(self, nodes):
token = mock.MagicMock(type='name', value='some_name')
retval = self.extension._name_or_const(token)
self.assertEqual(retval, nodes.Name.return_value)
nodes.Name.assert_called_once_with('some_name', 'load')
def test_parse_experiment(self):
mock_stream = MockTokenStream([
{
'type': 'string',
'value': 'some_experiment',
'lineno': 123
},
{
'type': 'comma'
},
{
'type': 'string',
'value': 'some_alternative'
},
{
'type': 'block_end'
},
])
self.parser.stream = mock_stream
self.extension.call_method = mock.MagicMock()
self.extension.parse_experiment(self.parser)
self.extension.call_method.assert_called_once_with('render_experiment',
mock.ANY,
lineno=123)
def test_update_remotes_wo_remotes(self, patched_conf):
patched_conf.API['remotes'] = []
mock_lock = mock.MagicMock()
value = RemoteExperiment._update_remotes(mock_lock)
self.assertEqual(list(value), [])
mock_lock.extend.assert_not_called()
def setUp(self):
self.conditional_true = AdminConditional(template='<true />', )
self.conditional_false = AdminConditional(template='no no no', )
self.experiment = Experiment.objects.create(
name='automatic_experiment',
alternatives={
'control': 'stuff',
'variant_1': 'stuff',
},
state=ENABLED_STATE,
)
self.request = mock.MagicMock()
def test_render_experiments_prepare_conditionals(self, _auto_enroll,
nodes):
caller = mock.MagicMock()
context = {'some': 'vars'}
_auto_enroll.return_value = 'mock script'
value = self.extension.render_experiments_prepare_conditionals(
context, caller)
_auto_enroll.assert_called_once_with({'some': 'vars'})
nodes.Markup.assert_called_once_with('mock script')
self.assertEqual(value, nodes.Markup.return_value)
def test_report(self):
instance = mock.MagicMock()
instance.name = "mock_experiment"
active = True
variate = 'mock_variate'
self.experiments._report(instance, active, variate)
self.assertIn('mock_experiment',
self.experiments.report['conditional'])
expected_report = {
'auto-enrolling': True,
'enrolled_alternative': 'mock_variate',
}
self.assertEqual(
expected_report,
self.experiments.report['conditional']['mock_experiment'])
def test_render_experiment_goal(self, get_template, _experiment_goal):
caller = mock.MagicMock()
context = {'some': 'vars'}
goal_name = 'some_goal'
tmplt = get_template.return_value
_experiment_goal.return_value = {'moar': 'vars'}
context2 = {
'some': 'vars',
'moar': 'vars',
}
retval = self.extension.render_experiment_goal(context, goal_name,
caller)
get_template.assert_called_once_with('experiments/goal.html')
self.assertEqual(retval, tmplt.render.return_value)
tmplt.render.assert_called_once_with(context2)
def test_report(self):
experiments = Experiments(self.context)
instance = mock.MagicMock()
instance.name = "mock_experiment"
active = True
variate = 'mock_variate'
experiments._report(instance, active, variate)
self.assertIn(
'mock_experiment', experiments.report['conditional'])
expected_report = {
'disabled': False,
'enrolled_alternative': 'mock_variate',
}
self.assertEqual(
expected_report,
experiments.report['conditional']['mock_experiment']
)
def test_parse_experiment_too_few_args(self):
mock_stream = MockTokenStream([
{
'type': 'string',
'value': 'some_experiment',
'lineno': 123
},
{
'type': 'block_end'
},
])
self.parser.stream = mock_stream
self.extension.call_method = mock.MagicMock()
with self.assertRaises(TemplateSyntaxError):
self.extension.parse_experiment(self.parser)
self.extension.call_method.assert_not_called()
def test_render_experiments_confirm_human(self, get_template,
_experiments_confirm_human):
caller = mock.MagicMock()
context = {'some': 'vars'}
tmplt = get_template.return_value
_experiments_confirm_human.return_value = {'moar': 'vars'}
context2 = {
'some': 'vars',
'moar': 'vars',
}
retval = self.extension.render_experiments_confirm_human(
context, caller)
get_template.assert_called_once_with('experiments/confirm_human.html')
self.assertEqual(retval, tmplt.render.return_value)
tmplt.render.assert_called_once_with(context2)
def test_parse_experiments_confirm_human(self, nodes):
mock_stream = MockTokenStream([
{
'type': 'block_end',
'lineno': 123
},
])
self.parser.stream = mock_stream
self.extension.call_method = mock.MagicMock()
self.extension.parse_experiments_confirm_human(self.parser)
nodes.ContextReference.assert_called_once_with()
args = [
nodes.ContextReference.return_value,
]
self.extension.call_method.assert_called_once_with(
'render_experiments_confirm_human', args, lineno=123)
def test_parse_experiment_goal(self, nodes):
mock_stream = MockTokenStream([
{
'type': 'string',
'value': 'some_goal',
'lineno': 123
},
{
'type': 'block_end'
},
])
self.parser.stream = mock_stream
self.extension.call_method = mock.MagicMock()
self.extension.parse_experiment_goal(self.parser)
nodes.ContextReference.assert_called_once_with()
nodes.Const.assert_called_once_with('some_goal')
const_node = nodes.Const.return_value
args = [nodes.ContextReference.return_value, const_node]
self.extension.call_method.assert_called_once_with(
'render_experiment_goal', args, lineno=123)
def test_update_remotes_but_omg_exception(
self, patched_conf, logger, update_or_create,
_fetch_remote_instances):
mock_server_config = {
'url': 'some_url',
'token': 'some_token',
}
patched_conf.API = {'remotes': [mock_server_config]}
mock_lock = mock.MagicMock()
_fetch_remote_instances.side_effect = ValueError('OMG Exception!')
value = RemoteExperiment._update_remotes(mock_lock)
value = list(value)
self.assertEqual(len(value), 1)
exc = value[0]
self.assertIsInstance(exc, RemoteApiException)
self.assertIn('OMG Exception!', repr(exc))
logger.warning.assert_not_called()
logger.exception.assert_called_once_with(
'Failed updating from remote experiments API')
mock_lock.extend.assert_called_once_with(
timeout=RemoteExperiment.MAX_WAIT_REMOTE_SYNC)
update_or_create.assert_not_called()
def setUp(self):
self.request = mock.MagicMock()
self.context = {'request': self.request}
self.experiments = Experiments(self.context)
def setUp(self):
self.request = mock.MagicMock()
self.site = mock.MagicMock()
self.modeladmin = RemoteExperimentAdmin(RemoteExperiment, self.site)
self.kwargs = {
'site': 'Some remote site',
'name': 'Some experiment',
'url': 'Some URL',
'admin_url': 'Some admin URL',
'state': 1,
'start_date': timezone.datetime(2001, 2, 3, 0, 1, 2),
'end_date': timezone.datetime(2001, 2, 3, 3, 4, 5),
'alternatives_list': ['alt1', 'alt2', 'control'],
'statistics': {
'alternatives': {
'alt1': 11,
'alt2': 22,
'control': 33,
},
'results': {
"goal_1": {
"control": {
"conversion_rate": 0.0,
"average_goal_actions": None,
"conversions": 0
},
"is_primary":
True,
"relevant":
True,
"alternatives": [
[
'alt1',
{
"conversion_rate": 100.0,
"confidence": None,
"improvement": 0.0,
"mann_whitney_confidence": None,
"average_goal_actions": 3.310344827586207,
"conversions": 29
},
],
[
'alt2',
{
"conversion_rate": 95.0,
"confidence": 34.56,
"improvement": 0.0,
"mann_whitney_confidence": None,
"average_goal_actions": 1.9666666666666666,
"conversions": 30
},
],
]
},
"goal_2": {
"control": {
"conversion_rate": 0.0,
"average_goal_actions": None,
"conversions": 0
},
"is_primary":
False,
"relevant":
True,
"alternatives": [
[
'alt1',
{
"conversion_rate": 10.0,
"confidence": 11.22,
"improvement": 0.0,
"mann_whitney_confidence": None,
"average_goal_actions": 1.1111,
"conversions": 2
},
],
[
'alt2',
{
"conversion_rate": 15.0,
"confidence": 67.89,
"improvement": 0.0,
"mann_whitney_confidence": None,
"average_goal_actions": 2.2222,
"conversions": 3
},
],
]
},
},
},
'batch': 14,
}
RemoteExperiment.objects.all().delete()
self.obj = RemoteExperiment.objects.create(**self.kwargs)
def setUp(self):
self.request = mock.MagicMock()
self.context = {'request': self.request}
