def test_is_valid_experiment(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
) as p:
p.return_value = "active"
is_valid = experiments.is_valid_experiment("test")
self.assertEqual(is_valid, True)
is_valid = experiments.is_valid_experiment("test2")
self.assertEqual(is_valid, False)
def test_none_returned_on_variant_call_with_no_times(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test_owner",
"type": "r2",
"version": "1",
"experiment": {
"id": 1,
"name": "test",
"variants": {
"active": 50,
"control_1": 25,
"control_2": 25,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
self.assertEqual(self.event_logger.log.call_count, 0)
variant = experiments.variant("test", user=self.user)
self.assertEqual(variant, None)
def test_does_not_log_bucketing_event(self):
event_queue = mock.Mock(spec=EventQueue)
filewatcher = mock.Mock(spec=FileWatcher)
span = mock.MagicMock(spec=ServerSpan)
filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"type": "feature_flag",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"targeting": {
"logged_in": [True, False]
},
"variants": {
"active": 100
},
},
}
}
experiments = Experiments(config_watcher=filewatcher,
server_span=span,
context_name="test")
self.assertEqual(event_queue.put.call_count, 0)
variant = experiments.variant("test",
user_id=self.user_id,
logged_in=True)
self.assertEqual(variant, "active")
self.assertEqual(event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id, logged_in=True)
self.assertEqual(event_queue.put.call_count, 0)
def test_bucketing_event_fields_without_baseplate_user(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test_owner",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
return_value="active",
):
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test",
user_id="t2_2",
app_name="r2",
logged_in=True)
self.assertEqual(self.event_logger.log.call_count, 1)
event_fields = self.event_logger.log.call_args[1]
self.assertEqual(event_fields["variant"], "active")
self.assertEqual(event_fields["user_id"], "t2_2")
self.assertEqual(event_fields["logged_in"], True)
self.assertEqual(event_fields["app_name"], "r2")
self.assertEqual(event_fields["event_type"], EventType.BUCKET)
self.assertNotEqual(event_fields["span"], None)
self.assertEqual(getattr(event_fields["experiment"], "id"), 1)
self.assertEqual(getattr(event_fields["experiment"], "name"), "test")
self.assertEqual(getattr(event_fields["experiment"], "owner"),
"test_owner")
self.assertEqual(getattr(event_fields["experiment"], "version"), "1")
def test_exposure_event_fields(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test_owner",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.expose(
"test",
variant_name="control_1",
user=self.user,
app_name="r2"
)
self.assertEqual(self.event_logger.log.call_count, 1)
event_fields = self.event_logger.log.call_args[1]
self.assertEqual(event_fields["variant"], "control_1")
self.assertEqual(event_fields["user_id"], "t2_1")
self.assertEqual(event_fields["logged_in"], True)
self.assertEqual(event_fields["app_name"], "r2")
self.assertEqual(event_fields["cookie_created_timestamp"], 10000)
self.assertEqual(event_fields["event_type"], EventType.EXPOSE)
self.assertNotEqual(event_fields["span"], None)
self.assertEqual(getattr(event_fields["experiment"], "id"), 1)
self.assertEqual(getattr(event_fields["experiment"], "name"), "test")
self.assertEqual(getattr(event_fields["experiment"], "owner"), "test_owner")
self.assertEqual(getattr(event_fields["experiment"], "version"), "1")
def test_get_all_experiment_names(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10
}
},
},
"test2": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10
}
},
},
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant"
) as p:
p.return_value = "active"
experiment_names = experiments.get_all_experiment_names()
self.assertEqual(len(experiment_names), 2)
self.assertEqual("test" in experiment_names, True)
self.assertEqual("test2" in experiment_names, True)
def test_bucketing_event_fields(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test_owner",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"id": 1,
"name": "test",
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
return_value="active",
):
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_queue.put.call_count, 1)
event = self.event_queue.put.call_args[0][0]
self.assertEqual(event.topic, "bucketing_events")
self.assertEqual(event.event_type, "bucket")
self.assertEqual(event.get_field("experiment_id"), 1)
self.assertEqual(event.get_field("experiment_name"), "test")
self.assertEqual(event.get_field("variant"), "active")
self.assertEqual(event.get_field("owner"), "test_owner")
self.assertEqual(event.get_field("user_id"), "t2_1")
self.assertEqual(event.get_field("loid_created"), 10000)
self.assertEqual(event.get_field("is_logged_out"), False)
def test_none_returned_on_variant_call_with_no_experiment(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test_owner",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
self.assertEqual(self.event_logger.log.call_count, 0)
variant = experiments.variant("test", user=self.user)
self.assertEqual(variant, None)
def test_that_bucketing_events_not_sent_if_no_variant(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
return_value=None,
):
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_queue.put.call_count, 0)
def test_that_bucketing_events_not_sent_if_cant_find_experiment(self):
self.mock_filewatcher.get_data.return_value = {
"other_test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
def test_that_bucketing_events_not_sent_if_cant_find_experiment(self):
self.mock_filewatcher.get_data.return_value = {
"other_test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id)
self.assertEqual(self.event_queue.put.call_count, 0)
def test_does_not_log_bucketing_event(self):
event_queue = mock.Mock(spec=EventQueue)
filewatcher = mock.Mock(spec=FileWatcher)
span = mock.MagicMock(spec=ServerSpan)
filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"type": "feature_flag",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"targeting": {
"logged_in": [True, False],
},
"variants": {
"active": 100,
},
},
},
}
experiments = Experiments(
config_watcher=filewatcher,
event_queue=event_queue,
server_span=span,
context_name="test",
)
self.assertEqual(event_queue.put.call_count, 0)
variant = experiments.variant(
"test",
user_id=self.user_id,
logged_in=True,
)
self.assertEqual(variant, "active")
self.assertEqual(event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id, logged_in=True)
self.assertEqual(event_queue.put.call_count, 0)
def test_that_bucketing_events_are_not_sent_with_override_false(self):
"""Don't send events when override is False"""
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
) as p:
p.return_value = "active"
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test",
user=self.user,
bucketing_event_override=False)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test",
user=self.user,
bucketing_event_override=False)
self.assertEqual(self.event_logger.log.call_count, 0)
p.return_value = None
experiments.variant("test",
user=self.user,
bucketing_event_override=False)
self.assertEqual(self.event_logger.log.call_count, 0)
def test_that_bucketing_events_are_always_sent_with_override_true(self):
"""Send bucketing events on invalid requests with override"""
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
) as p:
p.return_value = "active"
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test",
user_id=self.user_id,
bucketing_event_override=True)
self.assertEqual(self.event_queue.put.call_count, 1)
experiments.variant("test",
user_id=self.user_id,
bucketing_event_override=True)
self.assertEqual(self.event_queue.put.call_count, 2)
p.return_value = None
experiments.variant("test",
user_id=self.user_id,
bucketing_event_override=True)
self.assertEqual(self.event_queue.put.call_count, 3)
def test_that_bucketing_events_not_sent_if_experiment_disables(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"version": "1",
"start_ts": time.time() - THIRTY_DAYS,
"stop_ts": time.time() + THIRTY_DAYS,
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10
}
},
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
return_value="active"
), mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.should_log_bucketing",
return_value=False,
):
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test",
user=self.user,
bucketing_event_override=True)
self.assertEqual(self.event_logger.log.call_count, 0)
def test_that_bucketing_events_not_sent_if_cant_parse_config(self):
self.mock_filewatcher.get_data.side_effect = TypeError()
experiments = Experiments(
config_watcher=self.mock_filewatcher,
server_span=self.mock_span,
context_name="test",
event_logger=self.event_logger,
)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_logger.log.call_count, 0)
def test_that_bucketing_events_not_sent_if_cant_load_config(self):
self.mock_filewatcher.get_data.side_effect = WatchedFileNotAvailableError("path", None) # noqa
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user=self.user)
self.assertEqual(self.event_queue.put.call_count, 0)
def test_that_bucketing_events_not_sent_if_experiment_disables(self):
self.mock_filewatcher.get_data.return_value = {
"test": {
"id": 1,
"name": "test",
"owner": "test",
"type": "r2",
"expires":
(datetime.utcnow() + THIRTY_DAYS).strftime(ISO_DATE_FMT),
"experiment": {
"variants": {
"active": 10,
"control_1": 10,
"control_2": 10,
}
}
}
}
experiments = Experiments(
config_watcher=self.mock_filewatcher,
event_queue=self.event_queue,
server_span=self.mock_span,
context_name="test",
)
with mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.variant",
return_value="active",
), mock.patch(
"baseplate.experiments.providers.r2.R2Experiment.should_log_bucketing",
return_value=False,
):
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test", user_id=self.user_id)
self.assertEqual(self.event_queue.put.call_count, 0)
experiments.variant("test",
user_id=self.user_id,
bucketing_event_override=True)
self.assertEqual(self.event_queue.put.call_count, 0)
