def setup_method(self):
"""Prepare storages with test data."""
split_storage = InMemorySplitStorage()
segment_storage = InMemorySegmentStorage()
split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json')
with open(split_fn, 'r') as flo:
data = json.loads(flo.read())
for split in data['splits']:
split_storage.put(splits.from_raw(split))
segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentHumanBeignsChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
storages = {
'splits': split_storage,
'segments': segment_storage,
'impressions': InMemoryImpressionStorage(5000),
'events': InMemoryEventStorage(5000),
'telemetry': InMemoryTelemetryStorage()
}
impmanager = ImpressionsManager(ImpressionsMode.OPTIMIZED, True)
recorder = StandardRecorder(impmanager, storages['telemetry'], storages['events'],
storages['impressions'])
self.factory = SplitFactory('some_api_key', storages, True, recorder) # pylint:disable=attribute-defined-outside-init
def test_synchronize_splits(self, mocker):
split_storage = InMemorySplitStorage()
split_api = mocker.Mock()
split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123,
'till': 123}
split_sync = SplitSynchronizer(split_api, split_storage)
segment_storage = InMemorySegmentStorage()
segment_api = mocker.Mock()
segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2',
'key3'], 'removed': [], 'since': 123, 'till': 123}
segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage)
split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(),
mocker.Mock(), mocker.Mock())
synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
synchronizer.synchronize_splits(123)
inserted_split = split_storage.get('some_name')
assert isinstance(inserted_split, Split)
assert inserted_split.name == 'some_name'
if not segment_sync._worker_pool.wait_for_completion():
inserted_segment = segment_storage.get('segmentA')
assert inserted_segment.name == 'segmentA'
assert inserted_segment.keys == {'key1', 'key2', 'key3'}
def test_traffic_type_inc_dec_logic(self, mocker):
"""Test that adding/removing split, handles traffic types correctly."""
storage = InMemorySplitStorage()
split1 = mocker.Mock()
name1_prop = mocker.PropertyMock()
name1_prop.return_value = 'split1'
type(split1).name = name1_prop
split2 = mocker.Mock()
name2_prop = mocker.PropertyMock()
name2_prop.return_value = 'split1'
type(split2).name = name2_prop
tt_user = mocker.PropertyMock()
tt_user.return_value = 'user'
tt_account = mocker.PropertyMock()
tt_account.return_value = 'account'
type(split1).traffic_type_name = tt_user
type(split2).traffic_type_name = tt_account
storage.put(split1)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is False
storage.put(split2)
assert storage.is_valid_traffic_type('user') is False
assert storage.is_valid_traffic_type('account') is True
def setup_method(self):
"""Prepare storages with test data."""
split_storage = InMemorySplitStorage()
segment_storage = InMemorySegmentStorage()
split_fn = os.path.join(os.path.dirname(__file__), 'files',
'splitChanges.json')
with open(split_fn, 'r') as flo:
data = json.loads(flo.read())
for split in data['splits']:
split_storage.put(splits.from_raw(split))
segment_fn = os.path.join(os.path.dirname(__file__), 'files',
'segmentEmployeesChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
segment_fn = os.path.join(os.path.dirname(__file__), 'files',
'segmentHumanBeignsChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
self.factory = SplitFactory(
{ #pylint:disable=attribute-defined-outside-init
'splits': split_storage,
'segments': segment_storage,
'impressions': InMemoryImpressionStorage(5000),
'events': InMemoryEventStorage(5000),
'telemetry': InMemoryTelemetryStorage()
},
True)
def _build_localhost_factory(cfg):
"""Build and return a localhost factory for testing/development purposes."""
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(), # not used, just to avoid possible future errors.
'impressions': LocalhostImpressionsStorage(),
'events': LocalhostEventsStorage(),
'telemetry': LocalhostTelemetryStorage()
}
synchronizers = SplitSynchronizers(
LocalSplitSynchronizer(cfg['splitFile'], storages['splits']),
None, None, None, None, None,
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
), None, None, None, None, None,
)
ready_event = threading.Event()
synchronizer = LocalhostSynchronizer(synchronizers, tasks)
manager = Manager(ready_event, synchronizer, None, False)
manager.start()
return SplitFactory(
'localhost',
storages,
False,
ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], True, None),
manager,
ready_event
)
def test_get_split_names(self, mocker):
"""Test retrieving a list of all split names."""
split1 = mocker.Mock()
name1_prop = mocker.PropertyMock()
name1_prop.return_value = 'split1'
type(split1).name = name1_prop
split2 = mocker.Mock()
name2_prop = mocker.PropertyMock()
name2_prop.return_value = 'split2'
type(split2).name = name2_prop
storage = InMemorySplitStorage()
storage.put(split1)
storage.put(split2)
assert set(storage.get_split_names()) == set(['split1', 'split2'])
def test_get_all_splits(self, mocker):
"""Test retrieving a list of all split names."""
split1 = mocker.Mock()
name1_prop = mocker.PropertyMock()
name1_prop.return_value = 'split1'
type(split1).name = name1_prop
split2 = mocker.Mock()
name2_prop = mocker.PropertyMock()
name2_prop.return_value = 'split2'
type(split2).name = name2_prop
storage = InMemorySplitStorage()
storage.put(split1)
storage.put(split2)
all_splits = storage.get_all_splits()
assert next(s for s in all_splits if s.name == 'split1')
assert next(s for s in all_splits if s.name == 'split2')
def test_get_splits(self, mocker):
"""Test retrieving a list of passed splits."""
split1 = mocker.Mock()
name1_prop = mocker.PropertyMock()
name1_prop.return_value = 'split1'
type(split1).name = name1_prop
split2 = mocker.Mock()
name2_prop = mocker.PropertyMock()
name2_prop.return_value = 'split2'
type(split2).name = name2_prop
storage = InMemorySplitStorage()
storage.put(split1)
storage.put(split2)
splits = storage.fetch_many(['split1', 'split2', 'split3'])
assert len(splits) == 3
assert splits['split1'].name == 'split1'
assert splits['split2'].name == 'split2'
assert 'split3' in splits
def _build_localhost_factory(config):
"""Build and return a localhost factory for testing/development purposes."""
cfg = DEFAULT_CONFIG.copy()
cfg.update(config)
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(
), # not used, just to avoid possible future errors.
'impressions': LocalhostImpressionsStorage(),
'events': LocalhostEventsStorage(),
'telemetry': LocalhostTelemetryStorage()
}
ready_event = threading.Event()
tasks = {
'splits':
LocalhostSplitSynchronizationTask(cfg['splitFile'], storages['splits'],
cfg['featuresRefreshRate'],
ready_event)
}
tasks['splits'].start()
return SplitFactory('localhost', storages, False, None, tasks, ready_event)
def test_synchronize_splits_calling_segment_sync_once(self, mocker):
split_storage = InMemorySplitStorage()
split_api = mocker.Mock()
split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123,
'till': 123}
split_sync = SplitSynchronizer(split_api, split_storage)
counts = {'segments': 0}
def sync_segments(*_):
"""Sync Segments."""
counts['segments'] += 1
return True
segment_sync = mocker.Mock()
segment_sync.synchronize_segments.side_effect = sync_segments
segment_sync.segment_exist_in_storage.return_value = False
split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(),
mocker.Mock(), mocker.Mock())
synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
synchronizer.synchronize_splits(123, True)
assert counts['segments'] == 1
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals
auth_api_base_url=None, streaming_api_base_url=None):
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
http_client = HttpClient(
sdk_url=sdk_url,
events_url=events_url,
auth_url=auth_api_base_url,
timeout=cfg.get('connectionTimeout')
)
sdk_metadata = util.get_metadata(cfg)
apis = {
'auth': AuthAPI(http_client, api_key, sdk_metadata),
'splits': SplitsAPI(http_client, api_key),
'segments': SegmentsAPI(http_client, api_key),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
'events': EventsAPI(http_client, api_key, sdk_metadata),
'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
'telemetry': InMemoryTelemetryStorage()
}
imp_manager = ImpressionsManager(
storages['impressions'].put,
cfg['impressionsMode'],
True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
synchronizers = SplitSynchronizers(
SplitSynchronizer(apis['splits'], storages['splits']),
SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']),
ImpressionSynchronizer(apis['impressions'], storages['impressions'],
cfg['impressionsBulkSize']),
EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']),
TelemetrySynchronizer(apis['telemetry'], storages['telemetry']),
ImpressionsCountSynchronizer(apis['impressions'], imp_manager),
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
),
SegmentSynchronizationTask(
synchronizers.segment_sync.synchronize_segments,
cfg['segmentsRefreshRate'],
),
ImpressionsSyncTask(
synchronizers.impressions_sync.synchronize_impressions,
cfg['impressionsRefreshRate'],
),
EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']),
TelemetrySynchronizationTask(
synchronizers.telemetry_sync.synchronize_telemetry,
cfg['metricsRefreshRate'],
),
ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters)
)
synchronizer = Synchronizer(synchronizers, tasks)
sdk_ready_flag = threading.Event()
manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'],
streaming_api_base_url)
initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer")
initialization_thread.setDaemon(True)
initialization_thread.start()
storages['events'].set_queue_full_hook(tasks.events_task.flush)
storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush)
return SplitFactory(api_key, storages, cfg['labelsEnabled'],
imp_manager, manager, sdk_ready_flag)
def test_is_valid_traffic_type(self, mocker):
"""Test that traffic type validation works properly."""
split1 = mocker.Mock()
name1_prop = mocker.PropertyMock()
name1_prop.return_value = 'split1'
type(split1).name = name1_prop
split2 = mocker.Mock()
name2_prop = mocker.PropertyMock()
name2_prop.return_value = 'split2'
type(split2).name = name2_prop
split3 = mocker.Mock()
tt_user = mocker.PropertyMock()
tt_user.return_value = 'user'
tt_account = mocker.PropertyMock()
tt_account.return_value = 'account'
name3_prop = mocker.PropertyMock()
name3_prop.return_value = 'split3'
type(split3).name = name3_prop
type(split1).traffic_type_name = tt_user
type(split2).traffic_type_name = tt_account
type(split3).traffic_type_name = tt_user
storage = InMemorySplitStorage()
storage.put(split1)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is False
storage.put(split2)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
storage.put(split3)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
storage.remove('split1')
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
storage.remove('split2')
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is False
storage.remove('split3')
assert storage.is_valid_traffic_type('user') is False
assert storage.is_valid_traffic_type('account') is False
def test_store_get_changenumber(self):
"""Test that storing and retrieving change numbers works."""
storage = InMemorySplitStorage()
assert storage.get_change_number() == -1
storage.set_change_number(5)
assert storage.get_change_number() == 5
def test_kill_locally(self):
"""Test kill local."""
storage = InMemorySplitStorage()
split = Split('some_split', 123456789, False, 'some', 'traffic_type',
'ACTIVE', 1)
storage.put(split)
storage.set_change_number(1)
storage.kill_locally('test', 'default_treatment', 2)
assert storage.get('test') is None
storage.kill_locally('some_split', 'default_treatment', 0)
assert storage.get('some_split').change_number == 1
assert storage.get('some_split').killed is False
assert storage.get('some_split').default_treatment == 'some'
storage.kill_locally('some_split', 'default_treatment', 3)
assert storage.get('some_split').change_number == 3
def test_storing_retrieving_splits(self, mocker):
"""Test storing and retrieving splits works."""
storage = InMemorySplitStorage()
split = mocker.Mock(spec=Split)
name_property = mocker.PropertyMock()
name_property.return_value = 'some_split'
type(split).name = name_property
storage.put(split)
assert storage.get('some_split') == split
assert storage.get_split_names() == ['some_split']
assert storage.get_all_splits() == [split]
assert storage.get('nonexistant_split') is None
storage.remove('some_split')
assert storage.get('some_split') is None
def _build_in_memory_factory(api_key, config, sdk_url=None, events_url=None): # pylint: disable=too-many-locals
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
cfg = DEFAULT_CONFIG.copy()
cfg.update(config)
http_client = HttpClient(sdk_url=sdk_url,
events_url=events_url,
timeout=cfg.get('connectionTimeout'))
sdk_metadata = util.get_metadata(cfg)
apis = {
'splits': SplitsAPI(http_client, api_key),
'segments': SegmentsAPI(http_client, api_key),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata),
'events': EventsAPI(http_client, api_key, sdk_metadata),
'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
'telemetry': InMemoryTelemetryStorage()
}
# Synchronization flags
splits_ready_flag = threading.Event()
segments_ready_flag = threading.Event()
sdk_ready_flag = threading.Event()
tasks = {
'splits':
SplitSynchronizationTask(apis['splits'], storages['splits'],
cfg['featuresRefreshRate'],
splits_ready_flag),
'segments':
SegmentSynchronizationTask(apis['segments'], storages['segments'],
storages['splits'],
cfg['segmentsRefreshRate'],
segments_ready_flag),
'impressions':
ImpressionsSyncTask(apis['impressions'], storages['impressions'],
cfg['impressionsRefreshRate'],
cfg['impressionsBulkSize']),
'events':
EventsSyncTask(
apis['events'],
storages['events'],
cfg['eventsPushRate'],
cfg['eventsBulkSize'],
),
'telemetry':
TelemetrySynchronizationTask(apis['telemetry'], storages['telemetry'],
cfg['metricsRefreshRate'])
}
# Start tasks that have no dependencies
tasks['splits'].start()
tasks['impressions'].start()
tasks['events'].start()
tasks['telemetry'].start()
storages['events'].set_queue_full_hook(tasks['events'].flush)
storages['impressions'].set_queue_full_hook(tasks['impressions'].flush)
def split_ready_task():
"""Wait for splits to be ready and start fetching segments."""
splits_ready_flag.wait()
tasks['segments'].start()
def segment_ready_task():
"""Wait for segments to be ready and set the main ready flag."""
segments_ready_flag.wait()
sdk_ready_flag.set()
split_completion_thread = threading.Thread(target=split_ready_task)
split_completion_thread.setDaemon(True)
split_completion_thread.start()
segment_completion_thread = threading.Thread(target=segment_ready_task)
segment_completion_thread.setDaemon(True)
segment_completion_thread.start()
return SplitFactory(api_key,
storages,
cfg['labelsEnabled'],
apis,
tasks,
sdk_ready_flag,
impression_listener=_wrap_impression_listener(
cfg['impressionListener'], sdk_metadata))