def test_sync_all(self, mocker):
split_storage = mocker.Mock(spec=SplitStorage)
split_storage.get_change_number.return_value = 123
split_storage.get_segment_names.return_value = ['segmentA']
split_api = mocker.Mock()
split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123,
'till': 123}
split_sync = SplitSynchronizer(split_api, split_storage)
segment_storage = mocker.Mock(spec=SegmentStorage)
segment_storage.get_change_number.return_value = 123
segment_api = mocker.Mock()
segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2',
'key3'], 'removed': [], 'since': 123, 'till': 123}
segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage)
split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(),
mocker.Mock(), mocker.Mock())
synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
synchronizer.sync_all()
inserted_split = split_storage.put.mock_calls[0][1][0]
assert isinstance(inserted_split, Split)
assert inserted_split.name == 'some_name'
inserted_segment = segment_storage.update.mock_calls[0][1]
assert inserted_segment[0] == 'segmentA'
assert inserted_segment[1] == ['key1', 'key2', 'key3']
assert inserted_segment[2] == []
def test_sync_all_split_attempts(self, mocker):
"""Test that 3 attempts are done before failing."""
split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
counts = {'splits': 0, 'segments': 0}
def sync_splits(*_):
"""Sync Splits."""
counts['splits'] += 1
raise Exception('sarasa')
split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits
split_tasks = mocker.Mock(spec=SplitTasks)
synchronizer = Synchronizer(split_synchronizers, split_tasks)
synchronizer.sync_all()
assert counts['splits'] == 3
def test_sync_all_failed_splits(self, mocker):
api = mocker.Mock()
storage = mocker.Mock()
def run(x, c):
raise APIException("something broke")
api.fetch_splits.side_effect = run
split_sync = SplitSynchronizer(api, storage)
split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(),
mocker.Mock(), mocker.Mock())
sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
sychronizer.synchronize_splits(None) # APIExceptions are handled locally and should not be propagated!
sychronizer.sync_all() # sync_all should not throw!
def test_sync_all_segment_attempts(self, mocker):
"""Test that segments don't trigger retries."""
split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
counts = {'splits': 0, 'segments': 0}
def sync_segments(*_):
"""Sync Splits."""
counts['segments'] += 1
return False
split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments
split_tasks = mocker.Mock(spec=SplitTasks)
synchronizer = Synchronizer(split_synchronizers, split_tasks)
synchronizer.sync_all()
assert counts['segments'] == 1
def test_sync_all_failed_segments(self, mocker):
api = mocker.Mock()
storage = mocker.Mock()
split_storage = mocker.Mock(spec=SplitStorage)
split_storage.get_segment_names.return_value = ['segmentA']
split_sync = mocker.Mock(spec=SplitSynchronizer)
split_sync.synchronize_splits.return_value = None
def run(x, y):
raise APIException("something broke")
api.fetch_segment.side_effect = run
segment_sync = SegmentSynchronizer(api, split_storage, storage)
split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(),
mocker.Mock(), mocker.Mock())
sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
sychronizer.sync_all() # SyncAll should not throw!
assert not sychronizer._synchronize_segments()
def test_sync_all_failed_splits(self, mocker):
api = mocker.Mock()
storage = mocker.Mock()
def run(x):
raise APIException("something broke")
api.fetch_splits.side_effect = run
split_sync = SplitSynchronizer(api, storage)
split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(),
mocker.Mock(), mocker.Mock(),
mocker.Mock(), mocker.Mock())
sychronizer = Synchronizer(split_synchronizers,
mocker.Mock(spec=SplitTasks))
with pytest.raises(APIException):
sychronizer.synchronize_splits(None)
with pytest.raises(APIException):
sychronizer.sync_all()
def test_sync_all_ok(self, mocker):
"""Test that 3 attempts are done before failing."""
split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
counts = {'splits': 0, 'segments': 0}
def sync_splits(*_):
"""Sync Splits."""
counts['splits'] += 1
return []
def sync_segments(*_):
"""Sync Segments."""
counts['segments'] += 1
return True
split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits
split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments
split_tasks = mocker.Mock(spec=SplitTasks)
synchronizer = Synchronizer(split_synchronizers, split_tasks)
synchronizer.sync_all()
assert counts['splits'] == 1
assert counts['segments'] == 1
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals
auth_api_base_url=None, streaming_api_base_url=None):
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
http_client = HttpClient(
sdk_url=sdk_url,
events_url=events_url,
auth_url=auth_api_base_url,
timeout=cfg.get('connectionTimeout')
)
sdk_metadata = util.get_metadata(cfg)
apis = {
'auth': AuthAPI(http_client, api_key, sdk_metadata),
'splits': SplitsAPI(http_client, api_key, sdk_metadata),
'segments': SegmentsAPI(http_client, api_key, sdk_metadata),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
'events': EventsAPI(http_client, api_key, sdk_metadata),
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
}
imp_manager = ImpressionsManager(
cfg['impressionsMode'],
True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
synchronizers = SplitSynchronizers(
SplitSynchronizer(apis['splits'], storages['splits']),
SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']),
ImpressionSynchronizer(apis['impressions'], storages['impressions'],
cfg['impressionsBulkSize']),
EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']),
ImpressionsCountSynchronizer(apis['impressions'], imp_manager),
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
),
SegmentSynchronizationTask(
synchronizers.segment_sync.synchronize_segments,
cfg['segmentsRefreshRate'],
),
ImpressionsSyncTask(
synchronizers.impressions_sync.synchronize_impressions,
cfg['impressionsRefreshRate'],
),
EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']),
ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters)
)
synchronizer = Synchronizer(synchronizers, tasks)
preforked_initialization = cfg.get('preforkedInitialization', False)
sdk_ready_flag = threading.Event() if not preforked_initialization else None
manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'],
sdk_metadata, streaming_api_base_url, api_key[-4:])
storages['events'].set_queue_full_hook(tasks.events_task.flush)
storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush)
recorder = StandardRecorder(
imp_manager,
storages['events'],
storages['impressions'],
)
if preforked_initialization:
synchronizer.sync_all()
synchronizer._split_synchronizers._segment_sync.shutdown()
return SplitFactory(api_key, storages, cfg['labelsEnabled'],
recorder, manager, preforked_initialization=preforked_initialization)
initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer")
initialization_thread.setDaemon(True)
initialization_thread.start()
return SplitFactory(api_key, storages, cfg['labelsEnabled'],
recorder, manager, sdk_ready_flag)