def test_sync_all(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_change_number.return_value = 123 split_storage.get_segment_names.return_value = ['segmentA'] split_api = mocker.Mock() split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123, 'till': 123} split_sync = SplitSynchronizer(split_api, split_storage) segment_storage = mocker.Mock(spec=SegmentStorage) segment_storage.get_change_number.return_value = 123 segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) synchronizer.sync_all() inserted_split = split_storage.put.mock_calls[0][1][0] assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' inserted_segment = segment_storage.update.mock_calls[0][1] assert inserted_segment[0] == 'segmentA' assert inserted_segment[1] == ['key1', 'key2', 'key3'] assert inserted_segment[2] == []
def test_sync_all_split_attempts(self, mocker): """Test that 3 attempts are done before failing.""" split_synchronizers = mocker.Mock(spec=SplitSynchronizers) counts = {'splits': 0, 'segments': 0} def sync_splits(*_): """Sync Splits.""" counts['splits'] += 1 raise Exception('sarasa') split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits split_tasks = mocker.Mock(spec=SplitTasks) synchronizer = Synchronizer(split_synchronizers, split_tasks) synchronizer.sync_all() assert counts['splits'] == 3
def test_sync_all_failed_splits(self, mocker): api = mocker.Mock() storage = mocker.Mock() def run(x, c): raise APIException("something broke") api.fetch_splits.side_effect = run split_sync = SplitSynchronizer(api, storage) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) sychronizer.synchronize_splits(None) # APIExceptions are handled locally and should not be propagated! sychronizer.sync_all() # sync_all should not throw!
def test_sync_all_segment_attempts(self, mocker): """Test that segments don't trigger retries.""" split_synchronizers = mocker.Mock(spec=SplitSynchronizers) counts = {'splits': 0, 'segments': 0} def sync_segments(*_): """Sync Splits.""" counts['segments'] += 1 return False split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments split_tasks = mocker.Mock(spec=SplitTasks) synchronizer = Synchronizer(split_synchronizers, split_tasks) synchronizer.sync_all() assert counts['segments'] == 1
def test_sync_all_failed_segments(self, mocker): api = mocker.Mock() storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA'] split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None def run(x, y): raise APIException("something broke") api.fetch_segment.side_effect = run segment_sync = SegmentSynchronizer(api, split_storage, storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) sychronizer.sync_all() # SyncAll should not throw! assert not sychronizer._synchronize_segments()
def test_sync_all_failed_splits(self, mocker): api = mocker.Mock() storage = mocker.Mock() def run(x): raise APIException("something broke") api.fetch_splits.side_effect = run split_sync = SplitSynchronizer(api, storage) split_synchronizers = SplitSynchronizers(split_sync, mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) with pytest.raises(APIException): sychronizer.synchronize_splits(None) with pytest.raises(APIException): sychronizer.sync_all()
def test_sync_all_ok(self, mocker): """Test that 3 attempts are done before failing.""" split_synchronizers = mocker.Mock(spec=SplitSynchronizers) counts = {'splits': 0, 'segments': 0} def sync_splits(*_): """Sync Splits.""" counts['splits'] += 1 return [] def sync_segments(*_): """Sync Segments.""" counts['segments'] += 1 return True split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments split_tasks = mocker.Mock(spec=SplitTasks) synchronizer = Synchronizer(split_synchronizers, split_tasks) synchronizer.sync_all() assert counts['splits'] == 1 assert counts['segments'] == 1
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals auth_api_base_url=None, streaming_api_base_url=None): """Build and return a split factory tailored to the supplied config.""" if not input_validator.validate_factory_instantiation(api_key): return None http_client = HttpClient( sdk_url=sdk_url, events_url=events_url, auth_url=auth_api_base_url, timeout=cfg.get('connectionTimeout') ) sdk_metadata = util.get_metadata(cfg) apis = { 'auth': AuthAPI(http_client, api_key, sdk_metadata), 'splits': SplitsAPI(http_client, api_key, sdk_metadata), 'segments': SegmentsAPI(http_client, api_key, sdk_metadata), 'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']), 'events': EventsAPI(http_client, api_key, sdk_metadata), } if not input_validator.validate_apikey_type(apis['segments']): return None storages = { 'splits': InMemorySplitStorage(), 'segments': InMemorySegmentStorage(), 'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']), 'events': InMemoryEventStorage(cfg['eventsQueueSize']), } imp_manager = ImpressionsManager( cfg['impressionsMode'], True, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)) synchronizers = SplitSynchronizers( SplitSynchronizer(apis['splits'], storages['splits']), SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']), ImpressionSynchronizer(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']), ImpressionsCountSynchronizer(apis['impressions'], imp_manager), ) tasks = SplitTasks( SplitSynchronizationTask( synchronizers.split_sync.synchronize_splits, cfg['featuresRefreshRate'], ), SegmentSynchronizationTask( synchronizers.segment_sync.synchronize_segments, cfg['segmentsRefreshRate'], ), ImpressionsSyncTask( synchronizers.impressions_sync.synchronize_impressions, cfg['impressionsRefreshRate'], ), EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']), ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters) ) synchronizer = Synchronizer(synchronizers, tasks) preforked_initialization = cfg.get('preforkedInitialization', False) sdk_ready_flag = threading.Event() if not preforked_initialization else None manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'], sdk_metadata, streaming_api_base_url, api_key[-4:]) storages['events'].set_queue_full_hook(tasks.events_task.flush) storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush) recorder = StandardRecorder( imp_manager, storages['events'], storages['impressions'], ) if preforked_initialization: synchronizer.sync_all() synchronizer._split_synchronizers._segment_sync.shutdown() return SplitFactory(api_key, storages, cfg['labelsEnabled'], recorder, manager, preforked_initialization=preforked_initialization) initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer") initialization_thread.setDaemon(True) initialization_thread.start() return SplitFactory(api_key, storages, cfg['labelsEnabled'], recorder, manager, sdk_ready_flag)