def test_synchronize_segment(self, mocker): """Test particular segment update.""" split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) def change_number_mock(segment_name): if change_number_mock._count_a == 0: change_number_mock._count_a = 1 return -1 return 123 change_number_mock._count_a = 0 storage.get_change_number.side_effect = change_number_mock def fetch_segment_mock(segment_name, change_number): if fetch_segment_mock._count_a == 0: fetch_segment_mock._count_a = 1 return {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) segments_synchronizer.synchronize_segment('segmentA') api_calls = [call for call in api.fetch_segment.mock_calls] assert mocker.call('segmentA', -1) in api_calls assert mocker.call('segmentA', 123) in api_calls
def test_recreate(self, mocker): """Test recreate logic.""" from splitio.sync.segment import SegmentSynchronizer segments_synchronizer = SegmentSynchronizer(mocker.Mock(), mocker.Mock(), mocker.Mock()) current_pool = segments_synchronizer._worker_pool segments_synchronizer.recreate() assert segments_synchronizer._worker_pool != current_pool
def uwsgi_update_segments(user_config): """ Update segments task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) seconds = config['segmentsRefreshRate'] metadata = get_metadata(config) segment_sync = SegmentSynchronizer( SegmentsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), UWSGISplitStorage(get_uwsgi()), UWSGISegmentStorage(get_uwsgi()), ) pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access pool.start() split_storage = UWSGISplitStorage(get_uwsgi()) while True: try: for segment_name in split_storage.get_segment_names(): pool.submit_work(segment_name) time.sleep(seconds) except Exception: # pylint: disable=broad-except _LOGGER.error('Error updating segments') _LOGGER.debug('Error: ', exc_info=True)
def test_synchronize_splits(self, mocker): split_storage = InMemorySplitStorage() split_api = mocker.Mock() split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123, 'till': 123} split_sync = SplitSynchronizer(split_api, split_storage) segment_storage = InMemorySegmentStorage() segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) synchronizer.synchronize_splits(123) inserted_split = split_storage.get('some_name') assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' if not segment_sync._worker_pool.wait_for_completion(): inserted_segment = segment_storage.get('segmentA') assert inserted_segment.name == 'segmentA' assert inserted_segment.keys == {'key1', 'key2', 'key3'}
def test_sync_all(self, mocker): split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_change_number.return_value = 123 split_storage.get_segment_names.return_value = ['segmentA'] split_api = mocker.Mock() split_api.fetch_splits.return_value = {'splits': self.splits, 'since': 123, 'till': 123} split_sync = SplitSynchronizer(split_api, split_storage) segment_storage = mocker.Mock(spec=SegmentStorage) segment_storage.get_change_number.return_value = 123 segment_api = mocker.Mock() segment_api.fetch_segment.return_value = {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': 123, 'till': 123} segment_sync = SegmentSynchronizer(segment_api, split_storage, segment_storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) synchronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) synchronizer.sync_all() inserted_split = split_storage.put.mock_calls[0][1][0] assert isinstance(inserted_split, Split) assert inserted_split.name == 'some_name' inserted_segment = segment_storage.update.mock_calls[0][1] assert inserted_segment[0] == 'segmentA' assert inserted_segment[1] == ['key1', 'key2', 'key3'] assert inserted_segment[2] == []
def test_synchronize_segments_error(self, mocker): """On error.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] storage = mocker.Mock(spec=SegmentStorage) storage.get_change_number.return_value = -1 api = mocker.Mock() def run(x): raise APIException("something broke") api.fetch_segment.side_effect = run segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) assert not segments_synchronizer.synchronize_segments()
def test_sync_all_failed_segments(self, mocker): api = mocker.Mock() storage = mocker.Mock() split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA'] split_sync = mocker.Mock(spec=SplitSynchronizer) split_sync.synchronize_splits.return_value = None def run(x, y): raise APIException("something broke") api.fetch_segment.side_effect = run segment_sync = SegmentSynchronizer(api, split_storage, storage) split_synchronizers = SplitSynchronizers(split_sync, segment_sync, mocker.Mock(), mocker.Mock(), mocker.Mock()) sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks)) sychronizer.sync_all() # SyncAll should not throw! assert not sychronizer._synchronize_segments()
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals auth_api_base_url=None, streaming_api_base_url=None): """Build and return a split factory tailored to the supplied config.""" if not input_validator.validate_factory_instantiation(api_key): return None http_client = HttpClient( sdk_url=sdk_url, events_url=events_url, auth_url=auth_api_base_url, timeout=cfg.get('connectionTimeout') ) sdk_metadata = util.get_metadata(cfg) apis = { 'auth': AuthAPI(http_client, api_key, sdk_metadata), 'splits': SplitsAPI(http_client, api_key), 'segments': SegmentsAPI(http_client, api_key), 'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']), 'events': EventsAPI(http_client, api_key, sdk_metadata), 'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata) } if not input_validator.validate_apikey_type(apis['segments']): return None storages = { 'splits': InMemorySplitStorage(), 'segments': InMemorySegmentStorage(), 'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']), 'events': InMemoryEventStorage(cfg['eventsQueueSize']), 'telemetry': InMemoryTelemetryStorage() } imp_manager = ImpressionsManager( storages['impressions'].put, cfg['impressionsMode'], True, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)) synchronizers = SplitSynchronizers( SplitSynchronizer(apis['splits'], storages['splits']), SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']), ImpressionSynchronizer(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']), TelemetrySynchronizer(apis['telemetry'], storages['telemetry']), ImpressionsCountSynchronizer(apis['impressions'], imp_manager), ) tasks = SplitTasks( SplitSynchronizationTask( synchronizers.split_sync.synchronize_splits, cfg['featuresRefreshRate'], ), SegmentSynchronizationTask( synchronizers.segment_sync.synchronize_segments, cfg['segmentsRefreshRate'], ), ImpressionsSyncTask( synchronizers.impressions_sync.synchronize_impressions, cfg['impressionsRefreshRate'], ), EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']), TelemetrySynchronizationTask( synchronizers.telemetry_sync.synchronize_telemetry, cfg['metricsRefreshRate'], ), ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters) ) synchronizer = Synchronizer(synchronizers, tasks) sdk_ready_flag = threading.Event() manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'], streaming_api_base_url) initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer") initialization_thread.setDaemon(True) initialization_thread.start() storages['events'].set_queue_full_hook(tasks.events_task.flush) storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush) return SplitFactory(api_key, storages, cfg['labelsEnabled'], imp_manager, manager, sdk_ready_flag)
def test_synchronize_segments(self, mocker): """Test the normal operation flow.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = [ 'segmentA', 'segmentB', 'segmentC' ] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) def change_number_mock(segment_name): if segment_name == 'segmentA' and change_number_mock._count_a == 0: change_number_mock._count_a = 1 return -1 if segment_name == 'segmentB' and change_number_mock._count_b == 0: change_number_mock._count_b = 1 return -1 if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 storage.get_change_number.side_effect = change_number_mock # Setup a mocked segment api to return segments mentioned before. def fetch_segment_mock(segment_name, change_number, fetch_options): if segment_name == 'segmentA' and fetch_segment_mock._count_a == 0: fetch_segment_mock._count_a = 1 return { 'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': -1, 'till': 123 } if segment_name == 'segmentB' and fetch_segment_mock._count_b == 0: fetch_segment_mock._count_b = 1 return { 'name': 'segmentB', 'added': ['key4', 'key5', 'key6'], 'removed': [], 'since': -1, 'till': 123 } if segment_name == 'segmentC' and fetch_segment_mock._count_c == 0: fetch_segment_mock._count_c = 1 return { 'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123 } return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock from splitio.sync.segment import SegmentSynchronizer segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) assert segments_synchronizer.synchronize_segments() api_calls = [call for call in api.fetch_segment.mock_calls] assert mocker.call('segmentA', -1, FetchOptions(True)) in api_calls assert mocker.call('segmentB', -1, FetchOptions(True)) in api_calls assert mocker.call('segmentC', -1, FetchOptions(True)) in api_calls assert mocker.call('segmentA', 123, FetchOptions(True)) in api_calls assert mocker.call('segmentB', 123, FetchOptions(True)) in api_calls assert mocker.call('segmentC', 123, FetchOptions(True)) in api_calls segment_put_calls = storage.put.mock_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for call in segment_put_calls: _, positional_args, _ = call segment = positional_args[0] assert isinstance(segment, Segment) assert segment.name in segments_to_validate segments_to_validate.remove(segment.name)
def test_synchronize_segment_cdn(self, mocker): """Test particular segment update cdn bypass.""" mocker.patch( 'splitio.sync.segment._ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES', new=3) from splitio.sync.segment import SegmentSynchronizer split_storage = mocker.Mock(spec=SplitStorage) storage = mocker.Mock(spec=SegmentStorage) def change_number_mock(segment_name): change_number_mock._count_a += 1 if change_number_mock._count_a == 1: return -1 elif change_number_mock._count_a >= 2 and change_number_mock._count_a <= 3: return 123 elif change_number_mock._count_a <= 7: return 1234 return 12345 # Return proper cn for CDN Bypass change_number_mock._count_a = 0 storage.get_change_number.side_effect = change_number_mock def fetch_segment_mock(segment_name, change_number, fetch_options): fetch_segment_mock._count_a += 1 if fetch_segment_mock._count_a == 1: return { 'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': -1, 'till': 123 } elif fetch_segment_mock._count_a == 2: return {'added': [], 'removed': [], 'since': 123, 'till': 123} elif fetch_segment_mock._count_a == 3: return {'added': [], 'removed': [], 'since': 123, 'till': 1234} elif fetch_segment_mock._count_a >= 4 and fetch_segment_mock._count_a <= 6: return { 'added': [], 'removed': [], 'since': 1234, 'till': 1234 } elif fetch_segment_mock._count_a == 7: return { 'added': [], 'removed': [], 'since': 1234, 'till': 12345 } return {'added': [], 'removed': [], 'since': 12345, 'till': 12345} fetch_segment_mock._count_a = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) segments_synchronizer.synchronize_segment('segmentA') assert mocker.call('segmentA', -1, FetchOptions(True)) in api.fetch_segment.mock_calls assert mocker.call('segmentA', 123, FetchOptions(True)) in api.fetch_segment.mock_calls segments_synchronizer._backoff = Backoff(1, 0.1) segments_synchronizer.synchronize_segment('segmentA', 12345) assert mocker.call('segmentA', 12345, FetchOptions(True, 1234)) in api.fetch_segment.mock_calls assert len( api.fetch_segment.mock_calls ) == 8 # 2 ok + BACKOFF(2 since==till + 2 re-attempts) + CDN(2 since==till)
def test_normal_operation(self, mocker): """Test the normal operation flow.""" split_storage = mocker.Mock(spec=SplitStorage) split_storage.get_segment_names.return_value = ['segmentA', 'segmentB', 'segmentC'] # Setup a mocked segment storage whose changenumber returns -1 on first fetch and # 123 afterwards. storage = mocker.Mock(spec=SegmentStorage) def change_number_mock(segment_name): if segment_name == 'segmentA' and change_number_mock._count_a == 0: change_number_mock._count_a = 1 return -1 if segment_name == 'segmentB' and change_number_mock._count_b == 0: change_number_mock._count_b = 1 return -1 if segment_name == 'segmentC' and change_number_mock._count_c == 0: change_number_mock._count_c = 1 return -1 return 123 change_number_mock._count_a = 0 change_number_mock._count_b = 0 change_number_mock._count_c = 0 storage.get_change_number.side_effect = change_number_mock # Setup a mocked segment api to return segments mentioned before. def fetch_segment_mock(segment_name, change_number): if segment_name == 'segmentA' and fetch_segment_mock._count_a == 0: fetch_segment_mock._count_a = 1 return {'name': 'segmentA', 'added': ['key1', 'key2', 'key3'], 'removed': [], 'since': -1, 'till': 123} if segment_name == 'segmentB' and fetch_segment_mock._count_b == 0: fetch_segment_mock._count_b = 1 return {'name': 'segmentB', 'added': ['key4', 'key5', 'key6'], 'removed': [], 'since': -1, 'till': 123} if segment_name == 'segmentC' and fetch_segment_mock._count_c == 0: fetch_segment_mock._count_c = 1 return {'name': 'segmentC', 'added': ['key7', 'key8', 'key9'], 'removed': [], 'since': -1, 'till': 123} return {'added': [], 'removed': [], 'since': 123, 'till': 123} fetch_segment_mock._count_a = 0 fetch_segment_mock._count_b = 0 fetch_segment_mock._count_c = 0 api = mocker.Mock() api.fetch_segment.side_effect = fetch_segment_mock segments_synchronizer = SegmentSynchronizer(api, split_storage, storage) task = segment_sync.SegmentSynchronizationTask(segments_synchronizer.synchronize_segments, 0.5) task.start() time.sleep(0.7) assert task.is_running() stop_event = threading.Event() task.stop(stop_event) stop_event.wait() assert not task.is_running() api_calls = [call for call in api.fetch_segment.mock_calls] assert mocker.call('segmentA', -1) in api_calls assert mocker.call('segmentB', -1) in api_calls assert mocker.call('segmentC', -1) in api_calls assert mocker.call('segmentA', 123) in api_calls assert mocker.call('segmentB', 123) in api_calls assert mocker.call('segmentC', 123) in api_calls segment_put_calls = storage.put.mock_calls segments_to_validate = set(['segmentA', 'segmentB', 'segmentC']) for call in segment_put_calls: _, positional_args, _ = call segment = positional_args[0] assert isinstance(segment, Segment) assert segment.name in segments_to_validate segments_to_validate.remove(segment.name)