def test_add_events(self, mocker): """Test that adding impressions to storage works.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisEventsStorage(adapter, metadata) events = [ EventWrapper(event=Event('key1', 'user', 'purchase', 10, 123456, None), size=32768), EventWrapper(event=Event('key2', 'user', 'purchase', 10, 123456, None), size=32768), EventWrapper(event=Event('key3', 'user', 'purchase', 10, 123456, None), size=32768), EventWrapper(event=Event('key4', 'user', 'purchase', 10, 123456, None), size=32768), ] assert storage.put(events) is True list_of_raw_events = [json.dumps({ 'm': { # METADATA PORTION 's': metadata.sdk_version, 'n': metadata.instance_name, 'i': metadata.instance_ip, }, 'e': { # EVENT PORTION 'key': e.event.key, 'trafficTypeName': e.event.traffic_type_name, 'eventTypeId': e.event.event_type_id, 'value': e.event.value, 'timestamp': e.event.timestamp, 'properties': e.event.properties, } }) for e in events] # To deal with python2 & 3 differences in hashing/order when dumping json. list_of_raw_json_strings_called = adapter.rpush.mock_calls[0][1][1:] list_of_events_called = [ json.loads(event) for event in list_of_raw_json_strings_called ] list_of_events_sent = [ json.loads(event) for event in list_of_raw_events ] for item in list_of_events_sent: assert item in list_of_events_called # assert adapter.rpush.mock_calls == [mocker.call('SPLITIO.events', to_validate)] # Assert that if an exception is thrown it's caught and False is returned adapter.reset_mock() def _raise_exc(*_): raise RedisAdapterException('something') adapter.rpush.side_effect = _raise_exc assert storage.put(events) is False
def uwsgi_report_impressions(user_config): """ Flush impressions task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) metadata = get_metadata(config) seconds = config['impressionsRefreshRate'] storage = UWSGIImpressionStorage(get_uwsgi()) impressions_sync = ImpressionSynchronizer( ImpressionsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata, config['impressionsMode']), storage, config['impressionsBulkSize']) while True: try: impressions_sync.synchronize_impressions() # pylint: disable=protected-access for _ in range(0, seconds): if storage.should_flush(): storage.acknowledge_flush() break time.sleep(1) except Exception: # pylint: disable=broad-except _LOGGER.error('Error posting impressions') _LOGGER.debug('Error: ', exc_info=True)
def test_add_latency_to_pipe(self, mocker): """Test incrementing latency.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisTelemetryStorage(adapter, metadata) storage.inc_latency('some_latency', 0) storage.inc_latency('some_latency', 1) storage.inc_latency('some_latency', 5) storage.inc_latency('some_latency', 5) storage.inc_latency('some_latency', 22) assert adapter.incr.mock_calls == [ mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/latency.some_latency.bucket.0'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/latency.some_latency.bucket.1'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/latency.some_latency.bucket.5'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/latency.some_latency.bucket.5') ]
def _build_redis_factory(api_key, cfg): """Build and return a split factory with redis-based storage.""" sdk_metadata = util.get_metadata(cfg) redis_adapter = redis.build(cfg) cache_enabled = cfg.get('redisLocalCacheEnabled', False) cache_ttl = cfg.get('redisLocalCacheTTL', 5) storages = { 'splits': RedisSplitStorage(redis_adapter, cache_enabled, cache_ttl), 'segments': RedisSegmentStorage(redis_adapter), 'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata), 'events': RedisEventsStorage(redis_adapter, sdk_metadata), } data_sampling = cfg.get('dataSampling', DEFAULT_DATA_SAMPLING) if data_sampling < _MIN_DEFAULT_DATA_SAMPLING_ALLOWED: _LOGGER.warning("dataSampling cannot be less than %.2f, defaulting to minimum", _MIN_DEFAULT_DATA_SAMPLING_ALLOWED) data_sampling = _MIN_DEFAULT_DATA_SAMPLING_ALLOWED recorder = PipelinedRecorder( redis_adapter.pipeline, ImpressionsManager(cfg['impressionsMode'], False, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)), storages['events'], storages['impressions'], data_sampling, ) return SplitFactory( api_key, storages, cfg['labelsEnabled'], recorder, )
def test_put_fetch_contains_ip_address_disabled(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) cfg = DEFAULT_CONFIG.copy() cfg.update({'IPAddressesEnabled': False}) metadata = get_metadata(cfg) storage = RedisTelemetryStorage(adapter, metadata) try: storage.inc_counter('counter1') storage.inc_counter('counter1') storage.inc_counter('counter2') assert adapter.get(storage._get_counter_key('counter1')) == '2' assert adapter.get(storage._get_counter_key('counter2')) == '1' storage.inc_latency('latency1', 3) storage.inc_latency('latency1', 3) storage.inc_latency('latency2', 6) assert adapter.get(storage._get_latency_key('latency1', 3)) == '2' assert adapter.get(storage._get_latency_key('latency2', 6)) == '1' storage.put_gauge('gauge1', 3) storage.put_gauge('gauge2', 1) assert adapter.get(storage._get_gauge_key('gauge1')) == '3' assert adapter.get(storage._get_gauge_key('gauge2')) == '1' finally: adapter.delete(storage._get_counter_key('counter1'), storage._get_counter_key('counter2'), storage._get_latency_key('latency1', 3), storage._get_latency_key('latency2', 6), storage._get_gauge_key('gauge1'), storage._get_gauge_key('gauge2'))
def test_post_impressions_ip_address_disabled(self, mocker): """Test impressions posting API call.""" httpclient = mocker.Mock(spec=client.HttpClient) httpclient.post.return_value = client.HttpResponse(200, '') cfg = DEFAULT_CONFIG.copy() cfg.update({'IPAddressesEnabled': False}) sdk_metadata = get_metadata(cfg) impressions_api = impressions.ImpressionsAPI(httpclient, 'some_api_key', sdk_metadata) response = impressions_api.flush_impressions(self.impressions) call_made = httpclient.post.mock_calls[0] # validate positional arguments assert call_made[1] == ('events', '/testImpressions/bulk', 'some_api_key') # validate key-value args (headers) assert call_made[2]['extra_headers'] == { 'SplitSDKVersion': 'python-%s' % __version__, } # validate key-value args (body) assert call_made[2]['body'] == self.expectedImpressions
def uwsgi_report_telemetry(user_config): """ Flush events task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) metadata = get_metadata(config) seconds = config.get('metricsRefreshRate', 30) storage = UWSGITelemetryStorage(get_uwsgi()) task = TelemetrySynchronizationTask( TelemetryAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), storage, None, # Period not needed. Task is being triggered manually. ) while True: try: task._flush_telemetry() #pylint: disable=protected-access time.sleep(seconds) except Exception: #pylint: disable=broad-except _LOGGER.error('Error posting metrics') _LOGGER.debug('Error: ', exc_info=True)
def uwsgi_report_events(user_config): """ Flush events task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) metadata = get_metadata(config) seconds = config.get('eventsRefreshRate', 30) storage = UWSGIEventStorage(get_uwsgi()) task = EventsSyncTask( EventsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), storage, None, # Period not needed. Task is being triggered manually. config['eventsBulkSize']) while True: try: task._send_events() #pylint: disable=protected-access for _ in xrange(0, seconds): if storage.should_flush(): storage.acknowledge_flush() break time.sleep(1) except Exception: #pylint: disable=broad-except _LOGGER.error('Error posting metrics') _LOGGER.debug('Error: ', exc_info=True)
def test_wrap_impressions(self, mocker): """Test wrap impressions.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisImpressionsStorage(adapter, metadata) impressions = [ Impression('key1', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key2', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key3', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key4', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654) ] to_validate = [json.dumps({ 'm': { # METADATA PORTION 's': metadata.sdk_version, 'n': metadata.instance_name, 'i': metadata.instance_ip, }, 'i': { # IMPRESSION PORTION 'k': impression.matching_key, 'b': impression.bucketing_key, 'f': impression.feature_name, 't': impression.treatment, 'r': impression.label, 'c': impression.change_number, 'm': impression.time, } }) for impression in impressions] assert storage._wrap_impressions(impressions) == to_validate
def uwsgi_update_segments(user_config): """ Update segments task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) seconds = config['segmentsRefreshRate'] metadata = get_metadata(config) segment_sync = SegmentSynchronizer( SegmentsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), UWSGISplitStorage(get_uwsgi()), UWSGISegmentStorage(get_uwsgi()), ) pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access pool.start() split_storage = UWSGISplitStorage(get_uwsgi()) while True: try: for segment_name in split_storage.get_segment_names(): pool.submit_work(segment_name) time.sleep(seconds) except Exception: # pylint: disable=broad-except _LOGGER.error('Error updating segments') _LOGGER.debug('Error: ', exc_info=True)
def test_put_fetch_contains(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) metadata = get_metadata({}) storage = RedisTelemetryStorage(adapter, metadata) try: storage.inc_counter('counter1') storage.inc_counter('counter1') storage.inc_counter('counter2') assert adapter.get(storage._get_counter_key('counter1')) == '2' assert adapter.get(storage._get_counter_key('counter2')) == '1' storage.inc_latency('latency1', 3) storage.inc_latency('latency1', 3) storage.inc_latency('latency2', 6) assert adapter.get(storage._get_latency_key('latency1', 3)) == '2' assert adapter.get(storage._get_latency_key('latency2', 6)) == '1' storage.put_gauge('gauge1', 3) storage.put_gauge('gauge2', 1) assert adapter.get(storage._get_gauge_key('gauge1')) == '3' assert adapter.get(storage._get_gauge_key('gauge2')) == '1' finally: adapter.delete(storage._get_counter_key('counter1'), storage._get_counter_key('counter2'), storage._get_latency_key('latency1', 3), storage._get_latency_key('latency2', 6), storage._get_gauge_key('gauge1'), storage._get_gauge_key('gauge2'))
def test_inc_gauge(self, mocker): """Test incrementing latency.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisTelemetryStorage(adapter, metadata) storage.put_gauge('gauge1', 123) storage.put_gauge('gauge2', 456) assert adapter.set.mock_calls == [ mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/gauge.gauge1', 123), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/gauge.gauge2', 456) ]
def test_put_fetch_contains(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) try: self._put_events(adapter, get_metadata({})) evts = adapter.lrange('SPLITIO.events', 0, 2) assert len(evts) == 3 for rawEvent in evts: event = json.loads(rawEvent) assert event['m']['i'] != 'NA' assert event['m']['n'] != 'NA' finally: adapter.delete('SPLITIO.events')
def test_put_fetch_contains(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) try: self._put_impressions(adapter, get_metadata({})) imps = adapter.lrange('SPLITIO.impressions', 0, 2) assert len(imps) == 3 for rawImpression in imps: impression = json.loads(rawImpression) assert impression['m']['i'] != 'NA' assert impression['m']['n'] != 'NA' finally: adapter.delete('SPLITIO.impressions')
def test_get_metadata(self, mocker): """Test the get_metadata function.""" get_ip_mock = mocker.Mock() get_host_mock = mocker.Mock() mocker.patch('splitio.client.util._get_ip', new=get_ip_mock) mocker.patch('splitio.client.util._get_hostname', new=get_host_mock) meta = util.get_metadata({'machineIp': 'some_ip', 'machineName': 'some_machine_name'}) assert get_ip_mock.mock_calls == [] assert get_host_mock.mock_calls == [] assert meta.instance_ip == 'some_ip' assert meta.instance_name == 'some_machine_name' assert meta.sdk_version == 'python-' + __version__ meta = util.get_metadata(config.DEFAULT_CONFIG) assert get_ip_mock.mock_calls == [mocker.call()] assert get_host_mock.mock_calls == [mocker.call(mocker.ANY)] get_ip_mock.reset_mock() get_host_mock.reset_mock() meta = util.get_metadata({}) assert get_ip_mock.mock_calls == [mocker.call()] assert get_host_mock.mock_calls == [mocker.call(mocker.ANY)]
def test_add_impressions(self, mocker): """Test that adding impressions to storage works.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisImpressionsStorage(adapter, metadata) impressions = [ Impression('key1', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key2', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key3', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key4', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654) ] assert storage.put(impressions) is True to_validate = [json.dumps({ 'm': { # METADATA PORTION 's': metadata.sdk_version, 'n': metadata.instance_name, 'i': metadata.instance_ip, }, 'i': { # IMPRESSION PORTION 'k': impression.matching_key, 'b': impression.bucketing_key, 'f': impression.feature_name, 't': impression.treatment, 'r': impression.label, 'c': impression.change_number, 'm': impression.time, } }) for impression in impressions] assert adapter.rpush.mock_calls == [ mocker.call('SPLITIO.impressions', *to_validate) ] # Assert that if an exception is thrown it's caught and False is returned adapter.reset_mock() def _raise_exc(*_): raise RedisAdapterException('something') adapter.rpush.side_effect = _raise_exc assert storage.put(impressions) is False
def test_put_fetch_contains(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) try: metadata = get_metadata({}) storage = RedisEventsStorage(adapter, metadata) storage.put([ events.Event('key1', 'user', 'purchase', 3.5, 123456), events.Event('key2', 'user', 'purchase', 3.5, 123456), events.Event('key3', 'user', 'purchase', 3.5, 123456) ]) evts = adapter.lrange('SPLITIO.events', 0, 2) assert len(evts) == 3 finally: adapter.delete('SPLITIO.events')
def test_put_fetch_contains_ip_address_disabled(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) try: cfg = DEFAULT_CONFIG.copy() cfg.update({'IPAddressesEnabled': False}) self._put_events(adapter, get_metadata(cfg)) evts = adapter.lrange('SPLITIO.events', 0, 2) assert len(evts) == 3 for rawEvent in evts: event = json.loads(rawEvent) assert event['m']['i'] == 'NA' assert event['m']['n'] == 'NA' finally: adapter.delete('SPLITIO.events')
def _build_uwsgi_factory(config): """Build and return a split factory with redis-based storage.""" cfg = DEFAULT_CONFIG.copy() cfg.update(config) sdk_metadata = util.get_metadata(cfg) uwsgi_adapter = get_uwsgi() storages = { 'splits': UWSGISplitStorage(uwsgi_adapter), 'segments': UWSGISegmentStorage(uwsgi_adapter), 'impressions': UWSGIImpressionStorage(uwsgi_adapter), 'events': UWSGIEventStorage(uwsgi_adapter), 'telemetry': UWSGITelemetryStorage(uwsgi_adapter) } return SplitFactory(storages, cfg['labelsEnabled'], impression_listener=_wrap_impression_listener( cfg['impressionListener'], sdk_metadata))
def _build_redis_factory(config): """Build and return a split factory with redis-based storage.""" cfg = DEFAULT_CONFIG.copy() cfg.update(config) sdk_metadata = util.get_metadata(config) redis_adapter = redis.build(config) storages = { 'splits': RedisSplitStorage(redis_adapter), 'segments': RedisSegmentStorage(redis_adapter), 'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata), 'events': RedisEventsStorage(redis_adapter, sdk_metadata), 'telemetry': RedisTelemetryStorage(redis_adapter, sdk_metadata) } return SplitFactory(storages, cfg['labelsEnabled'], impression_listener=_wrap_impression_listener( cfg['impressionListener'], sdk_metadata))
def test_inc_counter(self, mocker): """Test incrementing latency.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisTelemetryStorage(adapter, metadata) storage.inc_counter('some_counter_1') storage.inc_counter('some_counter_1') storage.inc_counter('some_counter_1') storage.inc_counter('some_counter_2') storage.inc_counter('some_counter_2') assert adapter.incr.mock_calls == [ mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/count.some_counter_1'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/count.some_counter_1'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/count.some_counter_1'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/count.some_counter_2'), mocker.call('SPLITIO/' + metadata.sdk_version + '/' + metadata.instance_name + '/count.some_counter_2') ]
def _build_uwsgi_factory(api_key, cfg): """Build and return a split factory with redis-based storage.""" sdk_metadata = util.get_metadata(cfg) uwsgi_adapter = get_uwsgi() storages = { 'splits': UWSGISplitStorage(uwsgi_adapter), 'segments': UWSGISegmentStorage(uwsgi_adapter), 'impressions': UWSGIImpressionStorage(uwsgi_adapter), 'events': UWSGIEventStorage(uwsgi_adapter), 'telemetry': UWSGITelemetryStorage(uwsgi_adapter) } return SplitFactory( api_key, storages, cfg['labelsEnabled'], ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], True, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)) )
def test_post_impressions(self, mocker): """Test impressions posting API call.""" httpclient = mocker.Mock(spec=client.HttpClient) httpclient.post.return_value = client.HttpResponse(200, '') cfg = DEFAULT_CONFIG.copy() cfg.update({ 'IPAddressesEnabled': True, 'machineName': 'some_machine_name', 'machineIp': '123.123.123.123' }) sdk_metadata = get_metadata(cfg) impressions_api = impressions.ImpressionsAPI(httpclient, 'some_api_key', sdk_metadata) response = impressions_api.flush_impressions(self.impressions) call_made = httpclient.post.mock_calls[0] # validate positional arguments assert call_made[1] == ('events', '/testImpressions/bulk', 'some_api_key') # validate key-value args (headers) assert call_made[2]['extra_headers'] == { 'SplitSDKVersion': 'python-%s' % __version__, 'SplitSDKMachineIP': '123.123.123.123', 'SplitSDKMachineName': 'some_machine_name', 'SplitSDKImpressionsMode': 'OPTIMIZED' } # validate key-value args (body) assert call_made[2]['body'] == self.expectedImpressions httpclient.reset_mock() def raise_exception(*args, **kwargs): raise client.HttpClientException('some_message') httpclient.post.side_effect = raise_exception with pytest.raises(APIException) as exc_info: response = impressions_api.flush_impressions(self.impressions) assert exc_info.type == APIException assert exc_info.value.message == 'some_message'
def test_put_fetch_contains(self): """Test storing and retrieving splits in redis.""" adapter = _build_default_client({}) try: metadata = get_metadata({}) storage = RedisImpressionsStorage(adapter, metadata) storage.put([ impressions.Impression('key1', 'feature1', 'on', 'l1', 123456, 'b1', 321654), impressions.Impression('key2', 'feature1', 'on', 'l1', 123456, 'b1', 321654), impressions.Impression('key3', 'feature1', 'on', 'l1', 123456, 'b1', 321654) ]) imps = adapter.lrange('SPLITIO.impressions', 0, 2) assert len(imps) == 3 finally: adapter.delete('SPLITIO.impressions')
def test_auth(self, mocker): """Test auth API call.""" token = "eyJhbGciOiJIUzI1NiIsImtpZCI6IjVZOU05US45QnJtR0EiLCJ0eXAiOiJKV1QifQ.eyJ4LWFibHktY2FwYWJpbGl0eSI6IntcIk56TTJNREk1TXpjMF9NVGd5TlRnMU1UZ3dOZz09X3NlZ21lbnRzXCI6W1wic3Vic2NyaWJlXCJdLFwiTnpNMk1ESTVNemMwX01UZ3lOVGcxTVRnd05nPT1fc3BsaXRzXCI6W1wic3Vic2NyaWJlXCJdLFwiY29udHJvbF9wcmlcIjpbXCJzdWJzY3JpYmVcIixcImNoYW5uZWwtbWV0YWRhdGE6cHVibGlzaGVyc1wiXSxcImNvbnRyb2xfc2VjXCI6W1wic3Vic2NyaWJlXCIsXCJjaGFubmVsLW1ldGFkYXRhOnB1Ymxpc2hlcnNcIl19IiwieC1hYmx5LWNsaWVudElkIjoiY2xpZW50SWQiLCJleHAiOjE2MDIwODgxMjcsImlhdCI6MTYwMjA4NDUyN30.5_MjWonhs6yoFhw44hNJm3H7_YMjXpSW105DwjjppqE" httpclient = mocker.Mock(spec=client.HttpClient) payload = '{{"pushEnabled": true, "token": "{token}"}}'.format( token=token) cfg = DEFAULT_CONFIG.copy() cfg.update({ 'IPAddressesEnabled': True, 'machineName': 'some_machine_name', 'machineIp': '123.123.123.123' }) sdk_metadata = get_metadata(cfg) httpclient.get.return_value = client.HttpResponse(200, payload) auth_api = auth.AuthAPI(httpclient, 'some_api_key', sdk_metadata) response = auth_api.authenticate() assert response.push_enabled == True assert response.token == token call_made = httpclient.get.mock_calls[0] # validate positional arguments assert call_made[1] == ('auth', '/v2/auth', 'some_api_key') # validate key-value args (headers) assert call_made[2]['extra_headers'] == { 'SplitSDKVersion': 'python-%s' % __version__, 'SplitSDKMachineIP': '123.123.123.123', 'SplitSDKMachineName': 'some_machine_name' } httpclient.reset_mock() def raise_exception(*args, **kwargs): raise client.HttpClientException('some_message') httpclient.get.side_effect = raise_exception with pytest.raises(APIException) as exc_info: response = auth_api.authenticate() assert exc_info.type == APIException assert exc_info.value.message == 'some_message'
def _build_redis_factory(api_key, cfg): """Build and return a split factory with redis-based storage.""" sdk_metadata = util.get_metadata(cfg) redis_adapter = redis.build(cfg) cache_enabled = cfg.get('redisLocalCacheEnabled', False) cache_ttl = cfg.get('redisLocalCacheTTL', 5) storages = { 'splits': RedisSplitStorage(redis_adapter, cache_enabled, cache_ttl), 'segments': RedisSegmentStorage(redis_adapter), 'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata), 'events': RedisEventsStorage(redis_adapter, sdk_metadata), 'telemetry': RedisTelemetryStorage(redis_adapter, sdk_metadata) } return SplitFactory( api_key, storages, cfg['labelsEnabled'], ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], False, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)) )
def _build_localhost_factory(cfg): """Build and return a localhost factory for testing/development purposes.""" storages = { 'splits': InMemorySplitStorage(), 'segments': InMemorySegmentStorage(), # not used, just to avoid possible future errors. 'impressions': LocalhostImpressionsStorage(), 'events': LocalhostEventsStorage(), } synchronizers = SplitSynchronizers( LocalSplitSynchronizer(cfg['splitFile'], storages['splits']), None, None, None, None, ) tasks = SplitTasks( SplitSynchronizationTask( synchronizers.split_sync.synchronize_splits, cfg['featuresRefreshRate'], ), None, None, None, None, ) sdk_metadata = util.get_metadata(cfg) ready_event = threading.Event() synchronizer = LocalhostSynchronizer(synchronizers, tasks) manager = Manager(ready_event, synchronizer, None, False, sdk_metadata) manager.start() recorder = StandardRecorder( ImpressionsManager(cfg['impressionsMode'], True, None), storages['events'], storages['impressions'], ) return SplitFactory( 'localhost', storages, False, recorder, manager, ready_event )
def test_add_impressions_to_pipe(self, mocker): """Test that adding impressions to storage works.""" adapter = mocker.Mock(spec=RedisAdapter) metadata = get_metadata({}) storage = RedisImpressionsStorage(adapter, metadata) impressions = [ Impression('key1', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key2', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key3', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654), Impression('key4', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654) ] to_validate = [json.dumps({ 'm': { # METADATA PORTION 's': metadata.sdk_version, 'n': metadata.instance_name, 'i': metadata.instance_ip, }, 'i': { # IMPRESSION PORTION 'k': impression.matching_key, 'b': impression.bucketing_key, 'f': impression.feature_name, 't': impression.treatment, 'r': impression.label, 'c': impression.change_number, 'm': impression.time, } }) for impression in impressions] storage.add_impressions_to_pipe(impressions, adapter) assert adapter.rpush.mock_calls == [ mocker.call('SPLITIO.impressions', *to_validate) ]
def uwsgi_update_splits(user_config): """ Update splits task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) metadata = get_metadata(config) seconds = config['featuresRefreshRate'] split_sync = SplitSynchronizer( SplitsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), UWSGISplitStorage(get_uwsgi()), ) while True: try: split_sync.synchronize_splits() # pylint: disable=protected-access time.sleep(seconds) except Exception: # pylint: disable=broad-except _LOGGER.error('Error updating splits') _LOGGER.debug('Error: ', exc_info=True)
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals auth_api_base_url=None, streaming_api_base_url=None): """Build and return a split factory tailored to the supplied config.""" if not input_validator.validate_factory_instantiation(api_key): return None http_client = HttpClient( sdk_url=sdk_url, events_url=events_url, auth_url=auth_api_base_url, timeout=cfg.get('connectionTimeout') ) sdk_metadata = util.get_metadata(cfg) apis = { 'auth': AuthAPI(http_client, api_key, sdk_metadata), 'splits': SplitsAPI(http_client, api_key), 'segments': SegmentsAPI(http_client, api_key), 'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']), 'events': EventsAPI(http_client, api_key, sdk_metadata), 'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata) } if not input_validator.validate_apikey_type(apis['segments']): return None storages = { 'splits': InMemorySplitStorage(), 'segments': InMemorySegmentStorage(), 'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']), 'events': InMemoryEventStorage(cfg['eventsQueueSize']), 'telemetry': InMemoryTelemetryStorage() } imp_manager = ImpressionsManager( storages['impressions'].put, cfg['impressionsMode'], True, _wrap_impression_listener(cfg['impressionListener'], sdk_metadata)) synchronizers = SplitSynchronizers( SplitSynchronizer(apis['splits'], storages['splits']), SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']), ImpressionSynchronizer(apis['impressions'], storages['impressions'], cfg['impressionsBulkSize']), EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']), TelemetrySynchronizer(apis['telemetry'], storages['telemetry']), ImpressionsCountSynchronizer(apis['impressions'], imp_manager), ) tasks = SplitTasks( SplitSynchronizationTask( synchronizers.split_sync.synchronize_splits, cfg['featuresRefreshRate'], ), SegmentSynchronizationTask( synchronizers.segment_sync.synchronize_segments, cfg['segmentsRefreshRate'], ), ImpressionsSyncTask( synchronizers.impressions_sync.synchronize_impressions, cfg['impressionsRefreshRate'], ), EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']), TelemetrySynchronizationTask( synchronizers.telemetry_sync.synchronize_telemetry, cfg['metricsRefreshRate'], ), ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters) ) synchronizer = Synchronizer(synchronizers, tasks) sdk_ready_flag = threading.Event() manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'], streaming_api_base_url) initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer") initialization_thread.setDaemon(True) initialization_thread.start() storages['events'].set_queue_full_hook(tasks.events_task.flush) storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush) return SplitFactory(api_key, storages, cfg['labelsEnabled'], imp_manager, manager, sdk_ready_flag)