def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
segment_sync_task = SegmentSynchronizationTask(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISegmentStorage(get_uwsgi()),
None, # Split sotrage not needed, segments provided manually,
None, # Period not needed, task executed manually
None # Flag not needed, never consumed or set.
)
pool = workerpool.WorkerPool(20, segment_sync_task._update_segment) #pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for split in split_storage.get_all_splits():
for segment_name in split.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
metadata = get_metadata(config)
segment_sync = SegmentSynchronizer(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
UWSGISplitStorage(get_uwsgi()),
UWSGISegmentStorage(get_uwsgi()),
)
pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for segment_name in split_storage.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def test_gauges(self):
"""Test storing and popping gauges."""
storage = UWSGITelemetryStorage(get_uwsgi(True))
storage.put_gauge('some_gauge1', 123)
storage.put_gauge('some_gauge2', 456)
assert storage.pop_gauges() == {'some_gauge1': 123, 'some_gauge2': 456}
assert storage.pop_gauges() == {}
def test_get_all_splits(self, mocker):
"""Test fetching all splits."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
split_1 = mocker.Mock(spec=Split)
split_1.to_json.return_value = '{"name": "some_split_1"}'
split_name_1 = mocker.PropertyMock()
split_name_1.return_value = 'some_split_1'
type(split_1).name = split_name_1
split_2 = mocker.Mock(spec=Split)
split_2.to_json.return_value = '{"name": "some_split_2"}'
split_name_2 = mocker.PropertyMock()
split_name_2.return_value = 'some_split_2'
type(split_2).name = split_name_2
def _from_raw_mock(split_json):
split_mock = mocker.Mock(spec=Split)
name = mocker.PropertyMock()
name.return_value = json.loads(split_json)['name']
type(split_mock).name = name
return split_mock
mocker.patch('splitio.storage.uwsgi.splits.from_raw', new=_from_raw_mock)
storage.put(split_1)
storage.put(split_2)
splits = storage.get_all_splits()
s1 = next(split for split in splits if split.name == 'some_split_1')
s2 = next(split for split in splits if split.name == 'some_split_2')
def test_get_set_change_number(self, mocker):
"""Test setting and getting change number."""
uwsgi = get_uwsgi(True)
storage = UWSGISegmentStorage(uwsgi)
assert storage.get_change_number('some_segment') is None
storage.set_change_number('some_segment', 123)
assert storage.get_change_number('some_segment') == 123
def test_put_pop_events(self, mocker):
"""Test storing and fetching events."""
uwsgi = get_uwsgi(True)
storage = UWSGIEventStorage(uwsgi)
events = [
EventWrapper(event=Event('key1', 'user', 'purchase', 10, 123456,
None),
size=32768),
EventWrapper(event=Event('key2', 'user', 'purchase', 10, 123456,
None),
size=32768),
EventWrapper(event=Event('key3', 'user', 'purchase', 10, 123456,
None),
size=32768),
EventWrapper(event=Event('key4', 'user', 'purchase', 10, 123456,
None),
size=32768),
]
storage.put(events)
res = storage.pop_many(10)
assert res == [
Event('key1', 'user', 'purchase', 10, 123456, None),
Event('key2', 'user', 'purchase', 10, 123456, None),
Event('key3', 'user', 'purchase', 10, 123456, None),
Event('key4', 'user', 'purchase', 10, 123456, None)
]
def test_store_retrieve_segment(self, mocker):
"""Test storing and fetching segments."""
uwsgi = get_uwsgi(True)
storage = UWSGISegmentStorage(uwsgi)
segment = mocker.Mock(spec=Segment)
segment_keys = mocker.PropertyMock()
segment_keys.return_value = ['abc']
type(segment).keys = segment_keys
segment.to_json = {}
segment_name = mocker.PropertyMock()
segment_name.return_value = 'some_segment'
segment_change_number = mocker.PropertyMock()
segment_change_number.return_value = 123
type(segment).name = segment_name
type(segment).change_number = segment_change_number
from_raw_mock = mocker.Mock()
from_raw_mock.return_value = 'ok'
mocker.patch('splitio.models.segments.from_raw', new=from_raw_mock)
storage.put(segment)
assert storage.get('some_segment') == 'ok'
assert from_raw_mock.mock_calls == [
mocker.call({
'till': 123,
'removed': [],
'added': [u'abc'],
'name': 'some_segment'
})
]
assert storage.get('nonexistant-segment') is None
def uwsgi_report_events(user_config):
"""
Flush events task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config.get('eventsRefreshRate', 30)
storage = UWSGIEventStorage(get_uwsgi())
task = EventsSyncTask(
EventsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
storage,
None, # Period not needed. Task is being triggered manually.
config['eventsBulkSize'])
while True:
try:
task._send_events() #pylint: disable=protected-access
for _ in xrange(0, seconds):
if storage.should_flush():
storage.acknowledge_flush()
break
time.sleep(1)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error posting metrics')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_report_telemetry(user_config):
"""
Flush events task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config.get('metricsRefreshRate', 30)
storage = UWSGITelemetryStorage(get_uwsgi())
task = TelemetrySynchronizationTask(
TelemetryAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
storage,
None, # Period not needed. Task is being triggered manually.
)
while True:
try:
task._flush_telemetry() #pylint: disable=protected-access
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error posting metrics')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_splits(user_config):
"""
Update splits task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['featuresRefreshRate']
split_sync_task = SplitSynchronizationTask(
SplitsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISplitStorage(get_uwsgi()),
None, # Time not needed since the task will be triggered manually.
None # Ready flag not needed since it will never be set and consumed.
)
while True:
try:
split_sync_task._update_splits() #pylint: disable=protected-access
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error updating splits')
_LOGGER.debug('Error: ', exc_info=True)
def test_is_valid_traffic_type(self, mocker):
"""Test that traffic type validation works properly."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
from_raw_mock = self._get_from_raw_mock(mocker)
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
split_1 = from_raw_mock({'name': 'some_split_1', 'trafficTypeName': 'user'})
storage.put(split_1)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is False
split_2 = from_raw_mock({'name': 'some_split_2', 'trafficTypeName': 'account'})
storage.put(split_2)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
split_3 = from_raw_mock({'name': 'some_split_3', 'trafficTypeName': 'user'})
storage.put(split_3)
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
storage.remove('some_split_1')
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is True
storage.remove('some_split_2')
assert storage.is_valid_traffic_type('user') is True
assert storage.is_valid_traffic_type('account') is False
storage.remove('some_split_3')
assert storage.is_valid_traffic_type('user') is False
assert storage.is_valid_traffic_type('account') is False
def uwsgi_report_impressions(user_config):
"""
Flush impressions task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config['impressionsRefreshRate']
storage = UWSGIImpressionStorage(get_uwsgi())
impressions_sync = ImpressionSynchronizer(
ImpressionsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata, config['impressionsMode']), storage,
config['impressionsBulkSize'])
while True:
try:
impressions_sync.synchronize_impressions() # pylint: disable=protected-access
for _ in range(0, seconds):
if storage.should_flush():
storage.acknowledge_flush()
break
time.sleep(1)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error posting impressions')
_LOGGER.debug('Error: ', exc_info=True)
def test_set_get_changenumber(self, mocker):
"""Test setting and retrieving changenumber."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
assert storage.get_change_number() == None
storage.set_change_number(123)
assert storage.get_change_number() == 123
def test_counters(self):
"""Test storing and popping counters."""
storage = UWSGITelemetryStorage(get_uwsgi(True))
storage.inc_counter('some_counter')
storage.inc_counter('some_counter')
storage.inc_counter('some_counter')
assert storage.pop_counters() == {'some_counter': 3}
assert storage.pop_counters() == {}
def test_flush(self):
"""Test requesting, querying and acknowledging a flush."""
uwsgi = get_uwsgi(True)
storage = UWSGIImpressionStorage(uwsgi)
assert storage.should_flush() is False
storage.request_flush()
assert storage.should_flush() is True
storage.acknowledge_flush()
assert storage.should_flush() is False
def test_latencies(self):
"""Test storing and popping latencies."""
storage = UWSGITelemetryStorage(get_uwsgi(True))
storage.inc_latency('some_latency', 2)
storage.inc_latency('some_latency', 2)
storage.inc_latency('some_latency', 2)
assert storage.pop_latencies() == {
'some_latency': [0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
}
assert storage.pop_latencies() == {}
def test_put_pop_impressions(self, mocker):
"""Test storing and fetching impressions."""
uwsgi = get_uwsgi(True)
storage = UWSGIImpressionStorage(uwsgi)
impressions = [
Impression('key1', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654),
Impression('key2', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654),
Impression('key3', 'feature2', 'on', 'some_label', 123456, 'buck1', 321654),
Impression('key4', 'feature1', 'on', 'some_label', 123456, 'buck1', 321654)
]
storage.put(impressions)
res = storage.pop_many(10)
assert res == impressions
def test_get_split_names(self, mocker):
"""Test getting all split names."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
from_raw_mock = self._get_from_raw_mock(mocker)
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
split_1 = from_raw_mock({'name': 'some_split_1', 'trafficTypeName': 'user'})
split_2 = from_raw_mock({'name': 'some_split_2', 'trafficTypeName': 'user'})
storage.put(split_1)
storage.put(split_2)
assert set(storage.get_split_names()) == set(['some_split_1', 'some_split_2'])
storage.remove('some_split_1')
assert storage.get_split_names() == ['some_split_2']
def _build_uwsgi_factory(config):
"""Build and return a split factory with redis-based storage."""
cfg = DEFAULT_CONFIG.copy()
cfg.update(config)
sdk_metadata = util.get_metadata(cfg)
uwsgi_adapter = get_uwsgi()
storages = {
'splits': UWSGISplitStorage(uwsgi_adapter),
'segments': UWSGISegmentStorage(uwsgi_adapter),
'impressions': UWSGIImpressionStorage(uwsgi_adapter),
'events': UWSGIEventStorage(uwsgi_adapter),
'telemetry': UWSGITelemetryStorage(uwsgi_adapter)
}
return SplitFactory(storages,
cfg['labelsEnabled'],
impression_listener=_wrap_impression_listener(
cfg['impressionListener'], sdk_metadata))
def test_get_splits(self, mocker):
"""Test retrieving a list of passed splits."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
from_raw_mock = self._get_from_raw_mock(mocker)
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
split_1 = from_raw_mock({'name': 'some_split_1', 'trafficTypeName': 'user'})
split_2 = from_raw_mock({'name': 'some_split_2', 'trafficTypeName': 'user'})
storage.put(split_1)
storage.put(split_2)
splits = storage.fetch_many(['some_split_1', 'some_split_2', 'some_split_3'])
assert len(splits) == 3
assert splits['some_split_1'].name == 'some_split_1'
assert splits['some_split_2'].name == 'some_split_2'
assert 'some_split_3' in splits
def test_get_all_splits(self, mocker):
"""Test fetching all splits."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
from_raw_mock = self._get_from_raw_mock(mocker)
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
split_1 = from_raw_mock({'name': 'some_split_1', 'trafficTypeName': 'user'})
split_2 = from_raw_mock({'name': 'some_split_2', 'trafficTypeName': 'user'})
storage.put(split_1)
storage.put(split_2)
splits = storage.get_all_splits()
s1 = next(split for split in splits if split.name == 'some_split_1')
s2 = next(split for split in splits if split.name == 'some_split_2')
assert s1.traffic_type_name == 'user'
assert s2.traffic_type_name == 'user'
def _build_uwsgi_factory(api_key, cfg):
"""Build and return a split factory with redis-based storage."""
sdk_metadata = util.get_metadata(cfg)
uwsgi_adapter = get_uwsgi()
storages = {
'splits': UWSGISplitStorage(uwsgi_adapter),
'segments': UWSGISegmentStorage(uwsgi_adapter),
'impressions': UWSGIImpressionStorage(uwsgi_adapter),
'events': UWSGIEventStorage(uwsgi_adapter),
'telemetry': UWSGITelemetryStorage(uwsgi_adapter)
}
return SplitFactory(
api_key,
storages,
cfg['labelsEnabled'],
ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
)
def test_get_split_names(self, mocker):
"""Test getting all split names."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
split_1 = mocker.Mock(spec=Split)
split_1.to_json.return_value = '{"name": "split1"}'
split_name_1 = mocker.PropertyMock()
split_name_1.return_value = 'some_split_1'
type(split_1).name = split_name_1
split_2 = mocker.Mock(spec=Split)
split_2.to_json.return_value = '{"name": "split2"}'
split_name_2 = mocker.PropertyMock()
split_name_2.return_value = 'some_split_2'
type(split_2).name = split_name_2
storage.put(split_1)
storage.put(split_2)
assert set(storage.get_split_names()) == set(['some_split_1', 'some_split_2'])
storage.remove('some_split_1')
assert storage.get_split_names() == ['some_split_2']
def test_kill_locally(self):
"""Test kill local."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
split = Split('some_split', 123456789, False, 'some', 'traffic_type',
'ACTIVE', 1)
storage.put(split)
storage.set_change_number(1)
storage.kill_locally('test', 'default_treatment', 2)
assert storage.get('test') is None
storage.kill_locally('some_split', 'default_treatment', 0)
assert storage.get('some_split').change_number == 1
assert storage.get('some_split').killed is False
assert storage.get('some_split').default_treatment == 'some'
storage.kill_locally('some_split', 'default_treatment', 3)
assert storage.get('some_split').change_number == 3
def test_store_retrieve_split(self, mocker):
"""Test storing and retrieving splits."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
from_raw_mock = self._get_from_raw_mock(mocker)
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
raw_split = {'name': 'some_split', 'trafficTypeName': 'user'}
split = from_raw_mock(raw_split)
from_raw_mock.reset_mock() # clear mock calls so they don't interfere with the testing itself.
storage.put(split)
retrieved = storage.get('some_split')
assert retrieved.name == split.name and retrieved.traffic_type_name == split.traffic_type_name
assert from_raw_mock.mock_calls == [mocker.call(raw_split)]
assert split.to_json.mock_calls == [mocker.call()]
assert storage.get('nonexistant_split') is None
storage.remove('some_split')
assert storage.get('some_split') == None
def test_segment_contains(self, mocker):
"""Test that segment contains works properly."""
uwsgi = get_uwsgi(True)
storage = UWSGISegmentStorage(uwsgi)
from_raw_mock = mocker.Mock()
from_raw_mock.return_value = Segment('some_segment', ['abc'], 123)
mocker.patch('splitio.models.segments.from_raw', new=from_raw_mock)
segment = mocker.Mock(spec=Segment)
segment_keys = mocker.PropertyMock()
segment_keys.return_value = ['abc']
type(segment).keys = segment_keys
segment.to_json = {}
segment_name = mocker.PropertyMock()
segment_name.return_value = 'some_segment'
segment_change_number = mocker.PropertyMock()
segment_change_number.return_value = 123
type(segment).name = segment_name
type(segment).change_number = segment_change_number
storage.put(segment)
assert storage.segment_contains('some_segment', 'abc')
assert not storage.segment_contains('some_segment', 'qwe')
def uwsgi_update_splits(user_config):
"""
Update splits task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['featuresRefreshRate']
split_sync = SplitSynchronizer(
SplitsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISplitStorage(get_uwsgi()),
)
while True:
try:
split_sync.synchronize_splits() # pylint: disable=protected-access
time.sleep(seconds)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error updating splits')
_LOGGER.debug('Error: ', exc_info=True)
def test_store_retrieve_split(self, mocker):
"""Test storing and retrieving splits."""
uwsgi = get_uwsgi(True)
storage = UWSGISplitStorage(uwsgi)
split = mocker.Mock(spec=Split)
split.to_json.return_value = '{}'
split_name = mocker.PropertyMock()
split_name.return_value = 'some_split'
type(split).name = split_name
storage.put(split)
from_raw_mock = mocker.Mock()
from_raw_mock.return_value = 'ok'
mocker.patch('splitio.models.splits.from_raw', new=from_raw_mock)
retrieved = storage.get('some_split')
assert retrieved == 'ok'
assert from_raw_mock.mock_calls == [mocker.call('{}')]
assert split.to_json.mock_calls == [mocker.call()]
assert storage.get('nonexistant_split') is None
storage.remove('some_split')
assert storage.get('some_split') == None
