def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
segment_sync_task = SegmentSynchronizationTask(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISegmentStorage(get_uwsgi()),
None, # Split sotrage not needed, segments provided manually,
None, # Period not needed, task executed manually
None # Flag not needed, never consumed or set.
)
pool = workerpool.WorkerPool(20, segment_sync_task._update_segment) #pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for split in split_storage.get_all_splits():
for segment_name in split.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
metadata = get_metadata(config)
segment_sync = SegmentSynchronizer(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
UWSGISplitStorage(get_uwsgi()),
UWSGISegmentStorage(get_uwsgi()),
)
pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for segment_name in split_storage.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def test_failure_in_message_doesnt_breal(self, mocker):
"""Test that if a message cannot be parsed it is ignored and others are processed."""
class Worker:
def __init__(self):
self._worked = set()
def do_work(self, w):
if w == '55':
raise Exception('something')
self._worked.add(w)
worker = Worker()
wp = workerpool.WorkerPool(50, worker.do_work)
wp.start()
for x in range(0, 100):
wp.submit_work(str(x))
stop_event = threading.Event()
wp.stop(stop_event)
stop_event.wait(5)
assert stop_event.is_set()
for x in range(0, 100):
if x != 55:
assert str(x) in worker._worked
else:
assert str(x) not in worker._worked
def test_fail_in_msg_doesnt_break(self):
"""Test that if a message cannot be parsed it is ignored and others are processed."""
class Worker(object): #pylint: disable=
def __init__(self):
self.worked = set()
def do_work(self, work):
if work == '55':
raise Exception('something')
self.worked.add(work)
worker = Worker()
wpool = workerpool.WorkerPool(50, worker.do_work)
wpool.start()
for num in range(0, 100):
wpool.submit_work(str(num))
stop_event = threading.Event()
wpool.stop(stop_event)
stop_event.wait(5)
assert stop_event.is_set()
for num in range(0, 100):
if num != 55:
assert str(num) in worker.worked
else:
assert str(num) not in worker.worked
def recreate(self):
"""
Create worker_pool on forked processes.
"""
self._worker_pool = workerpool.WorkerPool(10, self.synchronize_segment)
self._worker_pool.start()
def test_normal_operation(self, mocker):
"""Test normal opeation works properly."""
worker_func = mocker.Mock()
wp = workerpool.WorkerPool(10, worker_func)
wp.start()
for x in range(0, 100):
wp.submit_work(str(x))
stop_event = threading.Event()
wp.stop(stop_event)
stop_event.wait(5)
assert stop_event.is_set()
calls = worker_func.mock_calls
for x in range(0, 100):
assert mocker.call(str(x)) in calls
def test_msg_acked_after_processed(self):
"""Test that events are only set after all the work in the pipeline is done."""
class Worker(object):
def __init__(self):
self.worked = set()
def do_work(self, work):
self.worked.add(work)
time.sleep(0.02) # will wait 2 seconds in total for 100 elements
worker = Worker()
wpool = workerpool.WorkerPool(50, worker.do_work)
wpool.start()
for num in range(0, 100):
wpool.submit_work(str(num))
wpool.wait_for_completion()
assert len(worker.worked) == 100
def __init__(self, segment_api, split_storage, segment_storage):
"""
Class constructor.
:param segment_api: API to retrieve segments from backend.
:type segment_api: splitio.api.SegmentApi
:param split_storage: Split Storage.
:type split_storage: splitio.storage.InMemorySplitStorage
:param segment_storage: Segment storage reference.
:type segment_storage: splitio.storage.SegmentStorage
"""
self._api = segment_api
self._split_storage = split_storage
self._segment_storage = segment_storage
self._worker_pool = workerpool.WorkerPool(10, self.synchronize_segment)
self._worker_pool.start()
def __init__(self, segment_api, segment_storage, split_storage, period, event): #pylint: disable=too-many-arguments
"""
Clas constructor.
:param segment_api: API to retrieve segments from backend.
:type segment_api: splitio.api.SegmentApi
:param segment_storage: Segment storage reference.
:type segment_storage: splitio.storage.SegmentStorage
:param event: Event to signal when all segments have finished initial sync.
:type event: threading.Event
"""
self._logger = logging.getLogger(self.__class__.__name__)
self._worker_pool = workerpool.WorkerPool(20, self._ensure_segment_is_updated)
self._task = asynctask.AsyncTask(self._main, period, on_init=self._on_init)
self._segment_api = segment_api
self._segment_storage = segment_storage
self._split_storage = split_storage
self._event = event
self._pending_initialization = []
