def uwsgi_update_segments(user_config): """ Update segments task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) seconds = config['segmentsRefreshRate'] segment_sync_task = SegmentSynchronizationTask( SegmentsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey']), UWSGISegmentStorage(get_uwsgi()), None, # Split sotrage not needed, segments provided manually, None, # Period not needed, task executed manually None # Flag not needed, never consumed or set. ) pool = workerpool.WorkerPool(20, segment_sync_task._update_segment) #pylint: disable=protected-access pool.start() split_storage = UWSGISplitStorage(get_uwsgi()) while True: try: for split in split_storage.get_all_splits(): for segment_name in split.get_segment_names(): pool.submit_work(segment_name) time.sleep(seconds) except Exception: #pylint: disable=broad-except _LOGGER.error('Error updating segments') _LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_segments(user_config): """ Update segments task. :param user_config: User-provided configuration. :type user_config: dict """ config = _get_config(user_config) seconds = config['segmentsRefreshRate'] metadata = get_metadata(config) segment_sync = SegmentSynchronizer( SegmentsAPI( HttpClient(1500, config.get('sdk_url'), config.get('events_url')), config['apikey'], metadata), UWSGISplitStorage(get_uwsgi()), UWSGISegmentStorage(get_uwsgi()), ) pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access pool.start() split_storage = UWSGISplitStorage(get_uwsgi()) while True: try: for segment_name in split_storage.get_segment_names(): pool.submit_work(segment_name) time.sleep(seconds) except Exception: # pylint: disable=broad-except _LOGGER.error('Error updating segments') _LOGGER.debug('Error: ', exc_info=True)
def test_failure_in_message_doesnt_breal(self, mocker): """Test that if a message cannot be parsed it is ignored and others are processed.""" class Worker: def __init__(self): self._worked = set() def do_work(self, w): if w == '55': raise Exception('something') self._worked.add(w) worker = Worker() wp = workerpool.WorkerPool(50, worker.do_work) wp.start() for x in range(0, 100): wp.submit_work(str(x)) stop_event = threading.Event() wp.stop(stop_event) stop_event.wait(5) assert stop_event.is_set() for x in range(0, 100): if x != 55: assert str(x) in worker._worked else: assert str(x) not in worker._worked
def test_fail_in_msg_doesnt_break(self): """Test that if a message cannot be parsed it is ignored and others are processed.""" class Worker(object): #pylint: disable= def __init__(self): self.worked = set() def do_work(self, work): if work == '55': raise Exception('something') self.worked.add(work) worker = Worker() wpool = workerpool.WorkerPool(50, worker.do_work) wpool.start() for num in range(0, 100): wpool.submit_work(str(num)) stop_event = threading.Event() wpool.stop(stop_event) stop_event.wait(5) assert stop_event.is_set() for num in range(0, 100): if num != 55: assert str(num) in worker.worked else: assert str(num) not in worker.worked
def recreate(self): """ Create worker_pool on forked processes. """ self._worker_pool = workerpool.WorkerPool(10, self.synchronize_segment) self._worker_pool.start()
def test_normal_operation(self, mocker): """Test normal opeation works properly.""" worker_func = mocker.Mock() wp = workerpool.WorkerPool(10, worker_func) wp.start() for x in range(0, 100): wp.submit_work(str(x)) stop_event = threading.Event() wp.stop(stop_event) stop_event.wait(5) assert stop_event.is_set() calls = worker_func.mock_calls for x in range(0, 100): assert mocker.call(str(x)) in calls
def test_msg_acked_after_processed(self): """Test that events are only set after all the work in the pipeline is done.""" class Worker(object): def __init__(self): self.worked = set() def do_work(self, work): self.worked.add(work) time.sleep(0.02) # will wait 2 seconds in total for 100 elements worker = Worker() wpool = workerpool.WorkerPool(50, worker.do_work) wpool.start() for num in range(0, 100): wpool.submit_work(str(num)) wpool.wait_for_completion() assert len(worker.worked) == 100
def __init__(self, segment_api, split_storage, segment_storage): """ Class constructor. :param segment_api: API to retrieve segments from backend. :type segment_api: splitio.api.SegmentApi :param split_storage: Split Storage. :type split_storage: splitio.storage.InMemorySplitStorage :param segment_storage: Segment storage reference. :type segment_storage: splitio.storage.SegmentStorage """ self._api = segment_api self._split_storage = split_storage self._segment_storage = segment_storage self._worker_pool = workerpool.WorkerPool(10, self.synchronize_segment) self._worker_pool.start()
def __init__(self, segment_api, segment_storage, split_storage, period, event): #pylint: disable=too-many-arguments """ Clas constructor. :param segment_api: API to retrieve segments from backend. :type segment_api: splitio.api.SegmentApi :param segment_storage: Segment storage reference. :type segment_storage: splitio.storage.SegmentStorage :param event: Event to signal when all segments have finished initial sync. :type event: threading.Event """ self._logger = logging.getLogger(self.__class__.__name__) self._worker_pool = workerpool.WorkerPool(20, self._ensure_segment_is_updated) self._task = asynctask.AsyncTask(self._main, period, on_init=self._on_init) self._segment_api = segment_api self._segment_storage = segment_storage self._split_storage = split_storage self._event = event self._pending_initialization = []