Esempio n. 1
0
    def setup_method(self):
        """Prepare storages with test data."""
        split_storage = InMemorySplitStorage()
        segment_storage = InMemorySegmentStorage()

        split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json')
        with open(split_fn, 'r') as flo:
            data = json.loads(flo.read())
        for split in data['splits']:
            split_storage.put(splits.from_raw(split))

        segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json')
        with open(segment_fn, 'r') as flo:
            data = json.loads(flo.read())
        segment_storage.put(segments.from_raw(data))

        segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentHumanBeignsChanges.json')
        with open(segment_fn, 'r') as flo:
            data = json.loads(flo.read())
        segment_storage.put(segments.from_raw(data))

        storages = {
            'splits': split_storage,
            'segments': segment_storage,
            'impressions': InMemoryImpressionStorage(5000),
            'events': InMemoryEventStorage(5000),
            'telemetry': InMemoryTelemetryStorage()
        }
        impmanager = ImpressionsManager(ImpressionsMode.OPTIMIZED, True)
        recorder = StandardRecorder(impmanager, storages['telemetry'], storages['events'],
                                    storages['impressions'])
        self.factory = SplitFactory('some_api_key', storages, True, recorder)  # pylint:disable=attribute-defined-outside-init
Esempio n. 2
0
    def setup_method(self):
        """Prepare storages with test data."""
        split_storage = InMemorySplitStorage()
        segment_storage = InMemorySegmentStorage()

        split_fn = os.path.join(os.path.dirname(__file__), 'files',
                                'splitChanges.json')
        with open(split_fn, 'r') as flo:
            data = json.loads(flo.read())
        for split in data['splits']:
            split_storage.put(splits.from_raw(split))

        segment_fn = os.path.join(os.path.dirname(__file__), 'files',
                                  'segmentEmployeesChanges.json')
        with open(segment_fn, 'r') as flo:
            data = json.loads(flo.read())
        segment_storage.put(segments.from_raw(data))

        segment_fn = os.path.join(os.path.dirname(__file__), 'files',
                                  'segmentHumanBeignsChanges.json')
        with open(segment_fn, 'r') as flo:
            data = json.loads(flo.read())
        segment_storage.put(segments.from_raw(data))

        self.factory = SplitFactory(
            {  #pylint:disable=attribute-defined-outside-init
                'splits': split_storage,
                'segments': segment_storage,
                'impressions': InMemoryImpressionStorage(5000),
                'events': InMemoryEventStorage(5000),
                'telemetry': InMemoryTelemetryStorage()
            },
            True)
Esempio n. 3
0
 def test_queue_full_hook_properties(self, mocker):
     """Test queue_full_hook is executed when the queue is full regarding properties."""
     storage = InMemoryEventStorage(200)
     queue_full_hook = mocker.Mock()
     storage.set_queue_full_hook(queue_full_hook)
     events = [EventWrapper(event=Event('key%d' % i, 'user', 'purchase', 12.5, 1, None),  size=32768) for i in range(160)]
     storage.put(events)
     assert queue_full_hook.mock_calls == [mocker.call()]
Esempio n. 4
0
 def test_queue_full_hook(self, mocker):
     """Test queue_full_hook is executed when the queue is full."""
     storage = InMemoryEventStorage(100)
     queue_full_hook = mocker.Mock()
     storage.set_queue_full_hook(queue_full_hook)
     events = [EventWrapper(event=Event('key%d' % i, 'user', 'purchase', 12.5, 321654, None),  size=1024) for i in range(0, 101)]
     storage.put(events)
     assert queue_full_hook.mock_calls == [mocker.call()]
    def test_clear(self):
        """Test clear method."""
        storage = InMemoryEventStorage(100)
        storage.put([
            EventWrapper(
                event=Event('key1', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            )
        ])

        assert storage._events.qsize() == 1
        storage.clear()
        assert storage._events.qsize() == 0
Esempio n. 6
0
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None,  # pylint:disable=too-many-arguments,too-many-locals
                             auth_api_base_url=None, streaming_api_base_url=None):
    """Build and return a split factory tailored to the supplied config."""
    if not input_validator.validate_factory_instantiation(api_key):
        return None

    http_client = HttpClient(
        sdk_url=sdk_url,
        events_url=events_url,
        auth_url=auth_api_base_url,
        timeout=cfg.get('connectionTimeout')
    )

    sdk_metadata = util.get_metadata(cfg)
    apis = {
        'auth': AuthAPI(http_client, api_key, sdk_metadata),
        'splits': SplitsAPI(http_client, api_key),
        'segments': SegmentsAPI(http_client, api_key),
        'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
        'events': EventsAPI(http_client, api_key, sdk_metadata),
        'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
    }

    if not input_validator.validate_apikey_type(apis['segments']):
        return None

    storages = {
        'splits': InMemorySplitStorage(),
        'segments': InMemorySegmentStorage(),
        'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
        'events': InMemoryEventStorage(cfg['eventsQueueSize']),
        'telemetry': InMemoryTelemetryStorage()
    }

    imp_manager = ImpressionsManager(
        storages['impressions'].put,
        cfg['impressionsMode'],
        True,
        _wrap_impression_listener(cfg['impressionListener'], sdk_metadata))

    synchronizers = SplitSynchronizers(
        SplitSynchronizer(apis['splits'], storages['splits']),
        SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']),
        ImpressionSynchronizer(apis['impressions'], storages['impressions'],
                               cfg['impressionsBulkSize']),
        EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']),
        TelemetrySynchronizer(apis['telemetry'], storages['telemetry']),
        ImpressionsCountSynchronizer(apis['impressions'], imp_manager),
    )

    tasks = SplitTasks(
        SplitSynchronizationTask(
            synchronizers.split_sync.synchronize_splits,
            cfg['featuresRefreshRate'],
        ),
        SegmentSynchronizationTask(
            synchronizers.segment_sync.synchronize_segments,
            cfg['segmentsRefreshRate'],
        ),
        ImpressionsSyncTask(
            synchronizers.impressions_sync.synchronize_impressions,
            cfg['impressionsRefreshRate'],
        ),
        EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']),
        TelemetrySynchronizationTask(
            synchronizers.telemetry_sync.synchronize_telemetry,
            cfg['metricsRefreshRate'],
        ),
        ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters)
    )

    synchronizer = Synchronizer(synchronizers, tasks)

    sdk_ready_flag = threading.Event()
    manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'],
                      streaming_api_base_url)

    initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer")
    initialization_thread.setDaemon(True)
    initialization_thread.start()

    storages['events'].set_queue_full_hook(tasks.events_task.flush)
    storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush)

    return SplitFactory(api_key, storages, cfg['labelsEnabled'],
                        imp_manager, manager, sdk_ready_flag)
    def test_push_pop_events(self):
        """Test pushing and retrieving events."""
        storage = InMemoryEventStorage(100)
        storage.put([
            EventWrapper(
                event=Event('key1', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            )
        ])
        storage.put([
            EventWrapper(
                event=Event('key2', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            )
        ])
        storage.put([
            EventWrapper(
                event=Event('key3', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            )
        ])

        # Assert impressions are retrieved in the same order they are inserted.
        assert storage.pop_many(1) == [
            Event('key1', 'user', 'purchase', 3.5, 123456, None)
        ]
        assert storage.pop_many(1) == [
            Event('key2', 'user', 'purchase', 3.5, 123456, None)
        ]
        assert storage.pop_many(1) == [
            Event('key3', 'user', 'purchase', 3.5, 123456, None)
        ]

        # Assert inserting multiple impressions at once works and maintains order.
        events = [
            EventWrapper(
                event=Event('key1', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            ),
            EventWrapper(
                event=Event('key2', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            ),
            EventWrapper(
                event=Event('key3', 'user', 'purchase', 3.5, 123456, None),
                size=1024,
            ),
        ]
        assert storage.put(events)

        # Assert events are retrieved in the same order they are inserted.
        assert storage.pop_many(1) == [
            Event('key1', 'user', 'purchase', 3.5, 123456, None)
        ]
        assert storage.pop_many(1) == [
            Event('key2', 'user', 'purchase', 3.5, 123456, None)
        ]
        assert storage.pop_many(1) == [
            Event('key3', 'user', 'purchase', 3.5, 123456, None)
        ]
Esempio n. 8
0
def _build_in_memory_factory(api_key, config, sdk_url=None, events_url=None):  # pylint: disable=too-many-locals
    """Build and return a split factory tailored to the supplied config."""
    if not input_validator.validate_factory_instantiation(api_key):
        return None

    cfg = DEFAULT_CONFIG.copy()
    cfg.update(config)
    http_client = HttpClient(sdk_url=sdk_url,
                             events_url=events_url,
                             timeout=cfg.get('connectionTimeout'))

    sdk_metadata = util.get_metadata(cfg)
    apis = {
        'splits': SplitsAPI(http_client, api_key),
        'segments': SegmentsAPI(http_client, api_key),
        'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata),
        'events': EventsAPI(http_client, api_key, sdk_metadata),
        'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
    }

    if not input_validator.validate_apikey_type(apis['segments']):
        return None

    storages = {
        'splits': InMemorySplitStorage(),
        'segments': InMemorySegmentStorage(),
        'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
        'events': InMemoryEventStorage(cfg['eventsQueueSize']),
        'telemetry': InMemoryTelemetryStorage()
    }

    # Synchronization flags
    splits_ready_flag = threading.Event()
    segments_ready_flag = threading.Event()
    sdk_ready_flag = threading.Event()

    tasks = {
        'splits':
        SplitSynchronizationTask(apis['splits'], storages['splits'],
                                 cfg['featuresRefreshRate'],
                                 splits_ready_flag),
        'segments':
        SegmentSynchronizationTask(apis['segments'], storages['segments'],
                                   storages['splits'],
                                   cfg['segmentsRefreshRate'],
                                   segments_ready_flag),
        'impressions':
        ImpressionsSyncTask(apis['impressions'], storages['impressions'],
                            cfg['impressionsRefreshRate'],
                            cfg['impressionsBulkSize']),
        'events':
        EventsSyncTask(
            apis['events'],
            storages['events'],
            cfg['eventsPushRate'],
            cfg['eventsBulkSize'],
        ),
        'telemetry':
        TelemetrySynchronizationTask(apis['telemetry'], storages['telemetry'],
                                     cfg['metricsRefreshRate'])
    }

    # Start tasks that have no dependencies
    tasks['splits'].start()
    tasks['impressions'].start()
    tasks['events'].start()
    tasks['telemetry'].start()

    storages['events'].set_queue_full_hook(tasks['events'].flush)
    storages['impressions'].set_queue_full_hook(tasks['impressions'].flush)

    def split_ready_task():
        """Wait for splits to be ready and start fetching segments."""
        splits_ready_flag.wait()
        tasks['segments'].start()

    def segment_ready_task():
        """Wait for segments to be ready and set the main ready flag."""
        segments_ready_flag.wait()
        sdk_ready_flag.set()

    split_completion_thread = threading.Thread(target=split_ready_task)
    split_completion_thread.setDaemon(True)
    split_completion_thread.start()
    segment_completion_thread = threading.Thread(target=segment_ready_task)
    segment_completion_thread.setDaemon(True)
    segment_completion_thread.start()
    return SplitFactory(api_key,
                        storages,
                        cfg['labelsEnabled'],
                        apis,
                        tasks,
                        sdk_ready_flag,
                        impression_listener=_wrap_impression_listener(
                            cfg['impressionListener'], sdk_metadata))