def _build_localhost_factory(cfg):
"""Build and return a localhost factory for testing/development purposes."""
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(), # not used, just to avoid possible future errors.
'impressions': LocalhostImpressionsStorage(),
'events': LocalhostEventsStorage(),
'telemetry': LocalhostTelemetryStorage()
}
synchronizers = SplitSynchronizers(
LocalSplitSynchronizer(cfg['splitFile'], storages['splits']),
None, None, None, None, None,
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
), None, None, None, None, None,
)
ready_event = threading.Event()
synchronizer = LocalhostSynchronizer(synchronizers, tasks)
manager = Manager(ready_event, synchronizer, None, False)
manager.start()
return SplitFactory(
'localhost',
storages,
False,
ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], True, None),
manager,
ready_event
)
def setup_method(self):
"""Prepare storages with test data."""
split_storage = InMemorySplitStorage()
segment_storage = InMemorySegmentStorage()
split_fn = os.path.join(os.path.dirname(__file__), 'files', 'splitChanges.json')
with open(split_fn, 'r') as flo:
data = json.loads(flo.read())
for split in data['splits']:
split_storage.put(splits.from_raw(split))
segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentEmployeesChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
segment_fn = os.path.join(os.path.dirname(__file__), 'files', 'segmentHumanBeignsChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
segment_storage.put(segments.from_raw(data))
storages = {
'splits': split_storage,
'segments': segment_storage,
'impressions': InMemoryImpressionStorage(5000),
'events': InMemoryEventStorage(5000),
'telemetry': InMemoryTelemetryStorage()
}
impmanager = ImpressionsManager(ImpressionsMode.OPTIMIZED, True)
recorder = StandardRecorder(impmanager, storages['telemetry'], storages['events'],
storages['impressions'])
self.factory = SplitFactory('some_api_key', storages, True, recorder) # pylint:disable=attribute-defined-outside-init
def _build_redis_factory(api_key, cfg):
"""Build and return a split factory with redis-based storage."""
sdk_metadata = util.get_metadata(cfg)
redis_adapter = redis.build(cfg)
cache_enabled = cfg.get('redisLocalCacheEnabled', False)
cache_ttl = cfg.get('redisLocalCacheTTL', 5)
storages = {
'splits': RedisSplitStorage(redis_adapter, cache_enabled, cache_ttl),
'segments': RedisSegmentStorage(redis_adapter),
'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata),
'events': RedisEventsStorage(redis_adapter, sdk_metadata),
}
data_sampling = cfg.get('dataSampling', DEFAULT_DATA_SAMPLING)
if data_sampling < _MIN_DEFAULT_DATA_SAMPLING_ALLOWED:
_LOGGER.warning("dataSampling cannot be less than %.2f, defaulting to minimum",
_MIN_DEFAULT_DATA_SAMPLING_ALLOWED)
data_sampling = _MIN_DEFAULT_DATA_SAMPLING_ALLOWED
recorder = PipelinedRecorder(
redis_adapter.pipeline,
ImpressionsManager(cfg['impressionsMode'], False,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata)),
storages['events'],
storages['impressions'],
data_sampling,
)
return SplitFactory(
api_key,
storages,
cfg['labelsEnabled'],
recorder,
)
def setup_method(self):
"""Prepare storages with test data."""
metadata = SdkMetadata('python-1.2.3', 'some_ip', 'some_name')
redis_client = build(DEFAULT_CONFIG.copy())
split_storage = RedisSplitStorage(redis_client, True)
segment_storage = RedisSegmentStorage(redis_client)
split_fn = os.path.join(os.path.dirname(__file__), 'files',
'splitChanges.json')
with open(split_fn, 'r') as flo:
data = json.loads(flo.read())
for split in data['splits']:
redis_client.set(split_storage._get_key(split['name']),
json.dumps(split))
redis_client.set(split_storage._SPLIT_TILL_KEY, data['till'])
segment_fn = os.path.join(os.path.dirname(__file__), 'files',
'segmentEmployeesChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
redis_client.sadd(segment_storage._get_key(data['name']),
*data['added'])
redis_client.set(segment_storage._get_till_key(data['name']),
data['till'])
segment_fn = os.path.join(os.path.dirname(__file__), 'files',
'segmentHumanBeignsChanges.json')
with open(segment_fn, 'r') as flo:
data = json.loads(flo.read())
redis_client.sadd(segment_storage._get_key(data['name']),
*data['added'])
redis_client.set(segment_storage._get_till_key(data['name']),
data['till'])
storages = {
'splits': split_storage,
'segments': segment_storage,
'impressions': RedisImpressionsStorage(redis_client, metadata),
'events': RedisEventsStorage(redis_client, metadata),
}
impmanager = ImpressionsManager(storages['impressions'].put,
ImpressionsMode.DEBUG)
recorder = PipelinedRecorder(redis_client.pipeline, impmanager,
storages['events'],
storages['impressions'])
self.factory = SplitFactory('some_api_key', storages, True, recorder) # pylint:disable=attribute-defined-outside-init
def _build_uwsgi_factory(api_key, cfg):
"""Build and return a split factory with redis-based storage."""
sdk_metadata = util.get_metadata(cfg)
uwsgi_adapter = get_uwsgi()
storages = {
'splits': UWSGISplitStorage(uwsgi_adapter),
'segments': UWSGISegmentStorage(uwsgi_adapter),
'impressions': UWSGIImpressionStorage(uwsgi_adapter),
'events': UWSGIEventStorage(uwsgi_adapter),
'telemetry': UWSGITelemetryStorage(uwsgi_adapter)
}
return SplitFactory(
api_key,
storages,
cfg['labelsEnabled'],
ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
)
def _build_redis_factory(api_key, cfg):
"""Build and return a split factory with redis-based storage."""
sdk_metadata = util.get_metadata(cfg)
redis_adapter = redis.build(cfg)
cache_enabled = cfg.get('redisLocalCacheEnabled', False)
cache_ttl = cfg.get('redisLocalCacheTTL', 5)
storages = {
'splits': RedisSplitStorage(redis_adapter, cache_enabled, cache_ttl),
'segments': RedisSegmentStorage(redis_adapter),
'impressions': RedisImpressionsStorage(redis_adapter, sdk_metadata),
'events': RedisEventsStorage(redis_adapter, sdk_metadata),
'telemetry': RedisTelemetryStorage(redis_adapter, sdk_metadata)
}
return SplitFactory(
api_key,
storages,
cfg['labelsEnabled'],
ImpressionsManager(storages['impressions'].put, cfg['impressionsMode'], False,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
)
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals
auth_api_base_url=None, streaming_api_base_url=None):
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
http_client = HttpClient(
sdk_url=sdk_url,
events_url=events_url,
auth_url=auth_api_base_url,
timeout=cfg.get('connectionTimeout')
)
sdk_metadata = util.get_metadata(cfg)
apis = {
'auth': AuthAPI(http_client, api_key, sdk_metadata),
'splits': SplitsAPI(http_client, api_key),
'segments': SegmentsAPI(http_client, api_key),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
'events': EventsAPI(http_client, api_key, sdk_metadata),
'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
'telemetry': InMemoryTelemetryStorage()
}
imp_manager = ImpressionsManager(
storages['impressions'].put,
cfg['impressionsMode'],
True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
synchronizers = SplitSynchronizers(
SplitSynchronizer(apis['splits'], storages['splits']),
SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']),
ImpressionSynchronizer(apis['impressions'], storages['impressions'],
cfg['impressionsBulkSize']),
EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']),
TelemetrySynchronizer(apis['telemetry'], storages['telemetry']),
ImpressionsCountSynchronizer(apis['impressions'], imp_manager),
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
),
SegmentSynchronizationTask(
synchronizers.segment_sync.synchronize_segments,
cfg['segmentsRefreshRate'],
),
ImpressionsSyncTask(
synchronizers.impressions_sync.synchronize_impressions,
cfg['impressionsRefreshRate'],
),
EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']),
TelemetrySynchronizationTask(
synchronizers.telemetry_sync.synchronize_telemetry,
cfg['metricsRefreshRate'],
),
ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters)
)
synchronizer = Synchronizer(synchronizers, tasks)
sdk_ready_flag = threading.Event()
manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'],
streaming_api_base_url)
initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer")
initialization_thread.setDaemon(True)
initialization_thread.start()
storages['events'].set_queue_full_hook(tasks.events_task.flush)
storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush)
return SplitFactory(api_key, storages, cfg['labelsEnabled'],
imp_manager, manager, sdk_ready_flag)