def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
metadata = get_metadata(config)
segment_sync = SegmentSynchronizer(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
UWSGISplitStorage(get_uwsgi()),
UWSGISegmentStorage(get_uwsgi()),
)
pool = workerpool.WorkerPool(20, segment_sync.synchronize_segment) # pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for segment_name in split_storage.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_report_impressions(user_config):
"""
Flush impressions task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config['impressionsRefreshRate']
storage = UWSGIImpressionStorage(get_uwsgi())
impressions_sync = ImpressionSynchronizer(
ImpressionsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata, config['impressionsMode']), storage,
config['impressionsBulkSize'])
while True:
try:
impressions_sync.synchronize_impressions() # pylint: disable=protected-access
for _ in range(0, seconds):
if storage.should_flush():
storage.acknowledge_flush()
break
time.sleep(1)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error posting impressions')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_splits(user_config):
"""
Update splits task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['featuresRefreshRate']
split_sync_task = SplitSynchronizationTask(
SplitsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISplitStorage(get_uwsgi()),
None, # Time not needed since the task will be triggered manually.
None # Ready flag not needed since it will never be set and consumed.
)
while True:
try:
split_sync_task._update_splits() #pylint: disable=protected-access
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error updating splits')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_segments(user_config):
"""
Update segments task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['segmentsRefreshRate']
segment_sync_task = SegmentSynchronizationTask(
SegmentsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISegmentStorage(get_uwsgi()),
None, # Split sotrage not needed, segments provided manually,
None, # Period not needed, task executed manually
None # Flag not needed, never consumed or set.
)
pool = workerpool.WorkerPool(20, segment_sync_task._update_segment) #pylint: disable=protected-access
pool.start()
split_storage = UWSGISplitStorage(get_uwsgi())
while True:
try:
for split in split_storage.get_all_splits():
for segment_name in split.get_segment_names():
pool.submit_work(segment_name)
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error updating segments')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_report_telemetry(user_config):
"""
Flush events task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config.get('metricsRefreshRate', 30)
storage = UWSGITelemetryStorage(get_uwsgi())
task = TelemetrySynchronizationTask(
TelemetryAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
storage,
None, # Period not needed. Task is being triggered manually.
)
while True:
try:
task._flush_telemetry() #pylint: disable=protected-access
time.sleep(seconds)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error posting metrics')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_report_events(user_config):
"""
Flush events task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
metadata = get_metadata(config)
seconds = config.get('eventsRefreshRate', 30)
storage = UWSGIEventStorage(get_uwsgi())
task = EventsSyncTask(
EventsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey'], metadata),
storage,
None, # Period not needed. Task is being triggered manually.
config['eventsBulkSize'])
while True:
try:
task._send_events() #pylint: disable=protected-access
for _ in xrange(0, seconds):
if storage.should_flush():
storage.acknowledge_flush()
break
time.sleep(1)
except Exception: #pylint: disable=broad-except
_LOGGER.error('Error posting metrics')
_LOGGER.debug('Error: ', exc_info=True)
def uwsgi_update_splits(user_config):
"""
Update splits task.
:param user_config: User-provided configuration.
:type user_config: dict
"""
config = _get_config(user_config)
seconds = config['featuresRefreshRate']
split_sync = SplitSynchronizer(
SplitsAPI(
HttpClient(1500, config.get('sdk_url'), config.get('events_url')),
config['apikey']),
UWSGISplitStorage(get_uwsgi()),
)
while True:
try:
split_sync.synchronize_splits() # pylint: disable=protected-access
time.sleep(seconds)
except Exception: # pylint: disable=broad-except
_LOGGER.error('Error updating splits')
_LOGGER.debug('Error: ', exc_info=True)
def _build_in_memory_factory(api_key, cfg, sdk_url=None, events_url=None, # pylint:disable=too-many-arguments,too-many-locals
auth_api_base_url=None, streaming_api_base_url=None):
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
http_client = HttpClient(
sdk_url=sdk_url,
events_url=events_url,
auth_url=auth_api_base_url,
timeout=cfg.get('connectionTimeout')
)
sdk_metadata = util.get_metadata(cfg)
apis = {
'auth': AuthAPI(http_client, api_key, sdk_metadata),
'splits': SplitsAPI(http_client, api_key),
'segments': SegmentsAPI(http_client, api_key),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata, cfg['impressionsMode']),
'events': EventsAPI(http_client, api_key, sdk_metadata),
'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
'telemetry': InMemoryTelemetryStorage()
}
imp_manager = ImpressionsManager(
storages['impressions'].put,
cfg['impressionsMode'],
True,
_wrap_impression_listener(cfg['impressionListener'], sdk_metadata))
synchronizers = SplitSynchronizers(
SplitSynchronizer(apis['splits'], storages['splits']),
SegmentSynchronizer(apis['segments'], storages['splits'], storages['segments']),
ImpressionSynchronizer(apis['impressions'], storages['impressions'],
cfg['impressionsBulkSize']),
EventSynchronizer(apis['events'], storages['events'], cfg['eventsBulkSize']),
TelemetrySynchronizer(apis['telemetry'], storages['telemetry']),
ImpressionsCountSynchronizer(apis['impressions'], imp_manager),
)
tasks = SplitTasks(
SplitSynchronizationTask(
synchronizers.split_sync.synchronize_splits,
cfg['featuresRefreshRate'],
),
SegmentSynchronizationTask(
synchronizers.segment_sync.synchronize_segments,
cfg['segmentsRefreshRate'],
),
ImpressionsSyncTask(
synchronizers.impressions_sync.synchronize_impressions,
cfg['impressionsRefreshRate'],
),
EventsSyncTask(synchronizers.events_sync.synchronize_events, cfg['eventsPushRate']),
TelemetrySynchronizationTask(
synchronizers.telemetry_sync.synchronize_telemetry,
cfg['metricsRefreshRate'],
),
ImpressionsCountSyncTask(synchronizers.impressions_count_sync.synchronize_counters)
)
synchronizer = Synchronizer(synchronizers, tasks)
sdk_ready_flag = threading.Event()
manager = Manager(sdk_ready_flag, synchronizer, apis['auth'], cfg['streamingEnabled'],
streaming_api_base_url)
initialization_thread = threading.Thread(target=manager.start, name="SDKInitializer")
initialization_thread.setDaemon(True)
initialization_thread.start()
storages['events'].set_queue_full_hook(tasks.events_task.flush)
storages['impressions'].set_queue_full_hook(tasks.impressions_task.flush)
return SplitFactory(api_key, storages, cfg['labelsEnabled'],
imp_manager, manager, sdk_ready_flag)
def _build_in_memory_factory(api_key, config, sdk_url=None, events_url=None): # pylint: disable=too-many-locals
"""Build and return a split factory tailored to the supplied config."""
if not input_validator.validate_factory_instantiation(api_key):
return None
cfg = DEFAULT_CONFIG.copy()
cfg.update(config)
http_client = HttpClient(sdk_url=sdk_url,
events_url=events_url,
timeout=cfg.get('connectionTimeout'))
sdk_metadata = util.get_metadata(cfg)
apis = {
'splits': SplitsAPI(http_client, api_key),
'segments': SegmentsAPI(http_client, api_key),
'impressions': ImpressionsAPI(http_client, api_key, sdk_metadata),
'events': EventsAPI(http_client, api_key, sdk_metadata),
'telemetry': TelemetryAPI(http_client, api_key, sdk_metadata)
}
if not input_validator.validate_apikey_type(apis['segments']):
return None
storages = {
'splits': InMemorySplitStorage(),
'segments': InMemorySegmentStorage(),
'impressions': InMemoryImpressionStorage(cfg['impressionsQueueSize']),
'events': InMemoryEventStorage(cfg['eventsQueueSize']),
'telemetry': InMemoryTelemetryStorage()
}
# Synchronization flags
splits_ready_flag = threading.Event()
segments_ready_flag = threading.Event()
sdk_ready_flag = threading.Event()
tasks = {
'splits':
SplitSynchronizationTask(apis['splits'], storages['splits'],
cfg['featuresRefreshRate'],
splits_ready_flag),
'segments':
SegmentSynchronizationTask(apis['segments'], storages['segments'],
storages['splits'],
cfg['segmentsRefreshRate'],
segments_ready_flag),
'impressions':
ImpressionsSyncTask(apis['impressions'], storages['impressions'],
cfg['impressionsRefreshRate'],
cfg['impressionsBulkSize']),
'events':
EventsSyncTask(
apis['events'],
storages['events'],
cfg['eventsPushRate'],
cfg['eventsBulkSize'],
),
'telemetry':
TelemetrySynchronizationTask(apis['telemetry'], storages['telemetry'],
cfg['metricsRefreshRate'])
}
# Start tasks that have no dependencies
tasks['splits'].start()
tasks['impressions'].start()
tasks['events'].start()
tasks['telemetry'].start()
storages['events'].set_queue_full_hook(tasks['events'].flush)
storages['impressions'].set_queue_full_hook(tasks['impressions'].flush)
def split_ready_task():
"""Wait for splits to be ready and start fetching segments."""
splits_ready_flag.wait()
tasks['segments'].start()
def segment_ready_task():
"""Wait for segments to be ready and set the main ready flag."""
segments_ready_flag.wait()
sdk_ready_flag.set()
split_completion_thread = threading.Thread(target=split_ready_task)
split_completion_thread.setDaemon(True)
split_completion_thread.start()
segment_completion_thread = threading.Thread(target=segment_ready_task)
segment_completion_thread.setDaemon(True)
segment_completion_thread.start()
return SplitFactory(api_key,
storages,
cfg['labelsEnabled'],
apis,
tasks,
sdk_ready_flag,
impression_listener=_wrap_impression_listener(
cfg['impressionListener'], sdk_metadata))