def __init__(self, config_file):
"""
This is the base class for a Panoptes to InfluxDB consumer. It reads metrics from Kafka and sends them to
InfluxDB write api.
This class defines default implementations of converting Panoptes Metric Group to InfluxDB line protocol
string format.
measurement,tag1=text1,tag2=text2 field1=val1,field2=val2 timestamp
It is expected that users would inherit and extend these methods if they wish to intercept
the data in any way before sending to InfluxDB.
The emission logic has fault tolerance and retries built in.
Args:
config_file (str): The name of config file
Attributes:
CONFIG_SPEC_FILE: The configspec file to use with configobj - please don't change this till you know what
you're doing
"""
try:
self._config = parse_config_file(config_file, self.CONFIG_SPEC_FILE)
except Exception as e:
sys.exit(u'Error parsing configuration file: {}'.format(repr(e)))
try:
self._panoptes_context = PanoptesInfluxDBConsumerContext()
except Exception as e:
sys.exit(u'Could not create a InfluxDB Context: {}'.format(repr(e)))
self._logger = self._panoptes_context.logger
self._install_signal_handlers()
self.influxdb_connection = None
self._initialize_influxdb_connection()
topics = make_topic_names_for_all_sites(self._panoptes_context, self._config[u'kafka'][u'queue'])
client_id = get_client_id(prefix=self._config[u'kafka'][u'group_id'])
self._consumer = PanoptesConsumer(self._panoptes_context,
consumer_type=get_consumer_type_from_name(self._config[u'kafka'][u'queue']),
topics=topics,
keys=None,
client_id=client_id,
group=self._config[u'kafka'][u'group_id'],
poll_timeout=self._config[u'kafka'][u'poll_timeout'],
session_timeout=self._config[u'kafka'][u'session_timeout'],
max_poll_records=self._config[u'kafka'][u'max_poll_records'],
callback=self._process_message,
validate=False)
self.influxdb_points = set()
self._last_emitted = 0
self.influxdb_points_batch_size = 0
self._consumer.start_consumer()
def lock(self):
"""
Attempts to get a lock for unique plugin instance
Returns:
KazooLock: The lock object
Raises:
Exception: Passes through any exceptions raised while trying to get a lock
"""
if self._lock:
return self._lock
logger = self.panoptes_context.logger
client_id = get_client_id(const.PLUGIN_CLIENT_ID_PREFIX)
"""
We acquire a lock for a plugin under it's name and the hash of it's configuration and data. The motivation is
that multiple instance of the plugins are allowed to execute in parallel - as long as they are acting
on different resources or using different configurations
"""
lock_path = u'/'.join([
const.PLUGIN_AGENT_LOCK_PATH, self.normalized_category, u'plugins',
u'lock', self.normalized_name, self.signature
])
logger.debug(
u'Attempting to get lock for plugin "%s", with lock path "%s" and identifier "%s" in %d '
u'seconds' % (self.name, lock_path, client_id,
const.PLUGIN_AGENT_LOCK_ACQUIRE_TIMEOUT))
self._lock = PanoptesLock(
context=self.panoptes_context,
path=lock_path,
timeout=const.PLUGIN_AGENT_LOCK_ACQUIRE_TIMEOUT,
retries=1,
identifier=client_id)
return self._lock
def start():
global panoptes_context, logger, resource_store, consumer
try:
panoptes_context = PanoptesResourceManagerContext()
except Exception as e:
sys.exit(u'Could not create a Panoptes Context: %s' % (str(e)))
logger = panoptes_context.logger
_install_signal_handlers()
client_id = get_client_id(const.RESOURCE_MANAGER_CLIENT_ID_PREFIX)
resource_store = PanoptesResourceStore(panoptes_context)
consumer = PanoptesResourcesConsumer(
panoptes_context=panoptes_context,
keys=None,
client_id=client_id,
group=const.RESOURCE_MANAGER_KAFKA_GROUP_ID,
poll_timeout=const.RESOURCE_MANAGER_KAFKA_POLL_TIMEOUT,
callback=handle_resources,
max_partition_fetch_bytes=const.
RESOURCE_MANAGER_MAX_PARTITION_FETCH_BYTES)
consumer.start_consumer()
def start(self):
"""
This function starts the Plugin Scheduler. It installs signal handlers, acquire an distributed lock and then
return a Celery application instance
The flow of the startup process is follows:
start -> _celery_beat_service_started (starts) -> plugin_scheduler_task_thread
The plugin_scheduler_task_thread runs the plugin_scheduler_task every "['plugin_type']['plugin_scan_interval']"
seconds, which comes from the system wide configuration file
The reason for this slightly convoluted startup is that because the plugin_scheduler_task needs the Celery Beat
Service instance object so that it can update the schedule periodically and this only available after the
_celery_beat_service_started callback function is called by Celery Beat
Returns:
celery.app: The Celery App instance to be used by the scheduler
"""
logger = self._logger
logger.info(u'%s Plugin Scheduler main thread: OS PID: %d' %
(self._plugin_type_display_name, get_os_tid()))
logger.info(
u'"Tour Of Duty" adjusted values: tasks: %d count, time: %d seconds, memory: %dMB'
% (self._tour_of_duty.adjusted_tasks,
self._tour_of_duty.adjusted_seconds,
self._tour_of_duty.adjusted_memory_growth_mb))
logger.info(u'Setting up signal handlers')
self._install_signal_handlers()
client_id = get_client_id(str(const.PLUGIN_CLIENT_ID_PREFIX))
lock_path = str(
const.LOCK_PATH_DELIMITER.join([
_f for _f in [
const.PLUGIN_SCHEDULER_LOCK_PATH, self._plugin_type,
self._plugin_subtype,
str('lock')
] if _f
]))
logger.info(
u'Creating lock object for %s Plugin Scheduler under lock path "%s"'
% (self._plugin_type, lock_path))
try:
self._lock = PanoptesLock(context=self._panoptes_context,
path=lock_path,
timeout=self._lock_timeout,
retries=0,
identifier=client_id)
except Exception as e:
sys.exit(u'Failed to create lock object: %s' % repr(e))
if self._lock.locked:
logger.info(u'Starting Celery Beat Service')
try:
self._celery = PanoptesCeleryInstance(
self._panoptes_context, self._celery_config).celery
self._celery.conf.update(
CELERYBEAT_MAX_LOOP_INTERVAL=self._config[
self._plugin_type][u'celerybeat_max_loop_interval'])
except:
logger.exception(u'Error trying to start Celery Beat Service')
return self._celery