def __init__(self, store_endpoint, datacenter, default_metric_host, default_metric_ttl, default_metric_state): """ :param store_endpoint: Riemann server endpoint :type store_endpoint: str :param datacenter: The current datacenter name :type datacenter: str :param default_metric_host: The default host value used when no other host has been provided from the metric resource_metadata. :type default_metric_host: str :param default_metric_ttl: The amount of time in seconds that events sent to Riemann are retained in its index. :type default_metric_ttl: int :param default_metric_state: The default state value used when no other state has been provided from the metric resource_metadata. :type default_metric_state: str """ super(RiemannClient, self).__init__() self._store_endpoint = store_endpoint self.store_uri = urlparse(self.store_endpoint) self.default_metric_host = default_metric_host self.default_metric_ttl = default_metric_ttl self.default_metric_state = default_metric_state self.datacenter = datacenter self._transport = self._create_transport(self.store_uri.scheme) self.client = Client(transport=self._transport, )
class RiemannHandler(Handler): def __init__(self, config=None): # Initialize Handler Handler.__init__(self, config) if riemann_client is None: logging.error("Failed to load riemann_client module") return # Initialize options self.host = self.config['host'] self.port = int(self.config['port']) self.transport = self.config['transport'] # Initialize client if self.transport == 'tcp': self.transport = TCPTransport(self.host, self.port) else: self.transport = UDPTransport(self.host, self.port) self.client = Client(self.transport) self._connect() def get_default_config_help(self): """ Returns the help text for the configuration options for this handler """ config = super(RiemannHandler, self).get_default_config_help() config.update({ 'host': '', 'port': '', 'transport': 'tcp or udp', }) return config def get_default_config(self): """ Return the default config for the handler """ config = super(RiemannHandler, self).get_default_config() config.update({ 'host': '', 'port': 123, 'transport': 'tcp', }) return config def process(self, metric): """ Send a metric to Riemann. """ event = self._metric_to_riemann_event(metric) try: self.client.send_event(event) except Exception, e: self.log.error( "RiemannHandler: Error sending event to Riemann: %s", e)
def __init__(self, store_endpoint, datacenter, default_metric_host, default_metric_ttl, default_metric_state): """ :param store_endpoint: Riemann server endpoint :type store_endpoint: str :param datacenter: The current datacenter name :type datacenter: str :param default_metric_host: The default host value used when no other host has been provided from the metric resource_metadata. :type default_metric_host: str :param default_metric_ttl: The amount of time in seconds that events sent to Riemann are retained in its index. :type default_metric_ttl: int :param default_metric_state: The default state value used when no other state has been provided from the metric resource_metadata. :type default_metric_state: str """ super(RiemannClient, self).__init__() self._store_endpoint = store_endpoint self.store_uri = urlparse(self.store_endpoint) self.default_metric_host = default_metric_host self.default_metric_ttl = default_metric_ttl self.default_metric_state = default_metric_state self.datacenter = datacenter self._transport = self._create_transport(self.store_uri.scheme) self.client = Client( transport=self._transport, )
def _send(self, event): try: self.client.send_event(event) except (transport.RiemannError, struct.error, OSError, RuntimeError) as exc: LOG.warn("Could not deliver the message " "to the Riemann server! Trying again...") LOG.exception(exc) raise MetricsStoreError("Could not deliver the message " "to the Riemann server.") except TypeError as exc: LOG.debug("Message formatting issue with %r", Client.create_dict(event)) LOG.exception(exc) raise MetricsStoreError("Message formatting issue => " "%r: '%r'" % (Client.create_dict(event), exc))
def _send(self, event): try: self.client.send_event(event) except (transport.RiemannError, struct.error, OSError, RuntimeError) as exc: LOG.warn("Could not deliver the message " "to the Riemann server! Trying again...") LOG.exception(exc) raise MetricsStoreError("Could not deliver the message " "to the Riemann server.") except TypeError as exc: LOG.debug("Message formatting issue with %r", Client.create_dict(event)) LOG.exception(exc) raise MetricsStoreError( "Message formatting issue => " "%r: '%r'" % (Client.create_dict(event), exc) )
def query(q, config): from oshino.config import Config, load from riemann_client.client import Client cfg = load(config) riemann = cfg.riemann transport_cls = cfg.riemann.transport transport = transport_cls(riemann.host, riemann.port) print("Executing query: {0}".format(q)) with Client(transport) as client: print(client.query(q))
def init(self): logger.info("[riemann broker] I init the %s server connection to %s:%d" % (self.get_name(), str(self.host), self.port)) if self.use_udp: transport = UDPTransport(self.host, self.port) else: transport = TCPTransport(self.host, self.port) transport.connect() self.client = Client(transport)
def __init__(self, config=None): # Initialize Handler Handler.__init__(self, config) if riemann_client is None: logging.error("Failed to load riemann_client module") return # Initialize options self.host = self.config['host'] self.port = int(self.config['port']) self.transport = self.config['transport'] # Initialize client if self.transport == 'tcp': self.transport = TCPTransport(self.host, self.port) else: self.transport = UDPTransport(self.host, self.port) self.client = Client(self.transport) self._connect()
def main(): args = parsed_args(sys.argv[1:]) logging_level = logging.DEBUG if args.debug else logging.WARNING logging.basicConfig(level=logging_level) try: if args.protocol == 'tcp': transport = TCPTransport(args.host, args.port, args.timeout) elif args.protocol == 'tls': transport = TLSTransport( args.host, args.port, args.timeout, keyfile=args.keyfile, certfile=args.certfile, ca_certs=args.ca_certs) elif args.protocol == 'udp': transport = UDPTransport(args.host, args.port) else: raise RuntimeError('Transport {} is not supported'.format(args.transport)) fp = fping.Fping(args.fping_cmd, args.probe, interval=args.interval) client = Client(transport) for summary in fp.ping_summaries(args.target): try: send_summary(client, summary) except SendError as e: logging.warning('Unable to send {} to {} ({})'.format( e.summary, args.host, e)) except Exception as e: if args.debug: raise else: logging.error('Unhandled exception ({})'.format(e)) return 1 return 0
class RiemannHandler(Handler): def __init__(self, config=None): # Initialize Handler Handler.__init__(self, config) if riemann_client is None: logging.error("Failed to load riemann_client module") return # Initialize options self.host = self.config['host'] self.port = int(self.config['port']) self.transport = self.config['transport'] # Initialize client if self.transport == 'tcp': self.transport = TCPTransport(self.host, self.port) else: self.transport = UDPTransport(self.host, self.port) self.client = Client(self.transport) self._connect() def get_default_config_help(self): """ Returns the help text for the configuration options for this handler """ config = super(RiemannHandler, self).get_default_config_help() config.update({ 'host': '', 'port': '', 'transport': 'tcp or udp', }) return config def get_default_config(self): """ Return the default config for the handler """ config = super(RiemannHandler, self).get_default_config() config.update({ 'host': '', 'port': 123, 'transport': 'tcp', }) return config def process(self, metric): """ Send a metric to Riemann. """ event = self._metric_to_riemann_event(metric) try: self.client.send_event(event) except Exception as e: self.log.error( "RiemannHandler: Error sending event to Riemann: %s", e) def _metric_to_riemann_event(self, metric): """ Convert a metric to a dictionary representing a Riemann event. """ # Riemann has a separate "host" field, so remove from the path. path = '%s.%s.%s' % ( metric.getPathPrefix(), metric.getCollectorPath(), metric.getMetricPath() ) return self.client.create_event({ 'host': metric.host, 'service': path, 'time': metric.timestamp, 'metric_f': float(metric.value), 'ttl': metric.ttl, }) def _connect(self): self.transport.connect() def _close(self): """ Disconnect from Riemann. """ if hasattr(self, 'transport'): self.transport.disconnect() def __del__(self): self._close()
class RiemannClient(MetricsStoreClientBase): """Riemann client""" def __init__(self, store_endpoint, datacenter, default_metric_host, default_metric_ttl, default_metric_state): """ :param store_endpoint: Riemann server endpoint :type store_endpoint: str :param datacenter: The current datacenter name :type datacenter: str :param default_metric_host: The default host value used when no other host has been provided from the metric resource_metadata. :type default_metric_host: str :param default_metric_ttl: The amount of time in seconds that events sent to Riemann are retained in its index. :type default_metric_ttl: int :param default_metric_state: The default state value used when no other state has been provided from the metric resource_metadata. :type default_metric_state: str """ super(RiemannClient, self).__init__() self._store_endpoint = store_endpoint self.store_uri = urlparse(self.store_endpoint) self.default_metric_host = default_metric_host self.default_metric_ttl = default_metric_ttl self.default_metric_state = default_metric_state self.datacenter = datacenter self._transport = self._create_transport(self.store_uri.scheme) self.client = Client(transport=self._transport, ) def _create_transport(self, scheme): transport_types = dict( tcp=transport.TCPTransport, udp=transport.UDPTransport, ) _transport_cls = transport_types[scheme] return _transport_cls(host=self.store_uri.hostname, port=self.store_uri.port) @classmethod def get_name(cls): return "riemann" @classmethod def get_config_opts(cls): return super(RiemannClient, cls).get_config_opts() + [ cfg.StrOpt( 'store_endpoint', help="Complete Riemann endpoint, e.g. tcp://localhost:5555.", default=None, required=True), cfg.StrOpt( 'default_metric_host', default='openstack', help='The default host value used when no other host value is ' 'determined, such as from the metric resource_metadata.'), cfg.IntOpt( 'default_metric_ttl', default=86400, help='The default amount of time in seconds that events sent ' 'to Riemann are retained in its index.'), cfg.StrOpt( 'default_metric_state', default='ok', help='The default state value used when no other state value ' 'is determined, such as from the metric ' 'resource_metadata.'), cfg.StrOpt( 'datacenter', required=True, help='The current datacenter name', ), ] @property def store_endpoint(self): return self._store_endpoint def connect(self): try: self._transport.connect() except (OSError, RuntimeError) as exc: raise MetricsStoreError(exc) LOG.info("[Riemann] Client connected") def disconnect(self): try: self._transport.disconnect() except (OSError, RuntimeError) as exc: raise MetricsStoreError(exc) LOG.info("[Riemann] Client disconnected") def reconnect(self): try: LOG.debug("[Riemann] ==> Reconnecting...") self.disconnect() except MetricsStoreError as exc: LOG.exception(exc) try: self.connect() except MetricsStoreError as exc: LOG.exception(exc) LOG.debug("[Riemann] Socket connection re-established") def _send(self, event): try: self.client.send_event(event) except (transport.RiemannError, struct.error, OSError, RuntimeError) as exc: LOG.warn("Could not deliver the message " "to the Riemann server! Trying again...") LOG.exception(exc) raise MetricsStoreError("Could not deliver the message " "to the Riemann server.") except TypeError as exc: LOG.debug("Message formatting issue with %r", Client.create_dict(event)) LOG.exception(exc) raise MetricsStoreError("Message formatting issue => " "%r: '%r'" % (Client.create_dict(event), exc)) def send(self, metric): LOG.debug('Publishing metrics to `%s:%d`', self.store_uri.hostname, self.store_uri.port) try: event_msg = self.create_event(metric) except (MetricsStoreError, KeyError) as exc: LOG.exception(exc) raise MetricsStoreError(exc) for _ in range(2): # 2 attempts try: self._send(event_msg) break except Exception as exc: LOG.warn('Unable to send metric `%r`', metric) LOG.exception(exc) LOG.error("Connection issue --> try again...") self.reconnect() def create_event(self, metric): """Translates a dictionary of event attributes to an Event object :param dict metric: The attributes to be set on the event :returns: A protocol buffer ``Event`` object """ description = "" # Needs to be specified if any tags = [] # Needs to be specified if any metadata = metric.get("resource_metadata", {}) host = metadata.get("host", self.default_metric_host) ttl = metadata.get("ttl", self.default_metric_ttl) state = metadata.get("state", self.default_metric_state) try: timestamp = metric.get("timestamp", "") parsed = parser.parse(timestamp) epoch_seconds = calendar.timegm(parsed.timetuple()) except ValueError: error_message = "Could not parse the `%r` timestamp! " % timestamp LOG.exception(error_message) raise MetricsStoreError(error_message) event_dict = dict( time=epoch_seconds, ttl=int(ttl), host=host, service=metric["name"], state=state, metric_f=metric["value"], description=description, tags=tags, attributes=self._build_attributes(metric), ) try: return self.event_factory(event_dict) except TypeError as exc: LOG.exception(exc) raise MetricsStoreError(exc) def event_factory(self, data): event = Event() event.tags.extend(data.pop('tags', [])) for key, value in data.pop('attributes', {}).items(): attribute = event.attributes.add() attribute.key, attribute.value = key, value for name, value in data.items(): if value is not None: setattr(event, name, value) return event def _build_attributes(self, metric): ignored_metadata = ["host", "ttl"] metadata = metric.get("resource_metadata", {}) attributes = { key: val for key, val in metadata.items() if key not in ignored_metadata } attributes["datacenter"] = self.datacenter or "" attributes["resource_id"] = metric.get("resource_id", "") attributes["unit"] = metric.get("unit", "") return attributes
class RiemannBroker(BaseModule): def __init__(self, modconf): BaseModule.__init__(self, modconf) self.host = getattr(modconf, 'host', 'localhost') self.port = int(getattr(modconf, 'port', '5555')) self.use_udp = getattr(modconf, 'use_udp', '0') == '1' self.buffer = [] self.ticks = 0 self.tick_limit = int(getattr(modconf, 'tick_limit', '300')) # Called by Broker so we can do init stuff # Conf from arbiter! def init(self): logger.info("[riemann broker] I init the %s server connection to %s:%d" % (self.get_name(), str(self.host), self.port)) if self.use_udp: transport = UDPTransport(self.host, self.port) else: transport = TCPTransport(self.host, self.port) transport.connect() self.client = Client(transport) # Returns a list of perfdata events def get_check_result_perfdata_events(self, perf_data, timestamp, host, service): events = [] metrics = PerfDatas(perf_data).metrics for e in metrics.values(): event_data = { 'host': host, 'service': service, 'time': timestamp, #'state': 'WARNING', 'metric_f': e.value, 'description': e.name, } attributes = {} if e.uom is not None: attributes['unit'] = unicode(e.uom) if e.warning is not None: attributes['warning'] = unicode(e.warning) if e.critical is not None: attributes['critical'] = unicode(e.critical) if e.min is not None: attributes['min'] = unicode(e.min) if e.max is not None: attributes['max'] = unicode(e.max) if attributes is not {}: event_data['attributes'] = attributes events.append( self.client.create_event(event_data) ) return events # Returns state_update points for a given check_result_brok data def get_state_update_points(self, data): events = [] if data['state'] != data['last_state'] or \ data['state_type'] != data['last_state_type']: events.append( self.client.create_event( { 'host': data['host_name'], 'service': data['service_description'], 'time': data['last_chk'], 'attributes': { 'state': data['state'], 'state_type': data['state_type'], 'output': data['output'] } } ) ) return events # A service check result brok has just arrived, we UPDATE data info with this def manage_service_check_result_brok(self, b): data = b.data events = [] events.extend( self.get_check_result_perfdata_events( data['perf_data'], data['last_chk'], data['host_name'], data['service_description'] ) ) events.extend( self.get_state_update_points(data) ) try: logger.debug("[riemann broker] Launching: %s" % str(events)) except UnicodeEncodeError: pass self.buffer.extend(events) # A host check result brok has just arrived, we UPDATE data info with this def manage_host_check_result_brok(self, b): data = b.data events = [] events.extend( self.get_check_result_perfdata_events( data['perf_data'], data['last_chk'], data['host_name'], None, ) ) events.extend( self.get_state_update_points(data) ) try: logger.debug("[riemann broker] Launching: %s" % str(events)) except UnicodeEncodeError: pass self.buffer.extend(events) def manage_unknown_host_check_result_brok(self, b): data = b.data events = [] events.extend( self.get_check_result_perfdata_events( data['perf_data'], data['last_chk'], data['host_name'], None, ) ) try: logger.debug("[riemann broker] Launching: %s" % str(events)) except UnicodeEncodeError: pass self.buffer.extend(events) def manage_unknown_service_check_result_brok(self, b): data = b.data events = [] events.extend( self.get_check_result_perfdata_events( data['perf_data'], data['time_stamp'], data['host_name'], data['service_description'] ) ) try: logger.debug("[riemann broker] Launching: %s" % str(events)) except UnicodeEncodeError: pass self.buffer.extend(events) # A log brok has arrived, we UPDATE data info with this def manage_log_brok(self, b): log = b.data['log'] log_event = LogEvent(log) if len(log_event) > 0: event_data = { 'host': b.data['hostname'], 'service': b.data['service_desc'], 'description': b.data['event_type'], 'time': b.data['time'] } attributes = {} # Add each property of the service in the point for prop in log_event: attributes[prop[0]] = unicode(prop[1]) self.buffer.append( self.client.create_event(event_data) ) def hook_tick(self, brok): if self.ticks >= self.tick_limit: logger.error("[riemann broker] Buffering ticks exceeded. Freeing buffer") self.buffer = [] self.ticks = 0 if len(self.buffer) > 0: try: for event in self.buffer: self.client.send_event(event) self.buffer.remove(event) self.ticks = 0 except: self.ticks += 1 logger.error("[riemann broker] Sending data Failed. Buffering state : %s / %s" % (self.ticks, self.tick_limit))
class RiemannClient(MetricsStoreClientBase): """Riemann client""" def __init__(self, store_endpoint, datacenter, default_metric_host, default_metric_ttl, default_metric_state): """ :param store_endpoint: Riemann server endpoint :type store_endpoint: str :param datacenter: The current datacenter name :type datacenter: str :param default_metric_host: The default host value used when no other host has been provided from the metric resource_metadata. :type default_metric_host: str :param default_metric_ttl: The amount of time in seconds that events sent to Riemann are retained in its index. :type default_metric_ttl: int :param default_metric_state: The default state value used when no other state has been provided from the metric resource_metadata. :type default_metric_state: str """ super(RiemannClient, self).__init__() self._store_endpoint = store_endpoint self.store_uri = urlparse(self.store_endpoint) self.default_metric_host = default_metric_host self.default_metric_ttl = default_metric_ttl self.default_metric_state = default_metric_state self.datacenter = datacenter self._transport = self._create_transport(self.store_uri.scheme) self.client = Client( transport=self._transport, ) def _create_transport(self, scheme): transport_types = dict( tcp=transport.TCPTransport, udp=transport.UDPTransport, ) _transport_cls = transport_types[scheme] return _transport_cls( host=self.store_uri.hostname, port=self.store_uri.port ) @classmethod def get_name(cls): return "riemann" @classmethod def get_config_opts(cls): return super(RiemannClient, cls).get_config_opts() + [ cfg.StrOpt( 'store_endpoint', help="Complete Riemann endpoint, e.g. tcp://localhost:5555.", default=None, required=True), cfg.StrOpt( 'default_metric_host', default='openstack', help='The default host value used when no other host value is ' 'determined, such as from the metric resource_metadata.' ), cfg.IntOpt( 'default_metric_ttl', default=86400, help='The default amount of time in seconds that events sent ' 'to Riemann are retained in its index.' ), cfg.StrOpt( 'default_metric_state', default='ok', help='The default state value used when no other state value ' 'is determined, such as from the metric ' 'resource_metadata.' ), cfg.StrOpt( 'datacenter', required=True, help='The current datacenter name', ), ] @property def store_endpoint(self): return self._store_endpoint def connect(self): try: self._transport.connect() except (OSError, RuntimeError) as exc: raise MetricsStoreError(exc) LOG.info("[Riemann] Client connected") def disconnect(self): try: self._transport.disconnect() except (OSError, RuntimeError) as exc: raise MetricsStoreError(exc) LOG.info("[Riemann] Client disconnected") def reconnect(self): try: LOG.debug("[Riemann] ==> Reconnecting...") self.disconnect() except MetricsStoreError as exc: LOG.exception(exc) try: self.connect() except MetricsStoreError as exc: LOG.exception(exc) LOG.debug("[Riemann] Socket connection re-established") def _send(self, event): try: self.client.send_event(event) except (transport.RiemannError, struct.error, OSError, RuntimeError) as exc: LOG.warn("Could not deliver the message " "to the Riemann server! Trying again...") LOG.exception(exc) raise MetricsStoreError("Could not deliver the message " "to the Riemann server.") except TypeError as exc: LOG.debug("Message formatting issue with %r", Client.create_dict(event)) LOG.exception(exc) raise MetricsStoreError( "Message formatting issue => " "%r: '%r'" % (Client.create_dict(event), exc) ) def send(self, metric): LOG.debug('Publishing metrics to `%s:%d`', self.store_uri.hostname, self.store_uri.port) try: event_msg = self.create_event(metric) except (MetricsStoreError, KeyError) as exc: LOG.exception(exc) raise MetricsStoreError(exc) for _ in range(2): # 2 attempts try: self._send(event_msg) break except Exception as exc: LOG.warn('Unable to send metric `%r`', metric) LOG.exception(exc) LOG.error("Connection issue --> try again...") self.reconnect() def create_event(self, metric): """Translates a dictionary of event attributes to an Event object :param dict metric: The attributes to be set on the event :returns: A protocol buffer ``Event`` object """ description = "" # Needs to be specified if any tags = [] # Needs to be specified if any metadata = metric.get("resource_metadata", {}) host = metadata.get("host", self.default_metric_host) ttl = metadata.get("ttl", self.default_metric_ttl) state = metadata.get("state", self.default_metric_state) try: timestamp = metric.get("timestamp", "") parsed = parser.parse(timestamp) epoch_seconds = calendar.timegm(parsed.timetuple()) except ValueError: error_message = "Could not parse the `%r` timestamp! " % timestamp LOG.exception(error_message) raise MetricsStoreError(error_message) event_dict = dict( time=epoch_seconds, ttl=int(ttl), host=host, service=metric["name"], state=state, metric_f=metric["value"], description=description, tags=tags, attributes=self._build_attributes(metric), ) try: return self.event_factory(event_dict) except TypeError as exc: LOG.exception(exc) raise MetricsStoreError(exc) def event_factory(self, data): event = Event() event.tags.extend(data.pop('tags', [])) for key, value in data.pop('attributes', {}).items(): attribute = event.attributes.add() attribute.key, attribute.value = key, value for name, value in data.items(): if value is not None: setattr(event, name, value) return event def _build_attributes(self, metric): ignored_metadata = ["host", "ttl"] metadata = metric.get("resource_metadata", {}) attributes = { key: val for key, val in metadata.items() if key not in ignored_metadata } attributes["datacenter"] = self.datacenter or "" attributes["resource_id"] = metric.get("resource_id", "") attributes["unit"] = metric.get("unit", "") return attributes