class DdForwarder(object): QUEUE_SIZE = "queue_size" QUEUE_COUNT = "queue_count" RE_QUEUE_STAT = re.compile(r"\[.*\] Queue size: at (.*), (\d+) transaction\(s\), (\d+) KB") def __init__(self, logger, config): self.log_path = config.get('ddforwarder_log', '/var/log/ddforwarder.log') self.logger = logger self._gen = None def _init_metrics(self): self.metrics = {} def _add_metric(self, name, value, ts): if name in self.metrics: self.metrics[name].append((ts, value)) else: self.metrics[name] = [(ts, value)] def _parse_line(self, line): try: m = self.RE_QUEUE_STAT.match(line) if m is not None: ts, count, size = m.groups() self._add_metric(self.QUEUE_SIZE, size, round(float(ts))) self._add_metric(self.QUEUE_COUNT, count, round(float(ts))) except Exception as e: self.logger.exception(e) def check(self, agentConfig, move_end=True): if self.log_path and os.path.isfile(self.log_path): #reset metric points self._init_metrics() # Build our tail -f if self._gen is None: self._gen = TailFile(self.logger, self.log_path, self._parse_line).tail(line_by_line=False, move_end=move_end) # read until the end of file try: self._gen.next() self.logger.debug("Done ddforwarder check for file %s" % self.log_path) except StopIteration as e: self.logger.exception(e) self.logger.warn("Can't tail %s file" % self.log_path) return {'ddforwarder': self.metrics} else: self.logger.debug("Can't tail datadog forwarder log file: %s" % self.log_path) return {}
class Dogstream(object): @classmethod def init(cls, logger, log_path, parser_spec=None, parser_args=None, config=None): class_based = False parse_func = None parse_args = tuple(parser_args or ()) if parser_spec: try: parse_func = modules.load(parser_spec) if isinstance(parse_func, type): logger.info('Instantiating class-based dogstream') parse_func = parse_func( user_args=parse_args or (), logger=logger, log_path=log_path, config=config, ) parse_args = () class_based = True else: logger.info('Instantiating function-based dogstream') except Exception: logger.exception(traceback.format_exc()) logger.error( 'Could not load Dogstream line parser "%s" PYTHONPATH=%s' % (parser_spec, os.environ.get('PYTHONPATH', ''))) logger.info("dogstream: parsing %s with %s (requested %s)" % (log_path, parse_func, parser_spec)) else: logger.info("dogstream: parsing %s with default parser" % log_path) return cls(logger, log_path, parse_func, parse_args, class_based=class_based) def __init__(self, logger, log_path, parse_func=None, parse_args=(), class_based=False): self.logger = logger self.class_based = class_based self.log_path = log_path self.parse_func = parse_func or self._default_line_parser self.parse_args = parse_args self._gen = None self._values = None self._freq = 15 # Will get updated on each check() self._error_count = 0L self._line_count = 0L self.parser_state = {} def check(self, agentConfig, move_end=True): if self.log_path: self._freq = int(agentConfig.get('check_freq', 15)) self._values = [] self._events = [] # Build our tail -f if self._gen is None: self._gen = TailFile(self.logger, self.log_path, self._line_parser).tail( line_by_line=False, move_end=move_end) # read until the end of file try: self._gen.next() self.logger.debug("Done dogstream check for file {0}".format( self.log_path)) self.logger.debug("Found {0} metric points".format( len(self._values))) except StopIteration as e: self.logger.exception(e) self.logger.warn("Can't tail %s file" % self.log_path) check_output = self._aggregate(self._values) if self._events: check_output.update({"dogstreamEvents": self._events}) self.logger.debug("Found {0} events".format(len(self._events))) return check_output else: return {} def _line_parser(self, line): try: # alq - Allow parser state to be kept between invocations # This means a new argument can be passed the custom parsing function # to store context that can be shared between parsing of lines. # One example is a running counter, which is incremented each time # a line is processed. parsed = None if self.class_based: parsed = self.parse_func.parse_line(line) else: try: parsed = self.parse_func(self.logger, line, self.parser_state, *self.parse_args) except TypeError: # Arity of parse_func is 3 (old-style), not 4 parsed = self.parse_func(self.logger, line) self._line_count += 1 if parsed is None: return if isinstance(parsed, (tuple, dict)): parsed = [parsed] for datum in parsed: # Check if it's an event if isinstance(datum, dict): # An event requires at least a title or a body if 'msg_title' not in datum and 'msg_text' not in datum: continue # Populate the default fields if 'event_type' not in datum: datum['event_type'] = EventDefaults.EVENT_TYPE if 'timestamp' not in datum: datum['timestamp'] = time.time() # Make sure event_object and aggregation_key (synonyms) are set # FIXME when the backend treats those as true synonyms, we can # deprecate event_object. if 'event_object' in datum or 'aggregation_key' in datum: datum['aggregation_key'] = datum.get( 'event_object', datum.get('aggregation_key')) else: datum['aggregation_key'] = EventDefaults.EVENT_OBJECT datum['event_object'] = datum['aggregation_key'] self._events.append(datum) continue # Otherwise, assume it's a metric try: metric, ts, value, attrs = datum except Exception: continue # Validation invalid_reasons = [] try: # Bucket points into 15 second buckets ts = (int(float(ts)) / self._freq) * self._freq date = datetime.fromtimestamp(ts) assert date.year > 1990 except Exception: invalid_reasons.append('invalid timestamp') try: value = float(value) except Exception: invalid_reasons.append('invalid metric value') if invalid_reasons: self.logger.debug('Invalid parsed values %s (%s): "%s"', repr(datum), ', '.join(invalid_reasons), line) else: self._values.append((metric, ts, value, attrs)) except Exception: self.logger.debug("Error while parsing line %s" % line, exc_info=True) self._error_count += 1 self.logger.error("Parser error: %s out of %s" % (self._error_count, self._line_count)) def _default_line_parser(self, logger, line): sep = ' ' metric, _, line = partition(line.strip(), sep) timestamp, _, line = partition(line.strip(), sep) value, _, line = partition(line.strip(), sep) attributes = {} try: while line: keyval, _, line = partition(line.strip(), sep) key, val = keyval.split('=', 1) attributes[key] = val except Exception: logger.debug(traceback.format_exc()) return metric, timestamp, value, attributes def _aggregate(self, values): """ Aggregate values down to the second and store as: { "dogstream": [(metric, timestamp, value, {key: val})] } If there are many values per second for a metric, take the median """ output = [] values.sort(key=point_sorter) for (timestamp, metric, host_name, device_name), val_attrs in groupby(values, key=point_sorter): attributes = {} vals = [] for _metric, _timestamp, v, a in val_attrs: try: v = float(v) vals.append(v) attributes.update(a) except Exception: self.logger.debug("Could not convert %s into a float", v) if len(vals) == 1: val = vals[0] elif len(vals) > 1: val = vals[-1] else: # len(vals) == 0 continue metric_type = str(attributes.get('metric_type', '')).lower() if metric_type == 'gauge': val = float(val) elif metric_type == 'counter': val = sum(vals) output.append((metric, timestamp, val, attributes)) if output: return {"dogstream": output} else: return {}
class Dogstream(object): @classmethod def init(cls, logger, log_path, parser_spec=None, parser_args=None, config=None): class_based = False parse_func = None parse_args = tuple(parser_args or ()) if parser_spec: try: parse_func = modules.load(parser_spec) if isinstance(parse_func, type): logger.info('Instantiating class-based dogstream') parse_func = parse_func( user_args=parse_args or (), logger=logger, log_path=log_path, config=config, ) parse_args = () class_based = True else: logger.info('Instantiating function-based dogstream') except Exception: logger.exception(traceback.format_exc()) logger.error('Could not load Dogstream line parser "%s" PYTHONPATH=%s' % ( parser_spec, os.environ.get('PYTHONPATH', '')) ) logger.info("dogstream: parsing %s with %s (requested %s)" % (log_path, parse_func, parser_spec)) else: logger.info("dogstream: parsing %s with default parser" % log_path) return cls(logger, log_path, parse_func, parse_args, class_based=class_based) def __init__(self, logger, log_path, parse_func=None, parse_args=(), class_based=False): self.logger = logger self.class_based = class_based self.log_path = log_path self.parse_func = parse_func or self._default_line_parser self.parse_args = parse_args self._gen = None self._values = None self._freq = 15 # Will get updated on each check() self._error_count = 0L self._line_count = 0L self.parser_state = {} def check(self, agentConfig, move_end=True): if self.log_path: self._freq = int(agentConfig.get('check_freq', 15)) self._values = [] self._events = [] # Build our tail -f if self._gen is None: self._gen = TailFile(self.logger, self.log_path, self._line_parser).tail(line_by_line=False, move_end=move_end) # read until the end of file try: self._gen.next() self.logger.debug("Done dogstream check for file {0}".format(self.log_path)) self.logger.debug("Found {0} metric points".format(len(self._values))) except StopIteration as e: self.logger.exception(e) self.logger.warn("Can't tail %s file" % self.log_path) check_output = self._aggregate(self._values) if self._events: check_output.update({"dogstreamEvents": self._events}) self.logger.debug("Found {0} events".format(len(self._events))) return check_output else: return {} def _line_parser(self, line): try: # alq - Allow parser state to be kept between invocations # This means a new argument can be passed the custom parsing function # to store context that can be shared between parsing of lines. # One example is a running counter, which is incremented each time # a line is processed. parsed = None if self.class_based: parsed = self.parse_func.parse_line(line) else: try: parsed = self.parse_func(self.logger, line, self.parser_state, *self.parse_args) except TypeError: # Arity of parse_func is 3 (old-style), not 4 parsed = self.parse_func(self.logger, line) self._line_count += 1 if parsed is None: return if isinstance(parsed, (tuple, dict)): parsed = [parsed] for datum in parsed: # Check if it's an event if isinstance(datum, dict): # An event requires at least a title or a body if 'msg_title' not in datum and 'msg_text' not in datum: continue # Populate the default fields if 'event_type' not in datum: datum['event_type'] = EventDefaults.EVENT_TYPE if 'timestamp' not in datum: datum['timestamp'] = time.time() # Make sure event_object and aggregation_key (synonyms) are set # FIXME when the backend treats those as true synonyms, we can # deprecate event_object. if 'event_object' in datum or 'aggregation_key' in datum: datum['aggregation_key'] = datum.get('event_object', datum.get('aggregation_key')) else: datum['aggregation_key'] = EventDefaults.EVENT_OBJECT datum['event_object'] = datum['aggregation_key'] self._events.append(datum) continue # Otherwise, assume it's a metric try: metric, ts, value, attrs = datum except Exception: continue # Validation invalid_reasons = [] try: # Bucket points into 15 second buckets ts = (int(float(ts)) / self._freq) * self._freq date = datetime.fromtimestamp(ts) assert date.year > 1990 except Exception: invalid_reasons.append('invalid timestamp') try: value = float(value) except Exception: invalid_reasons.append('invalid metric value') if invalid_reasons: self.logger.debug('Invalid parsed values %s (%s): "%s"', repr(datum), ', '.join(invalid_reasons), line) else: self._values.append((metric, ts, value, attrs)) except Exception: self.logger.debug("Error while parsing line %s" % line, exc_info=True) self._error_count += 1 self.logger.error("Parser error: %s out of %s" % (self._error_count, self._line_count)) def _default_line_parser(self, logger, line): sep = ' ' metric, _, line = partition(line.strip(), sep) timestamp, _, line = partition(line.strip(), sep) value, _, line = partition(line.strip(), sep) attributes = {} try: while line: keyval, _, line = partition(line.strip(), sep) key, val = keyval.split('=', 1) attributes[key] = val except Exception: logger.debug(traceback.format_exc()) return metric, timestamp, value, attributes def _aggregate(self, values): """ Aggregate values down to the second and store as: { "dogstream": [(metric, timestamp, value, {key: val})] } If there are many values per second for a metric, take the median """ output = [] values.sort(key=point_sorter) for (timestamp, metric, host_name, device_name), val_attrs in groupby(values, key=point_sorter): attributes = {} vals = [] for _metric, _timestamp, v, a in val_attrs: try: v = float(v) vals.append(v) attributes.update(a) except Exception: self.logger.debug("Could not convert %s into a float", v) if len(vals) == 1: val = vals[0] elif len(vals) > 1: val = vals[-1] else: # len(vals) == 0 continue metric_type = str(attributes.get('metric_type', '')).lower() if metric_type == 'gauge': val = float(val) elif metric_type == 'counter': val = sum(vals) output.append((metric, timestamp, val, attributes)) if output: return {"dogstream": output} else: return {}
class DdForwarder(object): QUEUE_SIZE = "queue_size" QUEUE_COUNT = "queue_count" RE_QUEUE_STAT = re.compile( r"\[.*\] Queue size: at (.*), (\d+) transaction\(s\), (\d+) KB") def __init__(self, logger, config): self.log_path = config.get('ddforwarder_log', '/var/log/ddforwarder.log') self.logger = logger self._gen = None def _init_metrics(self): self.metrics = {} def _add_metric(self, name, value, ts): if name in self.metrics: self.metrics[name].append((ts, value)) else: self.metrics[name] = [(ts, value)] def _parse_line(self, line): try: m = self.RE_QUEUE_STAT.match(line) if m is not None: ts, count, size = m.groups() self._add_metric(self.QUEUE_SIZE, size, round(float(ts))) self._add_metric(self.QUEUE_COUNT, count, round(float(ts))) except Exception as e: self.logger.exception(e) def check(self, agentConfig, move_end=True): if self.log_path and os.path.isfile(self.log_path): #reset metric points self._init_metrics() # Build our tail -f if self._gen is None: self._gen = TailFile(self.logger, self.log_path, self._parse_line).tail(line_by_line=False, move_end=move_end) # read until the end of file try: self._gen.next() self.logger.debug("Done ddforwarder check for file %s" % self.log_path) except StopIteration as e: self.logger.exception(e) self.logger.warn("Can't tail %s file" % self.log_path) return {'ddforwarder': self.metrics} else: self.logger.debug("Can't tail datadog forwarder log file: %s" % self.log_path) return {}