def __init__(self, path, value, raw_value=None, timestamp=None, precision=0, host=None, metric_type='COUNTER'): """ Create new instance of the Metric class Takes: path=string: string the specifies the path of the metric value=[float|int]: the value to be submitted timestamp=[float|int]: the timestamp, in seconds since the epoch (as from time.time()) precision=int: the precision to apply. Generally the default (2) should work fine. """ # Validate the path, value and metric_type submitted if (path is None or value is None or metric_type not in self._METRIC_TYPES): raise DiamondException("Invalid parameter.") # If no timestamp was passed in, set it to the current time if timestamp is None: timestamp = int(time.time()) else: # If the timestamp isn't an int, then make it one if not isinstance(timestamp, int): try: timestamp = int(timestamp) except ValueError, e: raise DiamondException("Invalid parameter: %s" % e)
def __init__(self, path, value, raw_value=None, timestamp=None, precision=0, host=None, metric_type='COUNTER', ttl=None): """ Create new instance of the Metric class Takes: path=string: string the specifies the path of the metric value=[float|int]: the value to be submitted timestamp=[float|int]: the timestamp, in seconds since the epoch (as from time.time()) precision=int: the precision to apply. Generally the default (2) should work fine. """ # Validate the path, value and metric_type submitted if (None in [path, value] or metric_type not in ('COUNTER', 'GAUGE')): raise DiamondException( ("Invalid parameter when creating new " "Metric with path: %r value: %r " "metric_type: %r") % (path, value, metric_type)) # If no timestamp was passed in, set it to the current time if timestamp is None: timestamp = int(time.time()) else: # If the timestamp isn't an int, then make it one if not isinstance(timestamp, int): try: timestamp = int(timestamp) except ValueError as e: raise DiamondException( ("Invalid timestamp when " "creating new Metric %r: %s") % (path, e)) # The value needs to be a float or an int. If it is, great. If not, # try to cast it to one of those. if not isinstance(value, (int, float)): try: if precision == 0: value = round(float(value)) else: value = float(value) except ValueError as e: raise DiamondException(("Invalid value when creating new " "Metric %r: %s") % (path, e)) self.path = path self.value = value self.raw_value = raw_value self.timestamp = timestamp self.precision = precision self.host = host self.metric_type = metric_type self.ttl = ttl
def process_config(self): """ Intended to put any code that should be run after any config reload event """ if 'byte_unit' in self.config: if isinstance(self.config['byte_unit'], basestring): self.config['byte_unit'] = self.config['byte_unit'].split() if 'enabled' in self.config: self.config['enabled'] = str_to_bool(self.config['enabled']) if 'measure_collector_time' in self.config: self.config['measure_collector_time'] = str_to_bool( self.config['measure_collector_time']) # Raise an error if both whitelist and blacklist are specified if (self.config.get('metrics_whitelist', None) and self.config.get('metrics_blacklist', None)): raise DiamondException( 'Both metrics_whitelist and metrics_blacklist specified ' + 'in file %s' % self.configfile) if self.config.get('metrics_whitelist', None): self.config['metrics_whitelist'] = re.compile( self.config['metrics_whitelist']) elif self.config.get('metrics_blacklist', None): self.config['metrics_blacklist'] = re.compile( self.config['metrics_blacklist'])
def process_config(self): """ Intended to put any code that should be run after any config reload event """ if 'byte_unit' in self.config: if isinstance(self.config['byte_unit'], basestring): self.config['byte_unit'] = self.config['byte_unit'].split() if 'enabled' in self.config: self.config['enabled'] = str_to_bool(self.config['enabled']) if 'measure_collector_time' in self.config: self.config['measure_collector_time'] = str_to_bool( self.config['measure_collector_time']) # Raise an error if both whitelist and blacklist are specified if (self.config.get('metrics_whitelist', None) and self.config.get('metrics_blacklist', None)): raise DiamondException( 'Both metrics_whitelist and metrics_blacklist specified ' + 'in file %s' % configfile) # THOUGHTSPOT_CUSTOMIZATION_BEGIN # TODO(Pradeep): Move this out so that the whitelist is read just once # and not once for every collector if self.config.get('whitelist_file', None): lines = [] with open(self.config['whitelist_file']) as f: lines = f.read().splitlines() self.config['regex'] = [] start = "[[" + self.name + "]]" end_regex = "\[\[.*\]\]" comment_regex = "#.*" started = False for line in lines: if not line or re.match(comment_regex, line): continue if started == False and line == start: started = True elif started == True: if re.match(end_regex, line): break else: self.config['regex'].append(line) # THOUGHTSPOT_CUSTOMIZATION_END if self.config.get('metrics_whitelist', None): self.config['metrics_whitelist'] = re.compile( self.config['metrics_whitelist']) elif self.config.get('metrics_blacklist', None): self.config['metrics_blacklist'] = re.compile( self.config['metrics_blacklist'])
class Metric(object): _METRIC_TYPES = ['COUNTER', 'GAUGE'] def __init__(self, path, value, raw_value=None, timestamp=None, precision=0, host=None, metric_type='COUNTER', ttl=None): """ Create new instance of the Metric class Takes: path=string: string the specifies the path of the metric value=[float|int]: the value to be submitted timestamp=[float|int]: the timestamp, in seconds since the epoch (as from time.time()) precision=int: the precision to apply. Generally the default (2) should work fine. """ # Validate the path, value and metric_type submitted if (None in [path, value] or metric_type not in self._METRIC_TYPES): raise DiamondException("Invalid parameter.") # If no timestamp was passed in, set it to the current time if timestamp is None: timestamp = int(time.time()) else: # If the timestamp isn't an int, then make it one if not isinstance(timestamp, int): try: timestamp = int(timestamp) except ValueError, e: raise DiamondException("Invalid parameter: %s" % e) # The value needs to be a float or an int. If it is, great. If not, # try to cast it to one of those. if not isinstance(value, (int, float)): try: if precision == 0: value = round(float(value)) else: value = float(value) except ValueError, e: raise DiamondException("Invalid parameter: %s" % e)
def parse(cls, string): """ Parse a string and create a metric """ match = re.match( r'^(?P<name>[A-Za-z0-9\.\-_]+)\s+' + '(?P<value>[0-9\.]+)\s+' + '(?P<timestamp>[0-9\.]+)(\n?)$', string) try: groups = match.groupdict() # TODO: get precision from value string return Metric(groups['name'], groups['value'], float(groups['timestamp'])) except: raise DiamondException( "Metric could not be parsed from string: %s." % string)
def process_config(self): """ Intended to put any code that should be run after any config reload event """ if 'byte_unit' in self.config: if isinstance(self.config['byte_unit'], basestring): self.config['byte_unit'] = self.config['byte_unit'].split() if 'enabled' in self.config: self.config['enabled'] = str_to_bool(self.config['enabled']) if 'measure_collector_time' in self.config: self.config['measure_collector_time'] = str_to_bool( self.config['measure_collector_time']) wlist = self.config.get('metrics_whitelist', None) blist = self.config.get('metrics_blacklist', None) if (wlist and blist): # Raise an error if both whitelist and blacklist are specified raise DiamondException( 'Both metrics_whitelist and metrics_blacklist specified ' + 'in file %s' % self.configfile) # always a list so we can iterate if type(wlist) == str: wlist = [wlist] elif wlist is None: wlist = [] if type(blist) == str: blist = [blist] elif blist is None: blist = [] self.config['metrics_whitelist'] = [re.compile(x) for x in wlist] self.config['metrics_blacklist'] = [re.compile(x) for x in blist]
def get_hostname(config, method=None): """ Returns a hostname as configured by the user """ method = method or config.get('hostname_method', 'smart') # case insensitive method method = method.lower() if 'hostname' in config and method != 'shell': return config['hostname'] if method in get_hostname.cached_results: return get_hostname.cached_results[method] if method == 'shell': if 'hostname' not in config: raise DiamondException( "hostname must be set to a shell command for" " hostname_method=shell") else: proc = subprocess.Popen(config['hostname'], shell=True, stdout=subprocess.PIPE) hostname = proc.communicate()[0].strip() if proc.returncode != 0: raise subprocess.CalledProcessError(proc.returncode, config['hostname']) get_hostname.cached_results[method] = hostname return hostname if method == 'smart': hostname = get_hostname(config, 'fqdn_short') if hostname != 'localhost': get_hostname.cached_results[method] = hostname return hostname hostname = get_hostname(config, 'hostname_short') get_hostname.cached_results[method] = hostname return hostname if method == 'fqdn_short': hostname = socket.getfqdn().split('.')[0] get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'fqdn': hostname = socket.getfqdn().replace('.', '_') get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'fqdn_rev': hostname = socket.getfqdn().split('.') hostname.reverse() hostname = '.'.join(hostname) get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'uname_short': hostname = os.uname()[1].split('.')[0] get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'uname_rev': hostname = os.uname()[1].split('.') hostname.reverse() hostname = '.'.join(hostname) get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'hostname': hostname = socket.gethostname() get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'hostname_short': hostname = socket.gethostname().split('.')[0] get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'hostname_rev': hostname = socket.gethostname().split('.') hostname.reverse() hostname = '.'.join(hostname) get_hostname.cached_results[method] = hostname if hostname == '': raise DiamondException('Hostname is empty?!') return hostname if method == 'none': get_hostname.cached_results[method] = None return None raise NotImplementedError(config['hostname_method'])
def __init__(self, config, handlers): """ Create a new instance of the Collector class """ # Initialize Logger self.log = logging.getLogger('diamond') # Initialize Members self.name = self.__class__.__name__ self.handlers = handlers self.last_values = {} # Get Collector class cls = self.__class__ # Initialize config self.config = configobj.ConfigObj() # Check if default config is defined if self.get_default_config() is not None: # Merge default config self.config.merge(self.get_default_config()) # Merge default Collector config self.config.merge(config['collectors']['default']) # Check if Collector config section exists if cls.__name__ in config['collectors']: # Merge Collector config section self.config.merge(config['collectors'][cls.__name__]) # Check for config file in config directory configfile = os.path.join(config['server']['collectors_config_path'], cls.__name__) + '.conf' if os.path.exists(configfile): # Merge Collector config file self.config.merge(configobj.ConfigObj(configfile)) # Handle some config file changes transparently if isinstance(self.config['byte_unit'], basestring): self.config['byte_unit'] = self.config['byte_unit'].split() self.config['enabled'] = str_to_bool(self.config['enabled']) self.config['measure_collector_time'] = str_to_bool( self.config['measure_collector_time']) # Raise an error if both whitelist and blacklist are specified if self.config['metrics_whitelist'] and \ self.config['metrics_blacklist']: raise DiamondException( 'Both metrics_whitelist and metrics_blacklist specified ' + 'in file %s' % configfile) if self.config['metrics_whitelist']: self.config['metrics_whitelist'] = re.compile( self.config['metrics_whitelist']) elif self.config['metrics_blacklist']: self.config['metrics_blacklist'] = re.compile( self.config['metrics_blacklist']) self.collect_running = False
# try to cast it to one of those. if not isinstance(value, (int, float)): try: if precision == 0: value = round(float(value)) else: value = float(value) except ValueError, e: raise DiamondException(("Invalid value when creating new " "Metric %r: %s") % (path, e)) # If dimensions were passed in make sure they are a dict if dimensions is not None: if not isinstance(dimensions, dict): raise DiamondException( ("Invalid dimensions when " "creating new Metric %r: %s") % (path, dimensions)) else: dimensions = dict( (k, str(v)) for k, v in dimensions.iteritems() if v is not None and isinstance(v, (int, float, str, unicode)) and k is not None and isinstance(k, str)) self.dimensions = dimensions self.path = path self.value = value self.raw_value = raw_value self.timestamp = timestamp self.precision = precision self.metric_type = metric_type