def __init__(self,
path,
value,
raw_value=None,
timestamp=None,
precision=0,
host=None,
metric_type='COUNTER'):
"""
Create new instance of the Metric class
Takes:
path=string: string the specifies the path of the metric
value=[float|int]: the value to be submitted
timestamp=[float|int]: the timestamp, in seconds since the epoch
(as from time.time()) precision=int: the precision to apply.
Generally the default (2) should work fine.
"""
# Validate the path, value and metric_type submitted
if (path is None or value is None
or metric_type not in self._METRIC_TYPES):
raise DiamondException("Invalid parameter.")
# If no timestamp was passed in, set it to the current time
if timestamp is None:
timestamp = int(time.time())
else:
# If the timestamp isn't an int, then make it one
if not isinstance(timestamp, int):
try:
timestamp = int(timestamp)
except ValueError, e:
raise DiamondException("Invalid parameter: %s" % e)
def __init__(self,
path,
value,
raw_value=None,
timestamp=None,
precision=0,
host=None,
metric_type='COUNTER',
ttl=None):
"""
Create new instance of the Metric class
Takes:
path=string: string the specifies the path of the metric
value=[float|int]: the value to be submitted
timestamp=[float|int]: the timestamp, in seconds since the epoch
(as from time.time()) precision=int: the precision to apply.
Generally the default (2) should work fine.
"""
# Validate the path, value and metric_type submitted
if (None in [path, value] or metric_type not in ('COUNTER', 'GAUGE')):
raise DiamondException(
("Invalid parameter when creating new "
"Metric with path: %r value: %r "
"metric_type: %r") % (path, value, metric_type))
# If no timestamp was passed in, set it to the current time
if timestamp is None:
timestamp = int(time.time())
else:
# If the timestamp isn't an int, then make it one
if not isinstance(timestamp, int):
try:
timestamp = int(timestamp)
except ValueError as e:
raise DiamondException(
("Invalid timestamp when "
"creating new Metric %r: %s") % (path, e))
# The value needs to be a float or an int. If it is, great. If not,
# try to cast it to one of those.
if not isinstance(value, (int, float)):
try:
if precision == 0:
value = round(float(value))
else:
value = float(value)
except ValueError as e:
raise DiamondException(("Invalid value when creating new "
"Metric %r: %s") % (path, e))
self.path = path
self.value = value
self.raw_value = raw_value
self.timestamp = timestamp
self.precision = precision
self.host = host
self.metric_type = metric_type
self.ttl = ttl
def process_config(self):
"""
Intended to put any code that should be run after any config reload
event
"""
if 'byte_unit' in self.config:
if isinstance(self.config['byte_unit'], basestring):
self.config['byte_unit'] = self.config['byte_unit'].split()
if 'enabled' in self.config:
self.config['enabled'] = str_to_bool(self.config['enabled'])
if 'measure_collector_time' in self.config:
self.config['measure_collector_time'] = str_to_bool(
self.config['measure_collector_time'])
# Raise an error if both whitelist and blacklist are specified
if (self.config.get('metrics_whitelist', None)
and self.config.get('metrics_blacklist', None)):
raise DiamondException(
'Both metrics_whitelist and metrics_blacklist specified ' +
'in file %s' % self.configfile)
if self.config.get('metrics_whitelist', None):
self.config['metrics_whitelist'] = re.compile(
self.config['metrics_whitelist'])
elif self.config.get('metrics_blacklist', None):
self.config['metrics_blacklist'] = re.compile(
self.config['metrics_blacklist'])
def process_config(self):
"""
Intended to put any code that should be run after any config reload
event
"""
if 'byte_unit' in self.config:
if isinstance(self.config['byte_unit'], basestring):
self.config['byte_unit'] = self.config['byte_unit'].split()
if 'enabled' in self.config:
self.config['enabled'] = str_to_bool(self.config['enabled'])
if 'measure_collector_time' in self.config:
self.config['measure_collector_time'] = str_to_bool(
self.config['measure_collector_time'])
# Raise an error if both whitelist and blacklist are specified
if (self.config.get('metrics_whitelist', None)
and self.config.get('metrics_blacklist', None)):
raise DiamondException(
'Both metrics_whitelist and metrics_blacklist specified ' +
'in file %s' % configfile)
# THOUGHTSPOT_CUSTOMIZATION_BEGIN
# TODO(Pradeep): Move this out so that the whitelist is read just once
# and not once for every collector
if self.config.get('whitelist_file', None):
lines = []
with open(self.config['whitelist_file']) as f:
lines = f.read().splitlines()
self.config['regex'] = []
start = "[[" + self.name + "]]"
end_regex = "\[\[.*\]\]"
comment_regex = "#.*"
started = False
for line in lines:
if not line or re.match(comment_regex, line):
continue
if started == False and line == start:
started = True
elif started == True:
if re.match(end_regex, line):
break
else:
self.config['regex'].append(line)
# THOUGHTSPOT_CUSTOMIZATION_END
if self.config.get('metrics_whitelist', None):
self.config['metrics_whitelist'] = re.compile(
self.config['metrics_whitelist'])
elif self.config.get('metrics_blacklist', None):
self.config['metrics_blacklist'] = re.compile(
self.config['metrics_blacklist'])
class Metric(object):
_METRIC_TYPES = ['COUNTER', 'GAUGE']
def __init__(self,
path,
value,
raw_value=None,
timestamp=None,
precision=0,
host=None,
metric_type='COUNTER',
ttl=None):
"""
Create new instance of the Metric class
Takes:
path=string: string the specifies the path of the metric
value=[float|int]: the value to be submitted
timestamp=[float|int]: the timestamp, in seconds since the epoch
(as from time.time()) precision=int: the precision to apply.
Generally the default (2) should work fine.
"""
# Validate the path, value and metric_type submitted
if (None in [path, value] or metric_type not in self._METRIC_TYPES):
raise DiamondException("Invalid parameter.")
# If no timestamp was passed in, set it to the current time
if timestamp is None:
timestamp = int(time.time())
else:
# If the timestamp isn't an int, then make it one
if not isinstance(timestamp, int):
try:
timestamp = int(timestamp)
except ValueError, e:
raise DiamondException("Invalid parameter: %s" % e)
# The value needs to be a float or an int. If it is, great. If not,
# try to cast it to one of those.
if not isinstance(value, (int, float)):
try:
if precision == 0:
value = round(float(value))
else:
value = float(value)
except ValueError, e:
raise DiamondException("Invalid parameter: %s" % e)
def parse(cls, string):
"""
Parse a string and create a metric
"""
match = re.match(
r'^(?P<name>[A-Za-z0-9\.\-_]+)\s+' + '(?P<value>[0-9\.]+)\s+' +
'(?P<timestamp>[0-9\.]+)(\n?)$', string)
try:
groups = match.groupdict()
# TODO: get precision from value string
return Metric(groups['name'], groups['value'],
float(groups['timestamp']))
except:
raise DiamondException(
"Metric could not be parsed from string: %s." % string)
def process_config(self):
"""
Intended to put any code that should be run after any config reload
event
"""
if 'byte_unit' in self.config:
if isinstance(self.config['byte_unit'], basestring):
self.config['byte_unit'] = self.config['byte_unit'].split()
if 'enabled' in self.config:
self.config['enabled'] = str_to_bool(self.config['enabled'])
if 'measure_collector_time' in self.config:
self.config['measure_collector_time'] = str_to_bool(
self.config['measure_collector_time'])
wlist = self.config.get('metrics_whitelist', None)
blist = self.config.get('metrics_blacklist', None)
if (wlist and blist):
# Raise an error if both whitelist and blacklist are specified
raise DiamondException(
'Both metrics_whitelist and metrics_blacklist specified ' +
'in file %s' % self.configfile)
# always a list so we can iterate
if type(wlist) == str:
wlist = [wlist]
elif wlist is None:
wlist = []
if type(blist) == str:
blist = [blist]
elif blist is None:
blist = []
self.config['metrics_whitelist'] = [re.compile(x) for x in wlist]
self.config['metrics_blacklist'] = [re.compile(x) for x in blist]
def get_hostname(config, method=None):
"""
Returns a hostname as configured by the user
"""
method = method or config.get('hostname_method', 'smart')
# case insensitive method
method = method.lower()
if 'hostname' in config and method != 'shell':
return config['hostname']
if method in get_hostname.cached_results:
return get_hostname.cached_results[method]
if method == 'shell':
if 'hostname' not in config:
raise DiamondException(
"hostname must be set to a shell command for"
" hostname_method=shell")
else:
proc = subprocess.Popen(config['hostname'],
shell=True,
stdout=subprocess.PIPE)
hostname = proc.communicate()[0].strip()
if proc.returncode != 0:
raise subprocess.CalledProcessError(proc.returncode,
config['hostname'])
get_hostname.cached_results[method] = hostname
return hostname
if method == 'smart':
hostname = get_hostname(config, 'fqdn_short')
if hostname != 'localhost':
get_hostname.cached_results[method] = hostname
return hostname
hostname = get_hostname(config, 'hostname_short')
get_hostname.cached_results[method] = hostname
return hostname
if method == 'fqdn_short':
hostname = socket.getfqdn().split('.')[0]
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'fqdn':
hostname = socket.getfqdn().replace('.', '_')
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'fqdn_rev':
hostname = socket.getfqdn().split('.')
hostname.reverse()
hostname = '.'.join(hostname)
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'uname_short':
hostname = os.uname()[1].split('.')[0]
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'uname_rev':
hostname = os.uname()[1].split('.')
hostname.reverse()
hostname = '.'.join(hostname)
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'hostname':
hostname = socket.gethostname()
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'hostname_short':
hostname = socket.gethostname().split('.')[0]
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'hostname_rev':
hostname = socket.gethostname().split('.')
hostname.reverse()
hostname = '.'.join(hostname)
get_hostname.cached_results[method] = hostname
if hostname == '':
raise DiamondException('Hostname is empty?!')
return hostname
if method == 'none':
get_hostname.cached_results[method] = None
return None
raise NotImplementedError(config['hostname_method'])
def __init__(self, config, handlers):
"""
Create a new instance of the Collector class
"""
# Initialize Logger
self.log = logging.getLogger('diamond')
# Initialize Members
self.name = self.__class__.__name__
self.handlers = handlers
self.last_values = {}
# Get Collector class
cls = self.__class__
# Initialize config
self.config = configobj.ConfigObj()
# Check if default config is defined
if self.get_default_config() is not None:
# Merge default config
self.config.merge(self.get_default_config())
# Merge default Collector config
self.config.merge(config['collectors']['default'])
# Check if Collector config section exists
if cls.__name__ in config['collectors']:
# Merge Collector config section
self.config.merge(config['collectors'][cls.__name__])
# Check for config file in config directory
configfile = os.path.join(config['server']['collectors_config_path'],
cls.__name__) + '.conf'
if os.path.exists(configfile):
# Merge Collector config file
self.config.merge(configobj.ConfigObj(configfile))
# Handle some config file changes transparently
if isinstance(self.config['byte_unit'], basestring):
self.config['byte_unit'] = self.config['byte_unit'].split()
self.config['enabled'] = str_to_bool(self.config['enabled'])
self.config['measure_collector_time'] = str_to_bool(
self.config['measure_collector_time'])
# Raise an error if both whitelist and blacklist are specified
if self.config['metrics_whitelist'] and \
self.config['metrics_blacklist']:
raise DiamondException(
'Both metrics_whitelist and metrics_blacklist specified ' +
'in file %s' % configfile)
if self.config['metrics_whitelist']:
self.config['metrics_whitelist'] = re.compile(
self.config['metrics_whitelist'])
elif self.config['metrics_blacklist']:
self.config['metrics_blacklist'] = re.compile(
self.config['metrics_blacklist'])
self.collect_running = False
# try to cast it to one of those.
if not isinstance(value, (int, float)):
try:
if precision == 0:
value = round(float(value))
else:
value = float(value)
except ValueError, e:
raise DiamondException(("Invalid value when creating new "
"Metric %r: %s") % (path, e))
# If dimensions were passed in make sure they are a dict
if dimensions is not None:
if not isinstance(dimensions, dict):
raise DiamondException(
("Invalid dimensions when "
"creating new Metric %r: %s") % (path, dimensions))
else:
dimensions = dict(
(k, str(v)) for k, v in dimensions.iteritems()
if v is not None and isinstance(v, (int, float, str,
unicode))
and k is not None and isinstance(k, str))
self.dimensions = dimensions
self.path = path
self.value = value
self.raw_value = raw_value
self.timestamp = timestamp
self.precision = precision
self.metric_type = metric_type
