class StatsdStatsLogger(BaseStatsLogger):
def __init__(self, host='localhost', port=8125,
prefix='superset', statsd_client=None):
"""
Initializes from either params or a supplied, pre-constructed statsd client.
If statsd_client argument is given, all other arguments are ignored and the
supplied client will be used to emit metrics.
"""
if statsd_client:
self.client = statsd_client
else:
self.client = StatsClient(host=host, port=port, prefix=prefix)
def incr(self, key):
self.client.incr(key)
def decr(self, key):
self.client.decr(key)
def timing(self, key, value):
self.client.timing(key, value)
def gauge(self, key):
# pylint: disable=no-value-for-parameter
self.client.gauge(key)
class StatsdStatsLogger(BaseStatsLogger):
def __init__( # pylint: disable=super-init-not-called
self,
host: str = "localhost",
port: int = 8125,
prefix: str = "superset",
statsd_client: Optional[StatsClient] = None,
) -> None:
"""
Initializes from either params or a supplied, pre-constructed statsd client.
If statsd_client argument is given, all other arguments are ignored and the
supplied client will be used to emit metrics.
"""
if statsd_client:
self.client = statsd_client
else:
self.client = StatsClient(host=host, port=port, prefix=prefix)
def incr(self, key: str) -> None:
self.client.incr(key)
def decr(self, key: str) -> None:
self.client.decr(key)
def timing(self, key: str, value: float) -> None:
self.client.timing(key, value)
def gauge(self, key: str, value: float) -> None:
self.client.gauge(key, value)
class StaticticStatsD(object):
"""
Send stats to statsd.
"""
def __init__(self, hostname, host, port, prefix=None):
self.client = StatsClient(host, port, prefix=prefix)
self.hostname = hostname
def incr(self, metric, value=1, prefix=None):
"""
Increment 'metric' counter with 'value'.
"""
if prefix is not None:
metric = '%s.%s' % (prefix, metric)
self.client.incr(metric, value)
# separate metric for hostname
if self.hostname is not None:
metric = '%s.%s' % (self.hostname, metric)
self.client.incr(metric, value)
def timing(self, metric, value, prefix=None):
"""
Send 'metric' timing.
"""
if prefix is not None:
metric = '%s.%s' % (prefix, metric)
self.client.timing(metric, value)
# separate metric for hostname
if self.hostname is not None:
metric = '%s.%s' % (self.hostname, metric)
self.client.timing(metric, value)
class StatsdClient():
def __init__(self):
self.statsd_client = None
def init_app(self, app, *args, **kwargs):
self.active = app.config.get('STATSD_ENABLED')
self.namespace = "{}.notifications.{}.".format(
app.config.get('NOTIFY_ENVIRONMENT'),
app.config.get('NOTIFY_APP_NAME'))
if self.active:
self.statsd_client = StatsClient(
app.config.get('STATSD_HOST'),
app.config.get('STATSD_PORT'),
prefix=app.config.get('STATSD_PREFIX'))
def format_stat_name(self, stat):
return self.namespace + stat
def incr(self, stat, count=1, rate=1):
if self.active:
self.statsd_client.incr(self.format_stat_name(stat), count, rate)
def gauge(self, stat, count):
if self.active:
self.statsd_client.gauge(self.format_stat_name(stat), count)
def timing(self, stat, delta, rate=1):
if self.active:
self.statsd_client.timing(self.format_stat_name(stat), delta, rate)
def timing_with_dates(self, stat, start, end, rate=1):
if self.active:
delta = (start - end).total_seconds()
self.statsd_client.timing(self.format_stat_name(stat), delta, rate)
class StatsdWrapper:
"""Simple wrapper around the statsd client."""
statsd = None
def __init__(self, host, port, prefix):
if host:
self.statsd = StatsClient(
host=host,
port=port,
prefix=prefix,
)
def incr(self, *args):
if self.statsd:
self.statsd.incr(*args)
def decr(self, *args):
if self.statsd:
self.statsd.decr(*args)
def gauge(self, *args):
if self.statsd:
self.statsd.gauge(*args)
def timing(self, *args):
if self.statsd:
self.statsd.timing(*args)
def timer(self, *args):
if self.statsd:
self.statsd.timer(*args)
def set(self, *args):
if self.statsd:
self.statsd.set(*args)
class StatsdStatsLogger(BaseStatsLogger):
def __init__(self,
host="localhost",
port=8125,
prefix="superset",
statsd_client=None):
"""
Initializes from either params or a supplied, pre-constructed statsd client.
If statsd_client argument is given, all other arguments are ignored and the
supplied client will be used to emit metrics.
"""
if statsd_client:
self.client = statsd_client
else:
self.client = StatsClient(host=host, port=port, prefix=prefix)
def incr(self, key):
self.client.incr(key)
def decr(self, key):
self.client.decr(key)
def timing(self, key, value):
self.client.timing(key, value)
def gauge(self, key):
# pylint: disable=no-value-for-parameter
self.client.gauge(key)
class StatsDBackend(BaseBackend):
name = 'statsd'
def __init__(self, config):
self.config = config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'],
self.config['STATSD_PREFIX'])
def timing(self, stat_name, delta):
return self.statsd.timing(stat_name, delta, self.config['STATS_RATE'])
def incr(self, stat_name, count=1):
return self.statsd.incr(stat_name, count, self.config['STATS_RATE'])
def decr(self, stat_name, count=1):
return self.statsd.decr(stat_name, count, self.config['STATS_RATE'])
def gauge(self, stat_name, value, delta=False):
return self.statsd.gauge(stat_name, value, self.config['STATS_RATE'], delta)
class StatsdStatsLogger(BaseStatsLogger):
def __init__(self, host, port, prefix='superset'):
self.client = StatsClient(host=host, port=port, prefix=prefix)
def incr(self, key):
self.client.incr(key)
def decr(self, key):
self.client.decr(key)
def timing(self, key, value):
self.client.timing(key, value)
def gauge(self, key):
# pylint: disable=no-value-for-parameter
self.client.gauge(key)
class StatsdStatsLogger(BaseStatsLogger):
def __init__(self, host, port, prefix='superset'):
self.client = StatsClient(host=host, port=port, prefix=prefix)
def incr(self, key):
self.client.incr(key)
def decr(self, key):
self.client.decr(key)
def timing(self, key, value):
self.client.timing(key, value)
def gauge(self, key):
# pylint: disable=no-value-for-parameter
self.client.gauge(key)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'],
self.config['STATSD_PREFIX'])
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def set(self, *args, **kwargs):
return self.statsd.set(*args, **kwargs)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'], self.config['STATSD_PREFIX'])
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def set(self, *args, **kwargs):
return self.statsd.set(*args, **kwargs)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault("STATSD_HOST", "localhost")
self.config.setdefault("STATSD_PORT", 8125)
self.config.setdefault("STATSD_PREFIX", None)
self.app = app
self.statsd = StatsClient(
host=self.config["STATSD_HOST"], port=self.config["STATSD_PORT"], prefix=self.config["STATSD_PREFIX"]
)
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
bus = UART_Adapter('/dev/ttyUSB0')
stats = StatsClient('statsd', 8125, 'readtemp')
try:
with open('/boot/id.txt', 'r') as f:
sensor_id = f.readline().strip()
except:
sensor_id = 'unknown'
stats.gauge('online.%s' % sensor_id, 1)
led_toggle(1)
for rom in get_roms():
read_temperature(rom)
led_toggle(0)
elapsed_time = timer() - start
stats.timing('runtime.%s.elapsed' % sensor_id, int(1000 * elapsed_time))
stats.gauge('system.%s.cpu.temperature' % sensor_id,
read_cpu_temperature())
sleep_interval = CYCLE_TIME_SECONDS - elapsed_time
if sleep_interval > MIN_SLEEP_INTERVAL:
# Sleep some time before docker restarts the container
print('Sleeping for {0:.3f}s...'.format(sleep_interval))
time.sleep(sleep_interval)
elapsed_total = timer() - start
stats.timing('runtime.%s.total_elapsed' % sensor_id,
int(1000 * elapsed_total))
#conn.commit()
except:
exceptioncount+=1
exceptionlist.append(sys.exc_info()[0])
logger.exception("Exception:")
query2="update factual_duplicates_last_check set checked_on=now(),last_city_checked=%s"
values=list()
values.append(last_city_checked)
cur.execute(query2,values)
conn.commit()
print("Completed:",cityname,statename)
logger.info("Completed:%s %s",cityname,statename)
print ("total rows:",rowcount)
print ("duplicates:",duplicatecount)
print(exceptionlist)
print("total Exceptions:",exceptioncount)
dt = int((time.time() - start) )
statsd.timing('slept', dt)
print("total time taken (s):",dt)
logger.info("Total Rows:%s",rowcount)
logger.info("Duplicate Rows:%s",duplicatecount)
logger.info("Total Exceptions:%s",exceptioncount)
logger.info("Total time taken (s):%s",dt)
cur.close()
conn.close()
class Monitor():
def __init__(self, cliParam):
## 用作收集网络数据的worker
self.pool = Pool(processes=3)
##
self.cliParam = cliParam
##init influxdb statd
influxdb_config = cliParam["influxdb_config"]
self.sdc = StatsClient(host=check_format(
influxdb_config.get('host', '')),
port=influxdb_config.getint('port', 8120),
prefix="icmpping",
maxudpsize=512,
ipv6=False)
self.tmpdata = {}
def start(self):
fping_cmd = '%s %s' % (cliParam["fping_cmd"], " ".join(
cliParam["ip_list"]))
retry = 0
while retry < cliParam["fping_retry_max"]:
runtime = str(datetime.datetime.now())
status, out = commands.getstatusoutput(fping_cmd)
if status == 0 or status == 256:
break
else:
logger.error("status:%s,info:%s" % (status, out))
logger.warn('fping retry ..')
time.sleep(cliParam["fping_retry_interval"])
retry += 1
if retry > cliParam["fping_retry_max"]:
logger.error('fping retry count > retry_max')
sys.exit()
list_mtr_ips = []
for info in out.split('\n'):
#print info
cur_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
target = info.split(':')[0].strip()
if self.tmpdata.get(target) == None:
self.tmpdata[target] = {}
self.tmpdata['time'] = int(time.time() * 1000)
self.tmpdata[target]['hostname'] = check_format(
cliParam["targets_info"].get(target))
try:
self.tmpdata[target]['loss'] = int(
info.split('=')[1].split('/')[2].split('%')[0])
except Exception as e:
logger.error(self.tmpdata[target]['loss'])
logger.error(e)
if self.tmpdata[target].get('send_count') == None:
self.tmpdata[target]['send_count'] = 0
if self.tmpdata[target].get('warn_startTime') == None:
self.tmpdata[target]['warn_startTime'] = ''
if self.tmpdata[target]['loss'] < 100:
self.tmpdata[target]['max'] = Decimal(
info.split('/')[-1]).quantize(Decimal('0.00'))
self.tmpdata[target]['avg'] = Decimal(
info.split('/')[-2]).quantize(Decimal('0.00'))
self.tmpdata[target]['min'] = Decimal(
info.split('/')[-3].split('=')[-1]).quantize(
Decimal('0.00'))
# 写入数据influxdb
retry = 0
while retry < cliParam["retry_max"]:
try:
s_host = cliParam["my_name"]
s_ip = cliParam["my_public_ip"].replace('.', '_')
d_host = self.tmpdata[target]['hostname']
d_ip = target.replace('.', '_')
loss = int(info.split('=')[1].split('/')[0]) - int(
info.split('=')[1].split('/')[1])
data = '.'.join([s_host, s_ip, d_host, d_ip])
if self.tmpdata[target]['loss'] >= cliParam[
"send_data_loss_min"]:
self.sdc.incr('.'.join(['loss', data]), loss)
if self.tmpdata[target]['loss'] < 100:
self.sdc.timing('.'.join(['delay.max', data]),
self.tmpdata[target]['max'])
self.sdc.timing('.'.join(['delay.min', data]),
self.tmpdata[target]['min'])
self.sdc.timing('.'.join(['delay.avg', data]),
self.tmpdata[target]['avg'])
logger.info({
"s_host": s_host,
"s_ip": s_ip,
"d_host": d_host,
"d_ip": d_ip,
"loss": loss,
"data": data
})
break
else:
logger.info({
"s_host":
s_host,
"s_ip":
s_ip,
"d_host":
d_host,
"d_ip":
d_ip,
"loss":
loss,
"data":
data,
"info":
"not send data to influxdb when cur_loss:%s < send_data_loss_min:%s"
% (loss, self.send_data_loss_min)
})
break
except Exception as e:
logger.error("get fping info err : %s" % e)
time.sleep(cliParam["retry_interval"])
retry += 1
# 收集网络数据
if self.tmpdata[target]['loss'] >= cliParam["warn_loss"]\
and self.tmpdata[target]['send_count'] < cliParam["collect_max"] \
and cliParam["mtr_status"] == 'true':
## 延时发送mtr
list_mtr_ips.append(target)
if self.tmpdata[target]['loss'] < cliParam["warn_loss"]:
self.tmpdata[target]['send_count'] = 0
self.tmpdata[target]['warn_startTime'] = ''
if retry > cliParam["retry_max"]:
logger.error('write influxdb retry count > retry_max')
sys.exit()
if list_mtr_ips != []:
for target in list_mtr_ips:
collectParam = {
"target": target,
"mtr_cmd": cliParam["mtr_cmd"],
"ping_cmd": cliParam["ping_cmd"],
"fping_cmd": cliParam["fping_cmd"],
"my_name": cliParam["my_name"],
"my_public_ip": cliParam["my_public_ip"],
"mtr_logfile": cliParam["mtr_logfile"],
"loss_value": self.tmpdata[target]['loss'],
"runtime": runtime,
"send_mail_status": cliParam["send_mail_status"],
"target_hostname": cliParam["targets_info"].get(target)
}
if cliParam.get("admin_list", "") != "":
collectParam["admin_list"] = cliParam["admin_list"]
if self.tmpdata[target]['send_count'] < cliParam["collect_max"]:
##
self.pool.apply_async(collect_network_data, [collectParam])
##
self.tmpdata[target]['send_count'] += 1
logger.warn(self.tmpdata[target]['send_count'])
else:
if self.tmpdata[target]['warn_startTime'] == '':
self.tmpdata[target]['warn_startTime'] = cur_time
def run(self):
while True:
try:
self.start()
except Exception as e:
logger.error(e)
sys.exit(2)
class StatsD(object):
def __init__(self, app=None, config=None):
self.config = None
self.statsd = None
if app is not None:
self.init_app(app, config=config)
else:
self.app = None
def init_app(self, app, config=None):
if config is not None:
self.config = config
elif self.config is None:
self.config = app.config
self.config.setdefault('STATSD_HOST', 'localhost')
self.config.setdefault('STATSD_PORT', 8125)
self.config.setdefault('STATSD_PREFIX', None)
self.app = app
self.statsd = StatsClient(self.config['STATSD_HOST'],
self.config['STATSD_PORT'], self.config['STATSD_PREFIX'])
self.use_ms=self.config.get('STATSD_USEMS', True)
# Configure any of our middleware
self.setup_middleware()
def timer(self, *args, **kwargs):
return self.statsd.timer(*args, **kwargs)
def timing(self, *args, **kwargs):
return self.statsd.timing(*args, **kwargs)
def incr(self, *args, **kwargs):
return self.statsd.incr(*args, **kwargs)
def decr(self, *args, **kwargs):
return self.statsd.decr(*args, **kwargs)
def gauge(self, *args, **kwargs):
return self.statsd.gauge(*args, **kwargs)
def setup_middleware(self):
"""Helper to configure/setup any Flask-StatsD middleware"""
# Configure response time middleware (if desired)
self.config.setdefault('STATSD_CONFIGURE_MIDDLEWARE', True)
self.config.setdefault('STATSD_RESPONSE_METRIC_NAME', 'response.time')
self.config.setdefault('STATSD_NUMBER_OF_REQUESTS_METRIC_NAME', 'request.api')
self.config.setdefault('STATSD_RESPONSE_SAMPLE_RATE', 1)
self.config.setdefault('STATSD_RESPONSE_AUTO_TAG', True)
self.config.setdefault('STATSD_RESPONSE_ENDPOINT_TAG_FORMAT', 'endpoint_{0}')
self.config.setdefault('STATSD_RESPONSE_METHOD_TAG_FORMAT', 'method_{0}')
if self.config['STATSD_CONFIGURE_MIDDLEWARE']:
self.app.before_request(self.before_request)
self.app.after_request(self.after_request)
def before_request(self):
"""
statsd middleware handle for before each request
"""
# Set the request start time
g.flask_statsd_start_time = time.time()
g.flask_statsd_request_tags = []
# Add some default request tags
if self.config['STATSD_RESPONSE_AUTO_TAG']:
self.add_request_tags([
# Endpoint tag
self.config['STATSD_RESPONSE_ENDPOINT_TAG_FORMAT'].format(str(request.endpoint).lower()),
# Method tag
self.config['STATSD_RESPONSE_METHOD_TAG_FORMAT'].format(request.method.lower()),
])
# Send no of requests per second
metric = '.'.join([self.config['STATSD_NUMBER_OF_REQUESTS_METRIC_NAME'],
str(request.endpoint).lower(), request.method.lower()])
self.statsd.incr(metric, 1)
def after_request(self, response):
"""
statsd middleware handler for after each request
:param response: the response to be sent to the client
:type response: ``flask.Response``
:rtype: ``flask.Response``
"""
# Return early if we don't have the start time
if not hasattr(g, 'flask_statsd_start_time'):
return response
# Get the response time for this request
elapsed = time.time() - g.flask_statsd_start_time
# Convert the elapsed time to milliseconds if they want them
if self.use_ms:
elapsed = int(round(1000 * elapsed))
# Add some additional response tags
if self.config['STATSD_RESPONSE_AUTO_TAG']:
self.add_request_tags(['status_code_%s' % (response.status_code, )])
metric = self.config['STATSD_RESPONSE_METRIC_NAME']
tags = self.get_request_tags()
if tags:
metric = ".".join([metric] + tags)
# Emit our timing metric
self.statsd.timing(metric,
elapsed, rate=self.config['STATSD_RESPONSE_SAMPLE_RATE'])
# We ALWAYS have to return the original response
return response
def get_request_tags(self):
"""
Get the current list of tags set for this request
:rtype: list
"""
return getattr(g, 'flask_statsd_request_tags', [])
def add_request_tags(self, tags):
"""
Add the provided list of tags to the tags stored for this request
:param tags: tags to add to this requests tags
:type tags: list
:rtype: list
"""
# Get the current list of tags to append to
# DEV: We use this method since ``self.get_request_tags`` will ensure that we get a list back
current_tags = self.get_request_tags()
# Append our new tags, and return the new full list of tags for this request
g.flask_statsd_request_tags = current_tags + tags
return g.flask_statsd_request_tags
output = open('slow-{ts}.csv'.format(ts=ts[:10]), 'a')
line = "{ts};{rt};{slugs}".format(ts=ts, rt=rt,
slugs=";".join(slugs))
output.write(line)
output.write('\n')
print line
if action == 'users':
logger.setLevel(logging.INFO)
handler = logging.handlers.TimedRotatingFileHandler('users.log', when='D', interval=1)
handler.setLevel(logging.INFO)
logger.addHandler(handler)
statsd = StatsClient('localhost', 8125)
for a in TrackUsers(EventsHose(r, chan)):
t = a[2]
action = a[7]
statsd.timing('action.%s' % action, int(t))
logger.info(" ".join([str(b) for b in a]))
if action == 'errors':
log = logging.getLogger('raven')
log.setLevel(logging.DEBUG)
handler = logging.handlers.TimedRotatingFileHandler('raven.log', when='D', interval=1)
handler.setLevel(logging.DEBUG)
log.addHandler(handler)
hose = EventsHose(r, chan)
dumper = yaml.Dumper
for rq, query, message, ts, agent, source, body in TrackErrors(hose):
status = message['status']
print status
request = UNQUOTE.subn(r"\1", yaml.dump(query, allow_unicode=True,
default_flow_style=False).replace('!!python/unicode ', ''))[0]
print request