def cmd_hours_metrics():
days = {}
for account in CONFIG.accounts:
info(account.email, ':')
imap = connectIMAP(account)
try:
hoursMetricsAccount(imap, days)
finally:
imap.logout()
# open readonly b/c otherwise messages will be marked as read '\\Seen'
metrics = []
for day in days.keys():
counter = days[day]
tstamp = datetime.strptime(day, '%Y-%m-%d')
tstamp += timedelta(hours=12) # midday
tstamp = int(tstamp.strftime('%s'))
info('day %s/%d' % (day, tstamp))
for h in range(24):
info('%d: %d' % (h, counter[h]))
metrics.append(('daily.by_hour.%d.count' % h, counter[h], tstamp))
if CONFIG.graphiteHost is not None:
gc = GraphiteClient(prefix='spam',
graphite_server=CONFIG.graphiteHost,
graphite_port=CONFIG.graphitePort,
system_name='')
gc.send_list(metrics)
class GraphiteReporter(object):
def __init__(self, config, type='piautomator'):
self.g = None
self.type = type
self.config = config
def __getG(self):
def graphite_should_be_enabled():
has_graphite = self.g
graphite_enabled = self.config.get_setting(['graphite', 'enabled'],
False)
return not has_graphite and graphite_enabled
if graphite_should_be_enabled():
try:
self.g = GraphiteClient(
prefix='',
group='%s.' % self.type,
graphite_server=self.config.get_setting(
['graphite', 'host']))
except:
e = sys.exc_info()
logging.error("Unable to establish Graphite connection")
self.g = None
return self.g
def __destroyG(self):
if self.g:
try:
self.g.disconnect()
except:
None
self.g = None
def send_dict(self, data, timestamp=None):
g = self.__getG()
if g:
try:
g.send_dict(data, timestamp)
except:
self.__destroyG()
logging.error("Error sending to graphite")
def send(self, metric, value, timestamp=None):
g = self.__getG()
if g:
try:
g.send(metric, value, timestamp)
except:
self.__destroyG()
logging.error("Error sending to graphite")
class GraphiteReporter(object):
def __init__(self, config, type='piautomator'):
self.g = None
self.type = type
self.config = config
def __getG(self):
def graphite_should_be_enabled():
has_graphite = self.g
graphite_enabled = self.config.get_setting(['graphite', 'enabled'], False)
return not has_graphite and graphite_enabled
if graphite_should_be_enabled():
try:
self.g = GraphiteClient(prefix='', group='%s.' % self.type,
graphite_server=self.config.get_setting(['graphite', 'host']))
except:
e = sys.exc_info()
logging.error("Unable to establish Graphite connection")
self.g = None
return self.g
def __destroyG(self):
if self.g:
try:
self.g.disconnect()
except:
None
self.g = None
def send_dict(self, data, timestamp=None):
g = self.__getG()
if g:
try:
g.send_dict(data, timestamp)
except:
self.__destroyG()
logging.error("Error sending to graphite")
def send(self, metric, value, timestamp=None):
g = self.__getG()
if g:
try:
g.send(metric, value, timestamp)
except:
self.__destroyG()
logging.error("Error sending to graphite")
def __getG(self):
def graphite_should_be_enabled():
has_graphite = self.g
graphite_enabled = self.config.get_setting(['graphite', 'enabled'],
False)
return not has_graphite and graphite_enabled
if graphite_should_be_enabled():
try:
self.g = GraphiteClient(
prefix='',
group='%s.' % self.type,
graphite_server=self.config.get_setting(
['graphite', 'host']))
except:
e = sys.exc_info()
logging.error("Unable to establish Graphite connection")
self.g = None
return self.g
def write_player_data(url, players):
""" Writes a bunch of player data to the time-series database.
players is a list of dicts with the key "name" for the player's
username and another key for each stat we are tracking.
url is the url for the graphite instance to write to.
"""
timestamp = int(time.time())
points = []
for player in players:
for stat in player:
if stat != 'name':
points.append(
('grid.{}.{}'.format(stat, player['name']), player[stat]))
host, port = url.split(':')
port = int(port)
client = GraphiteClient(host, port, formatter=TemplateFormatter("{name}"))
client.send_list(points, timestamp=timestamp)
def recordDailyMetrics(emailLog, counts):
msg = '\nspam-rescore moves for the last 24 hours:\n'
total = 0
metrics = []
for email in counts.keys():
msg += '%s: %d message(s)\n' % (email, counts[email])
metrics.append(
('daily.by_email.' + email.replace('@', '_').replace('.', '_'),
counts[email]))
total += counts[email]
if CONFIG.emailDailySummary:
emailLog.info(msg)
metrics.append(('daily.total', total))
if CONFIG.graphiteHost is not None:
gc = GraphiteClient(prefix='spam',
graphite_server=CONFIG.graphiteHost,
graphite_port=CONFIG.graphitePort,
system_name='')
gc.send_list(metrics)
def __getG(self):
def graphite_should_be_enabled():
has_graphite = self.g
graphite_enabled = self.config.get_setting(['graphite', 'enabled'], False)
return not has_graphite and graphite_enabled
if graphite_should_be_enabled():
try:
self.g = GraphiteClient(prefix='', group='%s.' % self.type,
graphite_server=self.config.get_setting(['graphite', 'host']))
except:
e = sys.exc_info()
logging.error("Unable to establish Graphite connection")
self.g = None
return self.g
def recordSpamMetrics(messages):
if CONFIG.graphiteHost is not None:
gc = GraphiteClient(prefix='spam',
graphite_server=CONFIG.graphiteHost,
graphite_port=CONFIG.graphitePort,
system_name='')
durations = []
for m in messages:
if m.date:
durations.append(('duration.detect_minutes', m.ageMinutes()))
gc.send('daily.by_hour.%d.count' % m.date.hour, 1)
gc.send_list(durations)
def init_app(self, app):
"""Read config and set the hooks in place to monitor the application
:param app: The application to monitor
"""
config = app.config.get_namespace("FLASK_GRAPHITE_")
logger.info("config: {}".format(config))
config["graphite_server"] = config.pop("host", "localhost")
config["graphite_port"] = config.pop("port", 2023)
config.setdefault("group", "flask-graphite")
config.setdefault("autoreconnect", True)
config.pop("metric_template", None)
try:
app.graphite = GraphiteClient(**config)
except GraphiteSendException as e:
logger.error("can't instantiate graphite client: {}".format(e))
else:
logger.info("graphite client instantiated with config: %s", config)
for hook in default_hooks:
hook.register_into(app)
def cmd_hack():
#account = CONFIG.accounts[0]
#imap = connectIMAP(account)
#try:
# spamFolder = account.spamFolder
# info('check if %s exists:' % spamFolder)
# exists = imap.folder_exists(spamFolder)
# info('?', exists)
# if not exists:
# info('create folder:', imap.create_folder(spamFolder))
#finally:
# imap.logout()
#data = """spam-rescore moves for the last 24 hours:
#[email protected]: 0 message(s)
#[email protected]: 16 message(s)
#[email protected]: 17 message(s)"""
#data = "[email protected]: 2 message(s)"
#m = METRICS_PAT.findall(data)
#info(m)
# for s in data.split('\n'):
# info('try "%s":' % s)
# gc = GraphiteClient(prefix='test', graphite_server=CONFIG.graphiteHost, graphite_port=CONFIG.graphitePort, system_name='')
# for i in range(10):
# ts = time.time() - 1000 * i - i
# info('metric: %d' % i)
# gc.send('foo', i, ts)
# log = logging.getLogger('hack')
# log.info('hack running, conf is %s', str(CONFIG))
# counts = Counter()
# counts['*****@*****.**'] += 77
# counts['*****@*****.**'] += 88
# recordDailyMetrics(log, counts)
gc = GraphiteClient(prefix='test.by_minute',
graphite_server=CONFIG.graphiteHost,
graphite_port=CONFIG.graphitePort,
system_name='')
for i in range(1, 11):
info('%d ' % i)
gc.send('counters.a.count', i)
gc.send('counters.b.count', i * 2)
gc.send('counters.c.count', i * 3)
time.sleep(1)
import sys
import json
from time import sleep
from graphitesend import GraphiteClient, TemplateFormatter, GraphiteSendException
client = GraphiteClient('localhost',
2003,
formatter=TemplateFormatter("{name}"))
for line in sys.stdin:
data = json.loads(line)
stat = data['name']
if 'username' not in data['tags']:
continue
username = data['tags']['username']
metric = 'grid.{}.{}'.format(stat, username)
print(metric)
for pt in data['datapoints']:
(timestamp_ms, value, _) = pt
timestamp = int(timestamp_ms / 1000)
while True:
try:
client.send(metric, value, timestamp=timestamp)
break
except GraphiteSendException:
sleep(60)
from graphitesend import GraphiteClient
from json import loads
from kafka import KafkaConsumer
# Graphite host
GRAPHITE_HOSTNAME = "graphite"
# Kafka broker address and topic names definition
SERVER_URI = "<KAFKA_SERVER_URI>"
SERVER_PORT = "9092"
PREDICTION_TOPIC = "RAW_AND_PREDICTED"
# Kafka consumer setup
consumer = KafkaConsumer(f"{PREDICTION_TOPIC}",
bootstrap_servers=f"{SERVER_URI}:{SERVER_PORT}",
value_deserializer=lambda m: loads(m.decode('utf-8')))
# Graphite client setup
client = GraphiteClient(GRAPHITE_HOSTNAME)
# For every new message, send data to Graphite
for message in consumer:
key = message.key.decode('utf-8')
payload = message.value
client.send("predicted", float(payload['PREDICTED']))
client.send("real", float(payload['SILICA_CONCENTRATE']))
async def write_stage_test(self, graphite: GraphiteClient, app_id,
test_result, completion_time, job_group_alias):
key = f"app.{app_id}.job_group.{job_group_alias}.test.{self.test_name}"
graphite.send(key, test_result, completion_time.timestamp())
def default_client(mocker):
mocker.patch('socket.socket')
return GraphiteClient()
