def init_app(self, app):
self.config = app.config
enabled = self.config.setdefault("GRAPHITE_ENABLED", False)
self.app = app
if not enabled:
return
from greplin import scales
from greplin.scales.graphite import GraphitePeriodicPusher
from greplin.scales.meter import MeterStat
host = self.config.setdefault("GRAPHITE_HOST", "localhost")
port = self.config.setdefault("GRAPHITE_PORT", 2003)
prefix = self.config.setdefault("GRAPHITE_PREFIX", "doorman")
period = self.config.setdefault("GRAPHITE_REPORTING_INTERVAL", 60)
app.metrics = {}
for rule in app.url_map.iter_rules():
app.metrics[rule.endpoint] = scales.collection(
rule.endpoint,
MeterStat('count'),
scales.PmfStat('latency'),
)
app.graphite = GraphitePeriodicPusher(
host, port, period=period, prefix=prefix,
)
for rule in self.config.setdefault("GRAPHITE_ALLOW", ['*']):
app.graphite.allow(rule)
app.graphite.start()
return
def __init__(self, uri, config, masterGraph, mqtt, influx):
self.uri = uri
self.config = config
self.masterGraph = masterGraph
self.mqtt = mqtt
self.influx = influx
self.mqttTopic = self.topicFromConfig(self.config)
statPath = '/subscribed_topic/' + self.mqttTopic.decode(
'ascii').replace('/', '|')
scales.init(self, statPath)
self._mqttStats = scales.collection(statPath + '/incoming',
scales.IntStat('count'),
scales.RecentFpsStat('fps'))
rawBytes = self.subscribeMqtt()
rawBytes = rx.operators.do_action(self.countIncomingMessage)(rawBytes)
parsed = self.getParser()(rawBytes)
g = self.config
for conv in g.items(g.value(self.uri, ROOM['conversions'])):
parsed = self.conversionStep(conv)(parsed)
outputQuadsSets = rx.combine_latest(*[
self.makeQuads(parsed, plan)
for plan in g.objects(self.uri, ROOM['graphStatements'])
])
outputQuadsSets.subscribe_(self.updateQuads)
def __init__(self, min_level=NOTSET, include_loggers=NOTSET, exclude_loggers=NOTSET,
host=NOTSET, port=NOTSET, prefix=NOTSET, *args, **kwargs):
super(GraphiteProcessor, self).__init__(*args, **kwargs)
if min_level is NOTSET:
min_level = getattr(settings, 'GRAPHITE_LEVEL', logging.DEBUG)
if include_loggers is NOTSET:
include_loggers = getattr(settings, 'GRAPHITE_INCLUDE_LOGGERS', None)
if exclude_loggers is NOTSET:
exclude_loggers = getattr(settings, 'GRAPHITE_EXCLUDE_LOGGERS', None)
if host is NOTSET:
host = getattr(settings, 'GRAPHITE_HOST', '127.0.0.1')
if port is NOTSET:
port = getattr(settings, 'GRAPHITE_PORT', 2444)
if prefix is NOTSET:
prefix = getattr(settings, 'GRAPHITE_PREFIX', 'sentry')
self.min_level = min_level
self.include_loggers = include_loggers
self.exclude_loggers = exclude_loggers
self.host = host
self.port = port
self.prefix = prefix
# self.client = Client(host=self.host, port=self.port)
self.stats = scales.collection('/sentry', scales.IntStat('totalEvents'),
scales.IntDictStat('eventsByTop'))
global pusher
if pusher is None:
pusher = graphite.GraphitePeriodicPusher(self.host, self.port, self.prefix)
pusher.start()
def __init__(self, cluster_proxy):
log.debug("Starting metric capture")
self.stats = scales.collection('/cassandra',
scales.PmfStat('request_timer'),
scales.IntStat('connection_errors'),
scales.IntStat('write_timeouts'),
scales.IntStat('read_timeouts'),
scales.IntStat('unavailables'),
scales.IntStat('other_errors'),
scales.IntStat('retries'),
scales.IntStat('ignores'),
# gauges
scales.Stat('known_hosts',
lambda: len(cluster_proxy.metadata.all_hosts())),
scales.Stat('connected_to',
lambda: len(set(chain.from_iterable(s._pools.keys() for s in cluster_proxy.sessions)))),
scales.Stat('open_connections',
lambda: sum(sum(p.open_count for p in s._pools.values()) for s in cluster_proxy.sessions)))
self.request_timer = self.stats.request_timer
self.connection_errors = self.stats.connection_errors
self.write_timeouts = self.stats.write_timeouts
self.read_timeouts = self.stats.read_timeouts
self.unavailables = self.stats.unavailables
self.other_errors = self.stats.other_errors
self.retries = self.stats.retries
self.ignores = self.stats.ignores
self.known_hosts = self.stats.known_hosts
self.connected_to = self.stats.connected_to
self.open_connections = self.stats.open_connections
def __init__(self, graph, uri, pi, pinNumber):
self.graph, self.uri, self.pi = graph, uri, pi
self.pinNumber = pinNumber
scales.init(self, self.__class__.__name__)
self.stats = scales.collection(self.__class__.__name__,
scales.PmfStat('poll'),
scales.PmfStat('output'),
)
self.hostStateInit()
def __init__(self, redis):
"""
"""
self.log = logging.getLogger("{}.Gymkhana".format(__name__))
self.redis = redis
self.stats = scales.collection(
'/ghymkhana',
scales.IntDictStat('competition_add'),
scales.IntDictStat('competition_recovery'),
scales.IntDictStat('competition_update'),
scales.IntDictStat('competition_validation_failed'),
scales.IntDictStat('competition_removed'),
)
def testCollection(self):
"""Tests for a stat collection."""
collection = scales.collection('/thePath', scales.IntStat('count'),
scales.IntDictStat('histo'))
collection.count += 100
collection.histo['cheese'] += 12300
collection.histo['cheese'] += 45
self.assertEquals(
{'thePath': {
'count': 100,
'histo': {
'cheese': 12345
}
}}, scales.getStats())
def testCollection(self):
"""Tests for a stat collection."""
collection = scales.collection('/thePath', scales.IntStat('count'), scales.IntDictStat('histo'))
collection.count += 100
collection.histo['cheese'] += 12300
collection.histo['cheese'] += 45
self.assertEquals({
'thePath': {
'count': 100,
'histo': {
'cheese': 12345
}
}
}, scales.getStats())
def __init__(self, cluster_proxy):
log.debug("Starting metric capture")
self.stats_name = 'cassandra-{0}'.format(str(self._stats_counter))
Metrics._stats_counter += 1
self.stats = scales.collection(
self.stats_name,
scales.PmfStat('request_timer'),
scales.IntStat('connection_errors'),
scales.IntStat('write_timeouts'),
scales.IntStat('read_timeouts'),
scales.IntStat('unavailables'),
scales.IntStat('other_errors'),
scales.IntStat('retries'),
scales.IntStat('ignores'),
# gauges
scales.Stat('known_hosts',
lambda: len(cluster_proxy.metadata.all_hosts())),
scales.Stat(
'connected_to', lambda: len(
set(
chain.from_iterable(s._pools.keys() for s in
cluster_proxy.sessions)))),
scales.Stat(
'open_connections', lambda: sum(
sum(p.open_count for p in s._pools.values())
for s in cluster_proxy.sessions)))
# TODO, to be removed in 4.0
# /cassandra contains the metrics of the first cluster registered
if 'cassandra' not in scales._Stats.stats:
scales._Stats.stats['cassandra'] = scales._Stats.stats[
self.stats_name]
self.request_timer = self.stats.request_timer
self.connection_errors = self.stats.connection_errors
self.write_timeouts = self.stats.write_timeouts
self.read_timeouts = self.stats.read_timeouts
self.unavailables = self.stats.unavailables
self.other_errors = self.stats.other_errors
self.retries = self.stats.retries
self.ignores = self.stats.ignores
self.known_hosts = self.stats.known_hosts
self.connected_to = self.stats.connected_to
self.open_connections = self.stats.open_connections
def __init__(self, cluster_proxy):
log.debug("Starting metric capture")
self.stats_name = "cassandra-{0}".format(str(self._stats_counter))
Metrics._stats_counter += 1
self.stats = scales.collection(
self.stats_name,
scales.PmfStat("request_timer"),
scales.IntStat("connection_errors"),
scales.IntStat("write_timeouts"),
scales.IntStat("read_timeouts"),
scales.IntStat("unavailables"),
scales.IntStat("other_errors"),
scales.IntStat("retries"),
scales.IntStat("ignores"),
# gauges
scales.Stat("known_hosts", lambda: len(cluster_proxy.metadata.all_hosts())),
scales.Stat(
"connected_to", lambda: len(set(chain.from_iterable(s._pools.keys() for s in cluster_proxy.sessions)))
),
scales.Stat(
"open_connections",
lambda: sum(sum(p.open_count for p in s._pools.values()) for s in cluster_proxy.sessions),
),
)
# TODO, to be removed in 4.0
# /cassandra contains the metrics of the first cluster registered
if "cassandra" not in scales._Stats.stats:
scales._Stats.stats["cassandra"] = scales._Stats.stats[self.stats_name]
self.request_timer = self.stats.request_timer
self.connection_errors = self.stats.connection_errors
self.write_timeouts = self.stats.write_timeouts
self.read_timeouts = self.stats.read_timeouts
self.unavailables = self.stats.unavailables
self.other_errors = self.stats.other_errors
self.retries = self.stats.retries
self.ignores = self.stats.ignores
self.known_hosts = self.stats.known_hosts
self.connected_to = self.stats.connected_to
self.open_connections = self.stats.open_connections
def gatherProcessStats():
procStats = scales.collection(
'/process',
scales.DoubleStat('time'),
scales.DoubleStat('cpuPercent'),
scales.DoubleStat('memMb'),
)
proc = psutil.Process()
lastCpu = [0.]
def updateTimeStat():
now = time.time()
procStats.time = round(now, 3)
if now - lastCpu[0] > 3:
procStats.cpuPercent = round(proc.cpu_percent(),
6) # (since last call)
lastCpu[0] = now
procStats.memMb = round(proc.memory_info().rss / 1024 / 1024, 6)
task.LoopingCall(updateTimeStat).start(.1)
from pyzabbix import ZabbixSender, ZabbixMetric
from configparser import ConfigParser
from io import StringIO
try:
import _thread as thread
except ImportError:
import thread
log = logging.getLogger(__name__)
stats = scales.collection(
'/celery',
scales.IntStat('started'),
scales.IntStat('succeeded'),
scales.IntStat('failed'),
scales.IntStat('retried'),
scales.PmfStat('queuetime'),
scales.PmfStat('runtime'),
)
stats_queue = type('Stats:queues', (object, ), {})()
scales._Stats.initChild(stats_queue, 'queues', '', stats)
def task_handler(fn):
@wraps(fn)
def wrapper(self, event):
self.state.event(event)
task = self.state.tasks.get(event['uuid'])
return fn(self, event, task)
from greplin import scales
from tornado import ioloop, web, httpserver, gen
STATS = scales.collection('/web', scales.PmfStat('latency'))
class TimedHandler(web.RequestHandler):
@gen.coroutine
def get(self):
with STATS.latency.time():
self.write('finished')
if __name__ == "__main__":
application = web.Application([
(r"/", TimedHandler),
])
server = httpserver.HTTPServer(application)
server.bind(8888)
server.start(0)
ioloop.IOLoop.instance().start()
import os, contextlib
try:
from rdflib.Graph import Graph
except ImportError:
from rdflib import Graph
from rdflib import Namespace
from rdflib.parser import StringInputSource
from FuXi.Rete.Util import generateTokenSet
from FuXi.Rete import ReteNetwork
from FuXi.Rete.RuleStore import N3RuleStore
from greplin import scales
STATS = scales.collection('/web',
scales.PmfStat('readRules'))
from escapeoutputstatements import escapeOutputStatements
ROOM = Namespace("http://projects.bigasterisk.com/room/")
def _loadAndEscape(ruleStore, n3, outputPatterns):
ruleGraph = Graph(ruleStore)
# Can't escapeOutputStatements in the ruleStore since it
# doesn't support removals. Can't copy plainGraph into
# ruleGraph since something went wrong with traversing the
# triples inside quoted graphs, and I lose all the bodies
# of my rules. This serialize/parse version is very slow (400ms),
# but it only runs when the file changes.
plainGraph = Graph()
plainGraph.parse(StringInputSource(n3), format='n3') # for inference
import time, logging, math, os, sys, socket
from influxdb import InfluxDBClient
from rdflib import Namespace
from twisted.internet import task
from greplin import scales
log = logging.getLogger()
ROOM = Namespace('http://projects.bigasterisk.com/room/')
stats = scales.collection(
'/export_to_influxdb',
scales.PmfStat('exportToInflux'),
)
class RetentionPolicies(object):
def __init__(self, influx):
self.influx = influx
self.createdPolicies = set() # days
def getCreatedPolicy(self, days):
name = 'ret_%d_day' % days
if days not in self.createdPolicies:
self.influx.create_retention_policy(name,
duration='%dd' % days,
replication='1')
self.createdPolicies.add(days)
return name
class InfluxExporter(object):
def __init__(self, configGraph, influxHost='bang5'):
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from influxdb import InfluxDBClient
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
import cyclone.web
from rdflib import Namespace, Literal
from standardservice.logsetup import log, verboseLogging
from patchablegraph import PatchableGraph, CycloneGraphHandler, CycloneGraphEventsHandler
from private_config import deviceIp, cloudId, installId, macId, periodSec
ROOM = Namespace("http://projects.bigasterisk.com/room/")
STATS = scales.collection('/root',
scales.PmfStat('poll'),
)
authPlain = cloudId + ':' + installId
auth = binascii.b2a_base64(authPlain.encode('ascii')).strip(b'=\n')
class Poller(object):
def __init__(self, influx, graph):
self.influx = influx
self.graph = graph
reactor.callLater(0, self.poll)
@STATS.poll.time()
@inlineCallbacks
def poll(self):
ret = None
STATS = scales.collection(
"/api",
scales.IntStat('notfound'),
scales.IntStat('validation'),
scales.IntStat('errors'),
scales.IntStat('success'),
# account types
scales.PmfStat('get_account_type'),
scales.PmfStat('all_account_types'),
scales.PmfStat('add_account_type'),
scales.PmfStat('delete_account_type'),
scales.PmfStat('update_account_type'),
# accounts
scales.PmfStat('get_account'),
scales.PmfStat('all_accounts'),
scales.PmfStat('add_account'),
scales.PmfStat('delete_account'),
scales.PmfStat('update_account'),
scales.PmfStat('get_account_transactions'),
# transactions
scales.PmfStat('get_transaction'),
scales.PmfStat('all_transactions'),
scales.PmfStat('add_transaction'),
scales.PmfStat('delete_transaction'),
scales.PmfStat('update_transaction'),
scales.PmfStat('upload_transactions'),
)
from cyclone.httpclient import fetch
import cyclone
import logging, time, json, random, string, traceback
from logsetup import log, enableTwistedLog
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from export_to_influxdb import InfluxExporter
from tags import NfcDevice, FakeNfc, NfcError, AuthFailedError
ROOM = Namespace('http://projects.bigasterisk.com/room/')
ctx = ROOM['frontDoorWindowRfidCtx']
STATS = scales.collection(
'/root',
scales.PmfStat('cardReadPoll'),
scales.IntStat('newCardReads'),
)
class OutputPage(cyclone.web.RequestHandler):
def put(self):
arg = self.request.arguments
if arg.get('s') and arg.get('p'):
self._onQueryStringStatement(arg['s'][-1], arg['p'][-1],
self.request.body)
else:
self._onGraphBodyStatements(self.request.body,
self.request.headers)
post = put
from rdfdb.patch import Patch
from rdfdb.rdflibpatch import inContext
log = logging.getLogger()
logging.getLogger('serial').setLevel(logging.WARN)
ROOM = Namespace('http://projects.bigasterisk.com/room/')
HOST = Namespace('http://bigasterisk.com/ruler/host/')
ACTION_BASE = 10 # higher than any of the fixed command numbers
hostname = socket.gethostname()
CTX = ROOM['arduinosOn%s' % hostname]
STATS = scales.collection('/root',
)
etcd = etcd3.client(host='bang6', port=9022)
class Config(object):
def __init__(self, masterGraph, slowMode=False):
self.masterGraph = masterGraph
self.slowMode = slowMode
self.configGraph = ConjunctiveGraph()
self.etcPrefix = 'arduino/'
self.boards = []
self.reread()
from greplin import scales
from greplin.scales.meter import MeterStat
import config
import pusher
STATS = scales.collection(
"/web",
scales.IntStat('errors'),
scales.IntStat('success'),
scales.PmfStat('latency'),
scales.IntStat('counter'),
MeterStat('hits'),
)
def main():
"""Periodically send metrics out"""
stat_server = pusher.PeriodicPusher("localhost", 5001, "/stats", 5)
for allowed in config.STAT_RULES_ALLOWED:
stat_server.allow(allowed)
stat_server.run()
if __name__ == "__main__":
main()
pp-latchpony-service
This provides the views which are used in the dispatch routing set up.
PythonPro Limited
"""
import pkg_resources
from greplin import scales
from pyramid.view import view_config
STATS = scales.collection(
'/status',
scales.IntStat('ping_called'),
)
@view_config(route_name='home', request_method='GET', renderer='json')
def status(request):
"""This is used to 'ping' the web service to check if it's running.
:returns: a status dict which the configured view will return as JSON.
The dict has the form::
dict(
status="ok",
name="<project name>",
version="<egg version of pp.latchpony.service>"
import devices
from export_to_influxdb import InfluxExporter
log = logging.getLogger()
logging.getLogger('serial').setLevel(logging.WARN)
ROOM = Namespace('http://projects.bigasterisk.com/room/')
HOST = Namespace('http://bigasterisk.com/ruler/host/')
hostname = socket.gethostname()
CTX = ROOM['pi/%s' % hostname]
STATS = scales.collection('/root',
scales.PmfStat('configReread'),
scales.IntStat('pollException'),
scales.PmfStat('boardPoll'),
scales.PmfStat('sendOneshot'),
scales.PmfStat('outputStatements'),
)
def patchRandid():
"""
I'm concerned urandom is slow on raspberry pi, and I'm adding to
graphs a lot. Unclear what the ordered return values might do to
the balancing of the graph.
"""
_id_serial = [1000]
def randid():
_id_serial[0] += 1
return _id_serial[0]
import rdflib.plugins.memory
rdflib.plugins.memory.randid = randid
from rdfdb.patch import Patch
from rdfdb.rdflibpatch import inContext
log = logging.getLogger()
logging.getLogger('serial').setLevel(logging.WARN)
ROOM = Namespace('http://projects.bigasterisk.com/room/')
HOST = Namespace('http://bigasterisk.com/ruler/host/')
ACTION_BASE = 10 # higher than any of the fixed command numbers
hostname = socket.gethostname()
CTX = ROOM['arduinosOn%s' % hostname]
STATS = scales.collection('/root',
)
etcd = etcd3.client(host='bang6', port=9022)
class Config(object):
def __init__(self, masterGraph, slowMode=False):
self.masterGraph = masterGraph
self.slowMode = slowMode
self.configGraph = ConjunctiveGraph()
self.etcPrefix = 'arduino/'
self.boards = []
self.reread()
from __future__ import absolute_import, division, print_function
from greplin import scales
import json
import pyramid.config
import pytest
import webtest
STATS = scales.collection('/', scales.IntStat('errors'))
STATS.errors += 1
@pytest.fixture
def app():
config = pyramid.config.Configurator()
config.include('pyramid_scales')
return config.make_wsgi_app()
@pytest.fixture
def browser(app):
return webtest.TestApp(app)
def test_displays_metrics_as_html(browser):
r = browser.get('/scales/', status=200)
assert (b'<span class="key">errors</span> <span class="int">1</span>'
in r.body)
def test_displays_metrics_as_json(browser):
from dateutil.tz import tzlocal
from dateutil.relativedelta import relativedelta, FR
from rdflib import Namespace, Literal
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from twilight import isWithinTwilight
from logsetup import log, enableTwistedLog
from rdfdoc import Doc
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection('/root',
scales.PmfStat('update'),
)
@STATS.update.time()
def update(masterGraph):
stmt = lambda s, p, o: masterGraph.patchObject(ROOM.environment, s, p, o)
now = datetime.datetime.now(tzlocal())
stmt(DEV.environment, ROOM.localHour, Literal(now.hour))
stmt(DEV.environment, ROOM.localTimeToMinute,
Literal(now.strftime("%H:%M")))
stmt(DEV.environment, ROOM.localTimeToSecond,
Literal(now.strftime("%H:%M:%S")))
import contextlib
import datetime
from greplin import scales
import logger
STATS = scales.collection('/goaldecider',
scales.PmfStat('duration')
)
def write(filename=None):
try:
if filename is None:
filename=logger.filepath.replace(".py",".json")
logger.dbg("Writing metrics to {}".format(filename))
scales.dumpStatsTo(filename)
except Exception as e:
logger.dbg("Exception in stats writing")
logger.log_exception(e)
@contextlib.contextmanager
def simple_timer(name):
start_time = datetime.datetime.now()
yield
delta = datetime.datetime.now() - start_time
logger.log("{} duration: {} s".format(name, delta.total_seconds()))
return False
import logging
from greplin import scales
from pyramid.view import view_config
def get_log(extra=None):
m = "{}.{}".format(__name__, extra) if extra else __name__
return logging.getLogger(m)
STATS = scales.collection(
"/gymkhana",
scales.PmfStat("listing_recovery"),
scales.PmfStat("recover_competition"),
scales.PmfStat("load_competition"),
scales.PmfStat("dump_competition"),
scales.PmfStat("can_perform"),
scales.PmfStat("slot_types_for"),
)
@view_config(route_name="listing", request_method="GET", renderer="json")
def competition_listing(request):
"""Return a JSON list of competitions housed in the Gymkhana.
"""
results = []
# show how long its take to recover the list of org_ids we are housing:
with STATS.listing_recovery.time():
import treq
import math
import time
from twisted.internet.inotify import INotify
from twisted.python.filepath import FilePath
from light9 import networking
from light9.namespaces import L9, RDF
from light9.vidref.musictime import MusicTime
from light9.effect import effecteval
from greplin import scales
from txzmq import ZmqEndpoint, ZmqFactory, ZmqPushConnection
log = logging.getLogger('sequencer')
stats = scales.collection('/sequencer/',
scales.PmfStat('update'),
scales.DoubleStat('recentFps'),
)
_zmqClient=None
class TwistedZmqClient(object):
def __init__(self, service):
zf = ZmqFactory()
e = ZmqEndpoint('connect', 'tcp://%s:%s' % (service.host, service.port))
self.conn = ZmqPushConnection(zf, e)
def send(self, msg):
self.conn.push(msg)
def toCollectorJson(client, session, settings):
return json.dumps({'settings': settings,
#!/usr/bin/python
import os.path
import tornado.ioloop
from tornadoes import ESConnection
from greplin import scales
from greplin.scales.meter import MeterStat
import json
from hashlib import md5
import re
import unicodedata
STATS = scales.collection('/index', MeterStat('docs'))
BASE_PATH = '/Users/jisaacso/Documents/projects/bayes-impact/team-thorn/data/escort_all'
FBDUMP = os.path.join(BASE_PATH, 'escort_all.tsv')
es = ESConnection('localhost', 9200)
es.httprequest_kwargs = {
'request_timeout': 1500.00,
'connect_timeout': 1500.00
}
wspaceNuker = re.compile(' +')
def fold_accents(raw):
if type(raw) == str:
raw = unicode(raw, 'utf-8')
return ''.join([c for c in unicodedata.normalize('NFKD', raw).encode('ascii', 'ignore')])
from __future__ import absolute_import, division, print_function
from greplin import scales
import json
import pyramid.config
import pytest
import webtest
STATS = scales.collection(
'/',
scales.IntStat('errors')
)
STATS.errors += 1
@pytest.fixture
def app():
config = pyramid.config.Configurator()
config.include('pyramid_scales')
return config.make_wsgi_app()
@pytest.fixture
def browser(app):
return webtest.TestApp(app)
def test_displays_metrics_as_html(browser):
r = browser.get('/scales/', status=200)
assert (
from cycloneerr import PrettyErrorHandler
from docopt import docopt
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from mqtt_client import MqttClient
from rdflib import Namespace
from standardservice.logsetup import log, verboseLogging
from twisted.internet import reactor
import rdf_over_http
ROOM = Namespace('http://projects.bigasterisk.com/room/')
STATS = scales.collection(
'/root',
scales.PmfStat('putRequests'),
scales.PmfStat('statement'),
scales.PmfStat('mqttPublish'),
)
devs = {
ROOM['kitchenLight']: {
'root': 'h801_skylight',
'hasWhite': True,
},
ROOM['kitchenCounterLight']: {
'root': 'h801_counter',
'hasWhite': True,
},
ROOM['livingLampShelf']: {
'root': 'sonoff_0/switch/sonoff_basic_relay/command',
'values': 'binary',
#!/usr/bin/python
import os.path
import tornado.ioloop
from tornadoes import ESConnection
from greplin import scales
from greplin.scales.meter import MeterStat
import json
from hashlib import md5
import re
import unicodedata
STATS = scales.collection('/index', MeterStat('docs'))
BASE_PATH = '/Users/jisaacso/Documents/projects/bayes-impact/team-thorn/data/escort_all'
FBDUMP = os.path.join(BASE_PATH, 'escort_all.tsv')
es = ESConnection('localhost', 9200)
es.httprequest_kwargs = {
'request_timeout': 1500.00,
'connect_timeout': 1500.00
}
wspaceNuker = re.compile(' +')
def fold_accents(raw):
if type(raw) == str:
raw = unicode(raw, 'utf-8')
from rdflibtrig import addTrig
from graphop import graphEqual
from greplin import scales
from patchablegraph.patchsource import ReconnectingPatchSource
from rdfdb.rdflibpatch import patchQuads
log = logging.getLogger('fetch')
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection('/web',
scales.PmfStat('combineGraph'),
)
def parseRdf(text, contentType):
g = Graph()
g.parse(StringInputSource(text), format={
'text/n3': 'n3',
}[contentType])
return g
class RemoteData(object):
def __init__(self, onChange):
self.onChange = onChange
self.graph = ConjunctiveGraph()
self.patchSource = ReconnectingPatchSource(
URIRef('http://bang:9072/graph/home'),
from twisted.python.failure import Failure
from twisted.python.filepath import FilePath
import cyclone.web, cyclone.websocket
from rdflib import ConjunctiveGraph, URIRef, Graph
import twisted.internet.error
from rdfdb.file_vs_uri import correctToTopdirPrefix, fileForUri, uriFromFile, DirUriMap
from rdfdb.graphfile import GraphFile, PatchCb, GetSubgraph
from rdfdb.patch import Patch, ALLSTMTS
from rdfdb.rdflibpatch import patchQuads
from standardservice.scalessetup import gatherProcessStats
gatherProcessStats()
stats = scales.collection(
'/webServer',
scales.IntStat('clients'),
scales.IntStat('liveClients'),
scales.PmfStat('setAttr'),
)
graphStats = scales.collection(
'/graph',
scales.IntStat('statements'),
scales.RecentFpsStat('patchFps'),
)
fileStats = scales.collection(
'/file',
scales.IntStat('mappedGraphFiles'),
)
log = logging.getLogger('rdfdb')
import logging
import time
from greplin import scales
logger = logging.getLogger('sociallists.events')
STATS = scales.collection(
'/sociallists',
scales.PmfStat('feed_update'),
scales.IntStat('thumbnail_fetched_from_summary'),
scales.IntStat('thumbnail_fetched_from_content'),
scales.IntStat('thumbnail_fetched_from_link'),
scales.IntStat('thumbnail_fetched_not_found'),
scales.IntStat('thumbnail_is_direct'),
scales.IntStat('thumbnail_is_open_graph'),
scales.IntStat('thumbnail_is_twitter'),
scales.IntStat('thumbnail_is_link_rel'),
scales.IntStat('thumbnail_is_known_goodness'),
scales.IntStat('thumbnail_is_not_supported'),
scales.IntStat('thumbnail_is_img_tag'),
)
def log_stats():
thumbnail_total = (
STATS.thumbnail_fetched_from_link +
STATS.thumbnail_fetched_not_found +
STATS.thumbnail_fetched_from_content +
STATS.thumbnail_fetched_from_summary
from inference import infer, readRules
from actions import Actions
from inputgraph import InputGraph
from escapeoutputstatements import unquoteOutputStatements
sys.path.append("../../lib")
from logsetup import log
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
NS = {'': ROOM, 'dev': DEV}
STATS = scales.collection('/web',
scales.PmfStat('graphChanged'))
class Reasoning(object):
def __init__(self):
self.prevGraph = None
self.actions = Actions(sendToLiveClients)
self.rulesN3 = "(not read yet)"
self.inferred = Graph() # gets replaced in each graphChanged call
self.inputGraph = InputGraph([], self.graphChanged)
self.inputGraph.updateFileData()
def updateRules(self):
rulesPath = 'rules.n3'
from rdflib import URIRef, Namespace, Literal, RDF, RDFS, XSD, ConjunctiveGraph
from twisted.internet import reactor, task
import cyclone.web
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from standardservice.logsetup import log, verboseLogging
DEV = Namespace("http://projects.bigasterisk.com/device/")
ROOM = Namespace("http://projects.bigasterisk.com/room/")
ctx = DEV['dhcp']
STATS = scales.collection(
'/root',
scales.PmfStat('readLeases'),
scales.IntStat('filesDidntChange'),
)
def timeLiteral(dt):
return Literal(dt.replace(tzinfo=tzlocal()).isoformat(),
datatype=XSD.dateTime)
def macUri(macAddress: str) -> URIRef:
return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
class Poller:
def __init__(self, graph):
import os, contextlib
try:
from rdflib.Graph import Graph
except ImportError:
from rdflib import Graph
from rdflib import Namespace
from rdflib.parser import StringInputSource
from FuXi.Rete.Util import generateTokenSet
from FuXi.Rete import ReteNetwork
from FuXi.Rete.RuleStore import N3RuleStore
from greplin import scales
STATS = scales.collection('/web', scales.PmfStat('readRules'))
from escapeoutputstatements import escapeOutputStatements
ROOM = Namespace("http://projects.bigasterisk.com/room/")
def _loadAndEscape(ruleStore, n3, outputPatterns):
ruleGraph = Graph(ruleStore)
# Can't escapeOutputStatements in the ruleStore since it
# doesn't support removals. Can't copy plainGraph into
# ruleGraph since something went wrong with traversing the
# triples inside quoted graphs, and I lose all the bodies
# of my rules. This serialize/parse version is very slow (400ms),
# but it only runs when the file changes.
plainGraph = Graph()
from dateutil.tz import tzlocal
from rdflib import URIRef, Namespace, Literal, RDF, RDFS, XSD, ConjunctiveGraph
from twisted.internet import reactor, task
import cyclone.web
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from standardservice.logsetup import log, verboseLogging
DEV = Namespace("http://projects.bigasterisk.com/device/")
ROOM = Namespace("http://projects.bigasterisk.com/room/")
ctx = DEV['dhcp']
STATS = scales.collection('/root',
scales.PmfStat('readLeases'),
scales.IntStat('filesDidntChange'),
)
def timeLiteral(dt):
return Literal(dt.replace(tzinfo=tzlocal()).isoformat(),
datatype=XSD.dateTime)
def macUri(macAddress: str) -> URIRef:
return URIRef("http://bigasterisk.com/mac/%s" % macAddress.lower())
class Poller:
def __init__(self, graph):
self.graph = graph
self.fileTimes = {'/opt/dnsmasq/10.1/leases': 0, '/opt/dnsmasq/10.2/leases': 0}
task.LoopingCall(self.poll).start(2)
milliC = open('/sys/class/thermal/thermal_zone0/temp').read().strip()
c = float(milliC) / 1000.
f = c * 1.8 + 32
return [
(self.uri, ROOM['temperatureF'], Literal(round(f, 3), datatype=XSD['decimal'])),
]
def watchPrefixes(self):
# these uris will become dynamic! see note on watchPrefixes
# about eliminating it.
return [(self.uri, ROOM['temperatureF']),
]
pixelStats = scales.collection('/rgbPixels',
scales.PmfStat('updateOutput'),
scales.PmfStat('currentColors'),
scales.PmfStat('poll'),
)
@register
class RgbPixels(DeviceType):
"""chain of ws2812 rgb pixels on pin GPIO18"""
deviceType = ROOM['RgbPixels']
def hostStateInit(self):
self.anim = RgbPixelsAnimation(self.graph, self.uri, self.updateOutput)
log.debug('%s maxIndex = %s', self.uri, self.anim.maxIndex())
self.neo = rpi_ws281x.Adafruit_NeoPixel(self.anim.maxIndex() + 1, pin=18)
self.neo.begin()
colorOrder, stripType = self.anim.getColorOrder(self.graph, self.uri)
from dateutil.tz import tzlocal
from dateutil.relativedelta import relativedelta, FR
from rdflib import Namespace, Literal
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from patchablegraph import PatchableGraph, CycloneGraphEventsHandler, CycloneGraphHandler
from twilight import isWithinTwilight
from standardservice.logsetup import log, verboseLogging
from rdfdoc import Doc
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection(
'/root',
scales.PmfStat('update'),
)
class CycloneGraphEventsHandlerWithCors(CycloneGraphEventsHandler):
def flush(self):
self.set_header("Access-Control-Allow-Origin", "*")
return CycloneGraphEventsHandler.flush(self)
@STATS.update.time()
def update(masterGraph):
stmt = lambda s, p, o: masterGraph.patchObject(ROOM.environment, s, p, o)
now = datetime.datetime.now(tzlocal())
import cyclone.web
from cyclone.httpclient import fetch
import cyclone
import logging, time, json, random, string, traceback
from logsetup import log, enableTwistedLog
from greplin import scales
from greplin.scales.cyclonehandler import StatsHandler
from export_to_influxdb import InfluxExporter
from tags import NfcDevice, FakeNfc, NfcError, AuthFailedError
ROOM = Namespace('http://projects.bigasterisk.com/room/')
ctx = ROOM['frontDoorWindowRfidCtx']
STATS = scales.collection('/root',
scales.PmfStat('cardReadPoll'),
scales.IntStat('newCardReads'),
)
class OutputPage(cyclone.web.RequestHandler):
def put(self):
arg = self.request.arguments
if arg.get('s') and arg.get('p'):
self._onQueryStringStatement(arg['s'][-1], arg['p'][-1], self.request.body)
else:
self._onGraphBodyStatements(self.request.body, self.request.headers)
post = put
def _onQueryStringStatement(self, s, p, body):
subj = URIRef(s)
pred = URIRef(p)
turtleLiteral = self.request.body
try:
from rdflib.parser import StringInputSource
from rx.subjects import BehaviorSubject
from twisted.python.filepath import FilePath
from twisted.internet import reactor
from patchablegraph.patchsource import ReconnectingPatchSource
from rdfdb.rdflibpatch import patchQuads
from rdfdb.patch import Patch
log = logging.getLogger('fetch')
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
STATS = scales.collection(
'/web',
scales.PmfStat('combineGraph'),
)
def parseRdf(text: str, contentType: str):
g = Graph()
g.parse(StringInputSource(text), format={
'text/n3': 'n3',
}[contentType])
return g
class RemoteData(object):
def __init__(self, onChange: Callable[[], None]):
"""we won't fire onChange during init"""
self.onChange = onChange
from inference import infer, readRules
from actions import Actions, PutOutputsTable
from inputgraph import InputGraph
from escapeoutputstatements import unquoteOutputStatements
from standardservice.logsetup import log, verboseLogging
from patchablegraph import PatchableGraph, CycloneGraphHandler, CycloneGraphEventsHandler
ROOM = Namespace("http://projects.bigasterisk.com/room/")
DEV = Namespace("http://projects.bigasterisk.com/device/")
NS = {'': ROOM, 'dev': DEV}
STATS = scales.collection(
'/web',
scales.PmfStat('graphChanged'),
scales.PmfStat('updateRules'),
)
def ntStatement(stmt: Tuple[Node, Node, Node]):
def compact(u):
if isinstance(u, URIRef) and u.startswith(ROOM):
return 'room:' + u[len(ROOM):]
return u.n3()
return '%s %s %s .' % (compact(stmt[0]), compact(stmt[1]), compact(
stmt[2]))
class Reasoning(object):
from collector_config import config
#SourceUri = NewType('SourceUri', URIRef) # doesn't work
class SourceUri(URIRef):
pass
ROOM = Namespace("http://projects.bigasterisk.com/room/")
COLLECTOR = SourceUri(URIRef('http://bigasterisk.com/sse_collector/'))
STATS = scales.collection(
'/root',
scales.PmfStat('getState'),
scales.PmfStat('localStatementsPatch'),
scales.PmfStat('makeSyncPatch'),
scales.PmfStat('onPatch'),
scales.PmfStat('sendUpdatePatch'),
scales.PmfStat('replaceSourceStatements'),
)
class LocalStatements(object):
"""
functions that make statements originating from sse_collector itself
"""
def __init__(self, applyPatch: Callable[[URIRef, Patch], None]):
self.applyPatch = applyPatch
self._sourceState: Dict[SourceUri, URIRef] = {} # source: state URIRef
@STATS.localStatementsPatch.time()
