def __init__(self, uri: URIRef, config: Graph, masterGraph: PatchableGraph,
mqtt, internalMqtt, debugPageData):
self.uri = uri
self.config = config
self.masterGraph = masterGraph
self.debugPageData = debugPageData
self.mqtt = mqtt # deprecated
self.internalMqtt = internalMqtt
self.mqttTopic = self.topicFromConfig(self.config)
log.debug(f'new mqttTopic {self.mqttTopic}')
self.debugSub = {
'topic': self.mqttTopic.decode('ascii'),
'recentMessages': [],
'recentParsed': [],
'recentConversions': [],
'currentMetrics': [],
'currentOutputGraph': {
't': 1,
'n3': "(n3)"
},
}
self.debugPageData['subscribed'].append(self.debugSub)
rawBytes: Observable = self.subscribeMqtt(self.mqttTopic)
# rawBytes = rx.operators.do_action(self.countIncomingMessage)(rawBytes)
rawBytes.subscribe(on_next=self.countIncomingMessage)
def poll(self):
ret = None
startTime = time.time()
try:
url = (f'http://{deviceIp}/cgi-bin/cgi_manager').encode('ascii')
resp = yield fetch(
url,
method=b'POST',
headers={b'Authorization': [b'Basic %s' % auth]},
postdata=(f'''<LocalCommand>
<Name>get_usage_data</Name>
<MacId>0x{macId}</MacId>
</LocalCommand>
<LocalCommand>
<Name>get_price_blocks</Name>
<MacId>0x{macId}</MacId>
</LocalCommand>''').encode('ascii'),
timeout=10)
ret = json.loads(resp.body)
log.debug(ret)
if ret['demand_units'] != 'kW':
raise ValueError
if ret['summation_units'] != 'kWh':
raise ValueError
pts = [
dict(measurement='housePowerW',
fields=dict(value=float(ret['demand']) * 1000),
tags=dict(house='berkeley'),
time=int(startTime))]
sd = float(ret['summation_delivered'])
if sd > 0: # Sometimes nan
pts.append(dict(measurement='housePowerSumDeliveredKwh',
fields=dict(value=float()),
tags=dict(house='berkeley'),
time=int(startTime)))
if 'price' in ret:
pts.append(dict(
measurement='price',
fields=dict(price=float(ret['price']),
price_units=float(ret['price_units'])),
tags=dict(house='berkeley'),
time=int(startTime),
))
self.influx.write_points(pts, time_precision='s')
self.graph.patchObject(context=ROOM['powerEagle'],
subject=ROOM['housePower'],
predicate=ROOM['instantDemandWatts'],
newObject=Literal(float(ret['demand']) * 1000))
except Exception as e:
traceback.print_exc()
log.error("failed: %r", e)
log.error(repr(ret))
os.abort()
now = time.time()
goal = startTime + periodSec - .2
reactor.callLater(max(1, goal - now), self.poll)
def updateQuads(self, newGraphs):
newQuads = set.union(*newGraphs)
g = graphFromQuads(newQuads)
log.debug(f'{self.uri} update to {len(newQuads)} statements')
self.influx.exportToInflux(newQuads)
self.masterGraph.patchSubgraph(self.uri, g)
def _publish(self,
topic: str,
messageJson: object = None,
message: str = None):
log.debug(f'mqtt.publish {topic} {message} {messageJson}')
if messageJson is not None:
message = json.dumps(messageJson)
self.settings.mqtt.publish(topic.encode('ascii'),
message.encode('ascii'))
def poll(self):
ret = None
startTime = time.time()
try:
url = (f'http://{deviceIp}/cgi-bin/cgi_manager').encode('ascii')
resp = yield fetch(
url,
method=b'POST',
headers={b'Authorization': [b'Basic %s' % auth]},
postdata=(f'''<LocalCommand>
<Name>get_usage_data</Name>
<MacId>0x{macId}</MacId>
</LocalCommand>
<LocalCommand>
<Name>get_price_blocks</Name>
<MacId>0x{macId}</MacId>
</LocalCommand>''').encode('ascii'),
timeout=10)
ret = json.loads(resp.body)
log.debug(f"response body {ret}")
if ret['demand_units'] != 'kW':
raise ValueError
if ret['summation_units'] != 'kWh':
raise ValueError
demandW = float(ret['demand']) * 1000
self.out['w'].set(demandW)
sd = float(ret['summation_delivered'])
if sd > 0: # Sometimes nan
self.out['kwh'].set(sd)
if 'price' in ret:
self.out['price'].set(float(ret['price']))
self.graph.patchObject(context=ROOM['powerEagle'],
subject=ROOM['housePower'],
predicate=ROOM['instantDemandWatts'],
newObject=Literal(demandW))
POLL_SUCCESSES.inc()
except Exception as e:
POLL_ERRORS.inc()
traceback.print_exc()
log.error("failed: %r", e)
log.error(repr(ret))
now = time.time()
goal = startTime + periodSec - .2
reactor.callLater(max(1, goal - now), self.poll)
def replaceSourceStatements(self, source: SourceUri,
stmts: Sequence[Statement]):
log.debug('replaceSourceStatements with %s stmts', len(stmts))
newStmts = set(stmts)
with self.postDeleteStatements() as garbage:
for stmt, (sources, handlers) in self.table.items():
if source in sources:
if stmt not in stmts:
sources.remove(source)
if not sources and not handlers:
garbage.add(stmt)
else:
if stmt in stmts:
sources.add(source)
newStmts.discard(stmt)
self.applySourcePatch(source, Patch(addQuads=newStmts, delQuads=[]))
def addSseHandler(self, handler: PatchSink):
log.info('addSseHandler %r %r', handler, handler.streamId)
# fail early if id doesn't match
sources = self._sourcesForHandler(handler)
self.handlers.add(handler)
for source in sources:
if source not in self.clients and source != COLLECTOR:
log.debug('connect to patch source %s', source)
self._localStatements.setSourceState(source, ROOM['connect'])
self.clients[source] = ReconnectingPatchSource(
source,
listener=lambda p, fullGraph, source=source: self._onPatch(
source, p, fullGraph),
reconnectSecs=10)
log.debug('bring new client up to date')
self._sendUpdatePatch(handler)
def updateQuads(self, newGraphs):
newQuads = set.union(*newGraphs)
g = graphFromQuads(newQuads)
log.debug(f'{self.uri} update to {len(newQuads)} statements')
for quad in newQuads:
meas = quad[0].split('/')[-1]
if meas.startswith('airQuality'):
where_prefix, type_ = meas[len('airQuality'):].split('door')
where = where_prefix + 'door'
metric = 'air'
tags = {'loc': where.lower(), 'type': type_.lower()}
val = quad[2].toPython()
if metric not in collectors:
collectors[metric] = Gauge(metric,
'measurement',
labelnames=tags.keys())
collectors[metric].labels(**tags).set(val)
self.masterGraph.patchSubgraph(self.uri, g)
def _sendUpdatePatch(self, handler: Optional[PatchSink] = None):
"""
send a patch event out this handler to bring it up to date with
self.statements
"""
now = time.time()
selected = self.handlers
if handler is not None:
if handler not in self.handlers:
log.error("called _sendUpdatePatch on a handler that's gone")
return
selected = {handler}
# reduce loops here- prepare all patches at once
for h in selected:
period = .9
if 'Raspbian' in h.request.headers.get('user-agent', ''):
period = 5
if h.lastPatchSentTime > now - period:
continue
p = self.statements.makeSyncPatch(h,
set(self._sourcesForHandler(h)))
log.debug('makeSyncPatch for %r: %r', h, p.jsonRepr)
if not p.isNoop():
log.debug("send patch %s to %s", p.shortSummary(), h)
# This can be a giant line, which was a problem
# once. Might be nice for this service to try to break
# it up into multiple sends, although there's no
# guarantee at all since any single stmt could be any
# length.
h.sendEvent(message=jsonFromPatch(p).encode('utf8'),
event=b'patch')
h.lastPatchSentTime = now
else:
log.debug('nothing to send to %s', h)
self.loop.stop()
if __name__ == '__main__':
arg = docopt("""
Usage: mqtt_to_rdf.py [options]
-v Verbose
--cs=STR Only process config filenames with this substring
""")
verboseLogging(arg['-v'])
config = Graph()
for fn in Path('.').glob('conf/*.n3'):
if not arg['--cs'] or str(arg['--cs']) in str(fn):
log.debug(f'loading {fn}')
config.parse(str(fn), format='n3')
else:
log.debug(f'skipping {fn}')
masterGraph = PatchableGraph()
brokerHost = 'mosquitto-frontdoor.default.svc.cluster.local'
brokerPort = 10210
debugPageData = {
# schema in index.ts
'server': f'{brokerHost}:{brokerPort}',
'messagesSeen': 0,
'subscribed': [],
}