def setSourceState(self, source: SourceUri, state: URIRef):
"""
add a patch to the COLLECTOR graph about the state of this
source. state=None to remove the source.
"""
oldState = self._sourceState.get(source, None)
if state == oldState:
return
log.info('source state %s -> %s', source, state)
if oldState is None:
self._sourceState[source] = state
self.applyPatch(
COLLECTOR,
Patch(addQuads=[
(COLLECTOR, ROOM['source'], source, COLLECTOR),
(source, ROOM['state'], state, COLLECTOR),
]))
elif state is None:
del self._sourceState[source]
self.applyPatch(
COLLECTOR,
Patch(delQuads=[
(COLLECTOR, ROOM['source'], source, COLLECTOR),
(source, ROOM['state'], oldState, COLLECTOR),
]))
else:
self._sourceState[source] = state
self.applyPatch(
COLLECTOR,
Patch(addQuads=[
(source, ROOM['state'], state, COLLECTOR),
],
delQuads=[
(source, ROOM['state'], oldState, COLLECTOR),
]))
def onMessage(self, payload, isBinary):
msg = json.loads(payload)
if 'connectedAs' in msg:
self.connectionId = msg['connectedAs']
log.info(f'rdfdb calls us {self.connectionId}')
elif 'patch' in msg:
p = Patch(jsonRepr=payload.decode('utf8'))
log.debug("received patch %s", p.shortSummary())
self.sg.onPatchFromDb(p)
else:
log.warn('unknown msg from websocket: %s...', payload[:32])
def messageReceived(self, message: bytes):
if message == b'PING':
self.sendMessage('PONG')
return
log.debug("got message from %r: %s", self, message[:32])
p = Patch(jsonRepr=message.decode('utf8'))
self.settings.db.patch(p, sender=self.connectionId)
def patch(self,
patch: Patch,
sender: Optional[str] = None,
dueToFileChange: bool = False) -> None:
"""
apply this patch to the master graph then notify everyone about it
dueToFileChange if this is a patch describing an edit we read
*from* the file (such that we shouldn't write it back to the file)
"""
ctx = patch.getContext()
log.info("patching graph %s -%d +%d" %
(ctx, len(patch.delQuads), len(patch.addQuads)))
if hasattr(self, 'watchedFiles'): # todo: eliminate this
self.watchedFiles.aboutToPatch(ctx)
# an error here needs to drop the sender, and reset everyone
# else if we can't rollback the failing patch.
patchQuads(self.graph, patch.delQuads, patch.addQuads, perfect=True)
stats.graphLen = len(self.graph)
self._syncPatchToOtherClients(patch, sender)
if not dueToFileChange:
self.watchedFiles.dirtyFiles([ctx])
graphStats.statements = len(self.graph)
def patch(self, p: Patch) -> None:
"""send this patch to the server and apply it to our local
graph and run handlers"""
if not self.isConnected or self.currentClient is None:
log.warn("not currently connected- dropping patch")
return
if p.isNoop():
log.info("skipping no-op patch")
return
# these could fail if we're out of sync. One approach:
# Rerequest the full state from the server, try the patch
# again after that, then give up.
debugKey = '[id=%s]' % (id(p) % 1000)
log.debug("\napply local patch %s %s", debugKey, p)
try:
self._applyPatchLocally(p)
except ValueError as e:
log.error(e)
self.resync()
return
log.debug('runDepsOnNewPatch')
self.runDepsOnNewPatch(p)
log.debug('sendPatch')
self.currentClient.sendPatch(p)
log.debug('patch is done %s', debugKey)
def startCardRead(self, cardUri, text):
self.masterGraph.patch(
Patch(addQuads=[(sensor, ROOM['reading'], cardUri, ctx),
(cardUri, ROOM['cardText'], text, ctx)],
delQuads=[]))
log.info('%s :cardText %s .', cardUri.n3(), text.n3())
self._sendOneshot([(sensor, ROOM['startReading'], cardUri),
(cardUri, ROOM['cardText'], text)])
def fileGone(self) -> None:
"""
our file is gone; remove the statements from that context
"""
myQuads = [(s, p, o, self.uri) for s, p, o in self.getSubgraph(self.uri)
]
log.debug("dropping all statements from context %s", self.uri)
if myQuads:
self.patch(Patch(delQuads=myQuads), dueToFileChange=True)
def _onStatements(self, stmts):
g = self.settings.masterGraph
for s, p, o in stmts:
patch = g.getObjectPatch(CTX, s, p, o)
if o == ROOM['unset']:
patch = Patch(delQuads=patch.delQuads)
g.patch(patch)
nquads = g.serialize(None, format='nquads')
self.settings.dbFile.setContent(nquads)
def endCardRead(self, cardUri):
log.debug(f'{cardUri} has been gone for {self.expireSecs} sec')
delQuads = []
for spo in self.masterGraph._graph.triples(
(sensor, ROOM['reading'], cardUri)):
delQuads.append(spo + (ctx, ))
for spo in self.masterGraph._graph.triples(
(cardUri, ROOM['cardText'], None)):
delQuads.append(spo + (ctx, ))
self.masterGraph.patch(Patch(addQuads=[], delQuads=delQuads))
def _onFullGraph(self, message):
try:
g = ConjunctiveGraph()
g.parse(StringInputSource(message), format='json-ld')
p = Patch(addGraph=g)
self._sendPatch(p, fullGraph=True)
except Exception:
log.error(traceback.format_exc())
raise
self._fullGraphReceived = True
self._fullGraphTime = time.time()
self._patchesReceived += 1
def sendPatch(self, p: Patch):
# this is where we could concatenate little patches into a
# bigger one. Often, many statements will cancel each
# other out.
# also who's going to accumulate patches when server is down,
# or is that not allowed?
if self.connectionId is None:
raise ValueError("can't send patches before we get an id")
body = p.makeJsonRepr()
log.debug(f'connectionId={self.connectionId} sending patch {len(body)} bytes')
self.sendMessage(body.encode('utf8'))
def removeMappingNode(self, context, node):
"""
removes the statements with this node as subject or object, which
is the right amount of statements to remove a node that
patchMapping made.
"""
p = Patch(delQuads=[
spo + (context, ) for spo in chain(
self._graph.triples((None, None, node), context=context),
self._graph.triples((node, None, None), context=context))
])
self.patch(p)
def patchSubgraph(self, context, newGraph):
"""
replace all statements in 'context' with the quads in newGraph.
This is not cooperating with currentState.
"""
old = set(
quadsWithContextUris(self._graph.quads(
(None, None, None, context))))
new = set(quadsWithContextUris(newGraph))
p = Patch(delQuads=old - new, addQuads=new - old)
self.patch(p)
return p # for debugging
def _updateMasterWithNewPollStatements(self, dev, new):
prev = self._statementsFromInputs.get(dev, set())
# it's important that quads from different devices
# don't clash, since that can lead to inconsistent
# patches (e.g.
# dev1 changes value from 1 to 2;
# dev2 changes value from 2 to 3;
# dev1 changes from 2 to 4 but this patch will
# fail since the '2' statement is gone)
self.masterGraph.patch(Patch.fromDiff(inContext(prev, dev),
inContext(new, dev)))
self._statementsFromInputs[dev] = new
def _updateMasterWithNewPollStatements(self, dev, new):
prev = self._statementsFromInputs.get(dev, set())
# it's important that quads from different devices
# don't clash, since that can lead to inconsistent
# patches (e.g.
# dev1 changes value from 1 to 2;
# dev2 changes value from 2 to 3;
# dev1 changes from 2 to 4 but this patch will
# fail since the '2' statement is gone)
self.masterGraph.patch(Patch.fromDiff(inContext(prev, dev),
inContext(new, dev)))
self._statementsFromInputs[dev] = new
def getObjectPatch(self, context, subject, predicate, newObject):
"""send a patch which removes existing values for (s,p,*,c)
and adds (s,p,newObject,c). Values in other graphs are not affected.
newObject can be None, which will remove all (subj,pred,*) statements.
"""
existing = []
for spoc in quadsWithContextUris(
self._graph.quads((subject, predicate, None, context))):
existing.append(spoc)
toAdd = ([(subject, predicate, newObject,
context)] if newObject is not None else [])
return Patch(delQuads=existing, addQuads=toAdd).simplify()
def update(masterGraph, eventsInGraph, coll):
eventsInGraph = set()
recentEvents = set()
fetched_nonboring_docs = 0
for doc in coll.find({}, sort=[('t', -1)], limit=1000):
uri = uriFromMongoEvent(doc['_id'])
recentEvents.add(uri)
if uri in eventsInGraph:
fetched_nonboring_docs += 1
else:
try:
masterGraph.patch(Patch(addQuads=quadsForEvent(doc)))
eventsInGraph.add(uri)
fetched_nonboring_docs += 1
except Boring:
pass
if fetched_nonboring_docs > 100:
break
for uri in eventsInGraph.difference(recentEvents):
oldStatements = list(masterGraph.quads((uri, None, None, None)))
masterGraph.patch(Patch(delQuads=oldStatements))
eventsInGraph.remove(uri)
def patchMapping(self, context, subject, predicate, nodeClass, keyPred,
valuePred, newKey, newValue):
"""
creates/updates a structure like this:
?subject ?predicate [
a ?nodeClass;
?keyPred ?newKey;
?valuePred ?newValue ] .
There should be a complementary readMapping that gets you a
value since that's tricky too
"""
# as long as currentState is expensive and has the
# tripleFilter optimization, this looks like a mess. If
# currentState became cheap, a lot of code here could go away.
with self.currentState(tripleFilter=(subject, predicate,
None)) as current:
adds = set([])
for setting in current.objects(subject, predicate):
with self.currentState(tripleFilter=(setting, keyPred,
None)) as current2:
match = current2.value(setting, keyPred) == newKey
if match:
break
else:
setting = URIRef(subject +
"/map/%s" % random.randrange(999999999))
adds.update([
(subject, predicate, setting, context),
(setting, RDF.type, nodeClass, context),
(setting, keyPred, newKey, context),
])
with self.currentState(tripleFilter=(setting, valuePred,
None)) as current:
dels = set([])
for prev in current.objects(setting, valuePred):
dels.add((setting, valuePred, prev, context))
adds.add((setting, valuePred, newValue, context))
if adds != dels:
self.patch(Patch(delQuads=dels, addQuads=adds))
def replaceSourceStatements(self, source: SourceUri,
stmts: Sequence[Statement]):
log.debug('replaceSourceStatements with %s stmts', len(stmts))
newStmts = set(stmts)
with self.postDeleteStatements() as garbage:
for stmt, (sources, handlers) in self.table.items():
if source in sources:
if stmt not in stmts:
sources.remove(source)
if not sources and not handlers:
garbage.add(stmt)
else:
if stmt in stmts:
sources.add(source)
newStmts.discard(stmt)
self.applySourcePatch(source, Patch(addQuads=newStmts, delQuads=[]))
def __init__(self, graph, ip, key):
self.graph = graph
self.ip, self.key = ip, key
self.api = APIFactory(ip, psk=key)
self.gateway = Gateway()
devices_command = self.gateway.get_devices()
self.devices_commands = self.api.request(devices_command)
self.devices = self.api.request(self.devices_commands)
self.ctx = ROOM['tradfriHub']
self.graph.patch(Patch(
addQuads=[(s,p,o,self.ctx) for s,p,o in self.deviceStatements()]))
self.curStmts = []
task.LoopingCall(self.updateCur).start(60)
for dev in self.devices:
self.startObserve(dev)
def reread(self) -> None:
"""update the graph with any diffs from this file
n3 parser fails on "1.e+0" even though rdflib was emitting that itself
"""
old = self.getSubgraph(self.uri)
new = Graph()
try:
contents = open(self.path).read()
if contents.startswith("#new"):
log.debug("%s ignoring empty contents of my new file",
self.path)
# this is a new file we're starting, and we should not
# patch our graph as if it had just been cleared. We
# shouldn't even be here reading this, but
# lastWriteTimestamp didn't work.
return
new.parse(location=self.path, format='n3')
self.readPrefixes = dict(new.namespaces())
except SyntaxError as e:
print(e)
traceback.print_exc()
log.error("%s syntax error", self.path)
# todo: likely bug- if a file has this error upon first
# read, I think we don't retry it right.
return
except IOError as e:
log.error("%s rereading %s: %r", self.path, self.uri, e)
return
old = inContext(old, self.uri)
new = inContext(new, self.uri)
p = Patch.fromDiff(old, new)
if p:
log.debug("%s applying patch for changes in file", self.path)
self.patch(p, dueToFileChange=True)
else:
log.debug("old == new after reread of %s", self.path)
def makeSyncPatch(self, handler: PatchSink, sources: Set[SourceUri]):
# todo: this could run all handlers at once, which is how we
# use it anyway
adds = []
dels = []
with self.postDeleteStatements() as garbage:
for stmt, (stmtSources, handlers) in self.table.items():
belongsInHandler = not sources.isdisjoint(stmtSources)
handlerHasIt = handler in handlers
# log.debug("%s belong=%s has=%s",
# abbrevStmt(stmt), belongsInHandler, handlerHasIt)
if belongsInHandler and not handlerHasIt:
adds.append(stmt)
handlers.add(handler)
elif not belongsInHandler and handlerHasIt:
dels.append(stmt)
handlers.remove(handler)
if not handlers and not stmtSources:
garbage.add(stmt)
return Patch(addQuads=adds, delQuads=dels)
def addClient(self, newClient: WebsocketClient) -> None:
log.info("new connection: sending all graphs to %r" % newClient)
newClient.sendPatch(
Patch(addQuads=self.graph.quads(ALLSTMTS), delQuads=[]))
self.clients.append(newClient)
stats.clients = len(self.clients)
def sendPatch(self, p: Patch):
self.sendMessage(p.makeJsonRepr())
def updateCur(self, dev=None):
cur = [(s,p,o,self.ctx) for s,p,o in
self.currentStateStatements([dev] if dev else self.devices)]
self.graph.patch(Patch(addQuads=cur, delQuads=self.curStmts))
self.curStmts = cur
def lostRdfdbConnection(self) -> None:
self.isConnected = False
self.patch(Patch(delQuads=self._graph.quads()))
log.info(f'cleared graph to {len(self._graph)}')
log.error('graph is not updating- you need to restart')
self.connectSocket()