def start(self, restart=False): self._startQueues() if self.clock: self.clock.stop() self.clock = RealClock(self.end) if ResourceManagement.args.pt: from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes performancetest = PerformanceMeterMinutes() for w in self.wrappers: self.startWrapper(w, restart) if ResourceManagement.args.pt: w.addReceiver(performancetest) L.i(datetime.datetime.now()) if not self.args.noQuality: if not self.averageStreamQuality: self.averageStreamQuality = AverageStreamQuality(self, self.clock) else: self.averageStreamQuality.setClock(self.clock) self.clock.runAsync() raw_input("press Enter to end.\n") self.clock.stop()
def deploy(self, f, autostart=False): """ :param f: :param autostart: :return: a tuple with 3 elements. 1. status as string, 2. error message as string, 3. list of uuids of added wrapper """ L.i("Deploying", f) sensordescriptions = [] try: zFile = zipfile.ZipFile(f) if "deploy.json" in zFile.namelist(): deployDescription = JOb(zFile.open("deploy.json", "r")) sys.path.insert(0, f) if deployDescription.isList(): for dd in deployDescription: module = __import__(dd.module) wrapper = getattr(module, dd["class"])() self.addWrapper(wrapper) sensordescriptions.append(wrapper.getSensorDescription()) if autostart: self.startWrapper(wrapper) else: module = __import__(deployDescription.module) wrapper = getattr(module, deployDescription["class"])() self.addWrapper(wrapper) sensordescriptions.append(wrapper.getSensorDescription()) if autostart: self.startWrapper(wrapper) return "OK", "", sensordescriptions except Exception as e: L.w("Deployment of wrapper", f, "failed.", e.message) return "Fail", e.message, []
def registerExchanges(cls): for ex in RabbitMQ.exchanges: try: RabbitMQ.declareExchange(ex, _type="topic") except Exception as e: L.e('Exchange %s could not be declared: %s' % (ex, e.message)) L.e('Exception:', str(e))
def receiveHandler(self, item): parsedData, sensordescription, clock, quality = item L.d2(parsedData) # print parsedData if self.sql: self.sql.insert_observation(sensordescription, parsedData, dictdeepcopy(quality)) if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore: # if len(parsedData.fields)>0: g = self.annotator.annotateObservation(parsedData, sensordescription, clock, quality) del quality if ( ResourceManagement.args.messagebus and not sensordescription.no_publish_messagebus and "fields" in parsedData and len(parsedData.fields) > 0 ): message = g.serialize(format="n3") # print message key = sensordescription.messagebus.routingKey # self.messageBusQueue.add((parsedData.dumps(), self.rabbitmqchannel, RabbitMQ.exchange_data, key)) self.messageBusQueue.add((message, RabbitMQ.exchange_annotated_data, key)) if self.ui.api: self.ui.api.update_observation_cache(str(sensordescription.uuid), message) if ResourceManagement.args.triplestore: # TODO: The following line is commented out, since the Virtuoso makes so much trouble # ThreadedTriplestoreAdapter.getOrMake(sensordescription.graphName).addGraph(g) pass if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore: del g if ResourceManagement.args.aggregate: self.aggregationQueue.add((parsedData, sensordescription)) else: del parsedData
def aggregate(self, data, sensordescription): """ this method is called when stream data has been annotated. :param data: :param sensordescription: :return: The aggregated data """ if not data.fields: return None result = [] try: aggregation_objs = self.aggregation_objs[sensordescription.uuid] for key, agg in aggregation_objs.items(): agg_result = agg.control(data[key]) if agg_result: g, start, end, size = agg_result r = JSONObject() r.graph = g r.field = sensordescription.field[key] r.sensorID = sensordescription.sensorID r.propertyName = sensordescription.field[key].propertyName r.category = sensordescription.sensorType r.aggregationMethod = sensordescription.field[key].aggregationMethod r.aggregationConfiguration = sensordescription.field[key].aggregationConfiguration r.start = start r.end = end r.size = size result.append(r) return result except Exception as e: Log.e("aggregation failed due to Exception", e) return None
def __init__(self, gdi_config, rm): self.rm = rm self.PAGINATION_LIMIT = 100 connect_str = "host='%s' dbname='%s' user='******' password='******' port=%d" % ( gdi_config.host, gdi_config.database, gdi_config.username, gdi_config.password, gdi_config.port) self.conn = psycopg2.connect(connect_str) self.curs = self.conn.cursor() try: self.curs.execute("CREATE SCHEMA IF NOT EXISTS %s;" % SQL.SCHEMA) # self.curs.execute("CREATE TABLE IF NOT EXISTS %s.cp_sensors (sensor_uuid UUID CONSTRAINT uuid_key PRIMARY KEY, sensor_annotation_id VARCHAR, sercvice_category VARCHAR, traffic INTEGER, geom GEOMETRY(GEOMETRY, 4326) );" % (SQL.SCHEMA,)) self.curs.execute("CREATE TABLE IF NOT EXISTS %s.cp_sensors (sensor_uuid UUID CONSTRAINT uuid_key PRIMARY KEY, sensor_annotation_id VARCHAR, sercvice_category VARCHAR, traffic INTEGER, geom GEOMETRY(GEOMETRY, 4326) );" % ("public",)) cols = ["sampling_time TIMESTAMP", "sensor_uuid UUID", "observation_uuid UUID", "data JSON", "quality JSON"] query = 'CREATE TABLE IF NOT EXISTS %s.cp_observations ( %s, PRIMARY KEY (%s), FOREIGN KEY (sensor_uuid) REFERENCES %s.cp_sensors(sensor_uuid));\n' % (SQL.SCHEMA, ', '.join(cols), ", ".join(["observation_uuid"]), "public") self.curs.execute(query) # index over sampling_time and sensor_uuid # since a 'IF NOT EXISTS' is not available for us (version < 9.5) # the error is catched in a separate try-catch try: query = 'CREATE INDEX "timeindex" ON %s.cp_observations USING btree (sampling_time);' % (SQL.SCHEMA,) self.curs.execute(query) query = 'CREATE INDEX uuidindex ON %s.cp_observations USING btree (sensor_uuid);' % (SQL.SCHEMA,) self.curs.execute(query) except: pass # primary secondary observation_uuid map query = 'CREATE TABLE IF NOT EXISTS %s.p_s_observation_uuid (main UUID, secondary UUID);' % (SQL.SCHEMA,) self.curs.execute(query) self.conn.commit() L.i("SQL: schema/tables created") except Exception as e: L.e("SQL: Could not create schema/tables", e) self.conn.rollback()
def setReplayMode(self, mode): super(BrasovIncidentWrapper, self).setReplayMode(mode) # fieldnames of service: "id", "comments", "createdon", "description", "guid", "incidentState", "incidentid", "indsoft_publiclyvisible", "statecode", "ticketnumber", "timestamp", "title", "x", "y" fieldnames = [ "id", "comments", "createdon", "description", "guid", "i", "incidentState", "incidentid", "indsoft_publiclyvisible", "statecode", "ticketnumber", "timestamp", "title", "x", "y", ] try: fobj = AbstractWrapper.getFileObject(__file__, "incidents%d.csv" % self.number, "rU") self.historyreader = CSVHistoryReader(self, fobj, delimiter=",") self.historyreader.multiple_observations = False self.historyparser = CSVParser(self, fieldnames) except Exception as e: Log.e("setReplayMode in Brasov Incident Wrapper", self.number, e) self.historyreader = None
def next(self): url = self.source or self.wrapper.getSensorDescription().source try: return self.load(url) except: Log.e("HttpPullConnection: failed to load", url) return None
def __run(self): while not self.stop: sleep(self.delay) Log.i(self.counter, self.txt) # print self.counter, self.txt # print "ThreadedTriplestoreAdapter Buffer Size:", ThreadedTriplestoreAdapter.getTotalBufferSize() Log.i("ThreadedTriplestoreAdapter Buffer Size:", ThreadedTriplestoreAdapter.getTotalBufferSize()) self.counter = 0
def setReplayMode(self, mode): if mode: try: self.historyreader = CSVHistoryReader(self, AbstractWrapper.getFileObject(__file__, os.path.join("historicdata", "pollution-%s.csv" % self.sensorDescription.sensorID), "rU"), delimiter=';') self.historyparser = CSVParser(self, self.historyreader.headers) except Exception as e: Log.e(e) self.historyreader = None super(InternalBrasovWrapper, self).setReplayMode(mode)
def load(self, url): try: f = urllib2.urlopen(url, timeout=10) r = f.read() f.close() return r except: Log.e(self.__class__.__name__, "error in load") return None
def start(self): if self.replaymode: try: self.historyreader = CSVHistoryReader(self, AbstractWrapper.getFileObject(__file__, os.path.join("historicdata", "weatherAW-%s.csv" % self.sensorDescription.sensorID), "rU"), delimiter=';') self.historyparser = CSVParser(self, self.historyreader.headers) except Exception as e: Log.e(e) self.historyreader = None super(InternalWeatherAWWrapper, self).start()
def saveGraph(self, graph, graphName): serialisation = graph.serialize(destination=None, format='nt', encoding=None) queryString = "".join(["INSERT DATA INTO GRAPH <", self.getGraphURI(graphName), "> {", serialisation, "}"]) sparql = self.getSparqlObject(graphName, queryString) try: sparql.query() except HTTPError as e: L.e("Sparql Endpoint HTTPError in saveGraph:", str(e.code), e.reason) except Exception as e: L.e("Error in saveGraph:", e.message)
def removeSensorStream(self, sensor_uuid): sql = ("DELETE FROM cp_sensors WHERE sensor_uuid='%(sensor_uuid)s'" % {'sensor_uuid': sensor_uuid}) try: self.curs.execute(sql) self.conn.commit() return True except: self.conn.rollback() L.e("Cannot delete Sensor: " + str({'sensor_uuid': sensor_uuid})) L.e("SQL query used:", sql) return False
def removeAllSensorStreams(self): try: sql = "DELETE FROM cp_sensors" self.curs.execute(sql) self.conn.commit() return True except: self.conn.rollback() L.e("Cannot delete all Sensors") L.e("SQL query used", sql) return False
def deleteGraph(self, graphName): queryString = "DEFINE sql:log-enable 3 DROP SILENT GRAPH <" + self.getGraphURI(graphName) + ">" L.d("deleteGraph using query:", queryString) sparql = self.getSparqlObject(graphName, queryString) sparql.setTimeout(300) try: ret = sparql.query() return True except Exception as e: L.e("Error in deleteGraph:", e.message) return False
def createGraph(self, graphName): queryString = "CREATE GRAPH <" + self.getGraphURI(graphName) + ">" sparql = self.getSparqlObject(graphName, queryString) try: ret = sparql.query().convert() return True except HTTPError as e: L.e("Sparql Endpoint HTTPError in createGraph:", str(e.code), e.reason) except Exception as e: L.e("Error in createGraph:", e.message) return False
def saveTriple(self, graphName, subject, predicate, object): sparql = self.getSparqlObject(graphName) # insert into doesn't work with set default graph, have to "... INSERT DATA INTO <graph>..." queryString = "INSERT DATA INTO <" + self.getGraphURI( graphName) + "> { <" + subject + "> <" + predicate + "> <" + object + "> }" sparql.setQuery(queryString) try: sparql.query() except HTTPError as e: L.e("Sparql Endpoint HTTPError in saveTriple:", str(e.code), e.reason) except Exception as e: L.e("Error in saveTriple:", e.message)
def saveMultipleGraphs(self, serialisedGraph, graphName=None): queryString = "".join(["INSERT DATA { GRAPH <", self.getGraphURI(graphName), "> {", serialisedGraph, "}}"]) sparql = self.getSparqlObject(graphName, queryString) sparql.queryType = INSERT try: ret = sparql.query() except EndPointInternalError as e: #transaction deadlock case raise SPARQL_Exception() except EndPointNotFound as e: #temporarily 404 error raise SPARQL_Exception() except Exception as e: L.e("Error in saveMultipleGraphs:", e.message) raise StoreOffline_Exception()
def graphExists(self, graphName): queryString = "ASK { GRAPH <" + self.getGraphURI(graphName) + "> { ?s ?p ?o . }}" sparql = self.getSparqlObject(graphName, queryString) # print queryString try: ret = sparql.query() retList = ret.convert() # print retList return retList["boolean"] except HTTPError as e: L.e("Sparql Endpoint HTTPError in graphExists:", str(e.code), e.reason) except Exception as e: L.e("Error in graphExists:", e.message)
def initialise(self, description, clock): self.reputationSystem = ReputationSystem(description, clock) # self.reputationSystem.addSink(CSVSink()) # self.reputationSystem.addSink(VirtuosoSink(self.messageBusQueue)) # self.reputationSystem.addSink(TerminalSink()) self.reputationSystem.addSink(BlackHoleSink()) self.reputationSystem.addQoIMetric(Frequency()) self.reputationSystem.addQoIMetric(Completeness()) self.reputationSystem.addQoIMetric(Age()) self.reputationSystem.addQoIMetric(Latency()) self.reputationSystem.addQoIMetric(Correctness()) self.initialised = True L.d2("CpQoiSystem initialised")
def getStreamMinMaxDate(self, graphName, sensorName): queryString = """prefix ssn: <http://purl.oclc.org/NET/ssnx/ssn#> prefix tl: <http://purl.org/NET/c4dm/timeline.owl#> prefix sao: <http://purl.oclc.org/NET/UNIS/sao/sao#> prefix ces: <http://www.insight-centre.org/ces#> prefix so: <http://www.daml.org/services/owl-s/1.2/Service.owl#> prefix qoi: <http://purl.oclc.org/NET/UASO/qoi#> prefix prov: <http://www.w3.org/ns/prov#> SELECT MAX(str(?timeValue)) as ?maxDateTime MIN(str(?timeValue)) as ?minDateTime WHERE { ?observation ssn:observationResultTime ?time . ?observation ssn:observedBy <""" + sensorName + """> . ?time tl:at ?timeValue . }""" sparql = self.getSparqlObject(graphName, queryString) try: ret = sparql.query().convert() return ret except Exception as e: L.e("Error in getStreamMinMaxDate:", e.message)
def end(self): for w in self.wrappers: w.stop() if ResourceManagement.args.messagebus: self.messageBusQueue.stop() if ResourceManagement.args.aggregate: self.aggregationQueue.stop() if self.eventWrapper: self.eventWrapper.stop() self.receiverQueue.stop() if ResourceManagement.args.triplestore: ThreadedTriplestoreAdapter.stop() self.stopInterface() L.i("REPLAY ENDED")
def write(self, graphString): if ThreadedTriplestoreAdapter.triplestore: attempts = self.maxAttempts while attempts > 0: try: ThreadedTriplestoreAdapter.triplestore.saveMultipleGraphs(graphString, self.graphName) return except SPARQL_Exception as e: # self.buffer.put(graphString) attempts -= 1 L.e("Error inserting into virtuoso, reattempt") except StoreOffline_Exception as e: L.e( "ThreadedTriplestoreAdapter was unable to save data, store might be offline!") break
def update(self, data): L.d2("ReputationSystem: update called for Stream", self.description.fullSensorID) self.setTimestamp(data) returnValues = {} if data is not None: for m in self.metrics: value = m.update(data) if value: returnValues[value[0]] = value[1] for sink in self.sinkList: sink.update(m) self.addClockJob(True) self.avgQoIManager.calculateAvgQualities(returnValues) return returnValues
def _run(self): self.abort.clear() if not self.clock: raise Exception("no clock set?") while not self.abort.is_set(): if self.event.wait(1.0): if not self.abort.is_set(): try: self.update() except Exception as e: L.e("in _run", e.message) finally: self.clock.continue_running() self.event.clear()
def __registerSensorStream(self, sensor_uuid, sensor_id, sercvice_category, geom, epsg): sql = ( "INSERT INTO cp_sensors(sensor_uuid, sensor_annotation_id, sercvice_category, geom) VALUES('%(sensor_uuid)s' , '%(sensor_id)s' , '%(sercvice_category)s' ,ST_Transform(st_setsrid('%(geom)s'::geometry,%(epsg)s),4326))" % {'sensor_uuid': sensor_uuid, 'sensor_id': sensor_id, 'sercvice_category': sercvice_category, 'geom': geom.wkb_hex, 'epsg': epsg}) try: self.curs.execute(sql) self.conn.commit() return True except: self.conn.rollback() L.e("Cannot insert Sensor:", str( {'sensor_uuid': sensor_uuid, 'sensor_id': sensor_id, 'sercvice_category': sercvice_category, 'geom': geom.wkb_hex, 'epsg': epsg})) L.e("SQL query used:", sql) return False
def _runReplay(self): self.abort.clear() if not self.clock: raise Exception("no clock set?") while not self.abort.is_set(): if self.event.wait(1.0): if not self.abort.is_set(): try: self.__forEachWrapper("update") except Exception as e: L.e("in _run", e.message) finally: self.clock.continue_running() pass self.event.clear()
def aggregate(self, data, sensordescription): result = [] try: dftobjs = self.dftobjects[sensordescription.uuid] for f in dftobjs: g = dftobjs[f].control(data[f]) if g: r = JSONObject() r.graph = g r.sensorID = sensordescription.sensorID r.propertyType = sensordescription.field[f].propertyName r.category = sensordescription.sensorType result.append(r) return result except KeyError: Log.e("Dft aggregation failed") return None
def addWrapper(self, wrapper): # TODO: this should not be here if ResourceManagement.args.cleartriplestore: self.deleteGraphs(wrapper) sd = wrapper.getSensorDescription() try: if isinstance(sd, list): for _sd in sd: try: _sd.test() if ResourceManagement.args.aggregate: self.aggregator.wrapper_added(_sd) if self.gdiInterface: self.gdiInterface.registerSensorStreamFromWKT( _sd.uuid, _sd.sensorID, _sd.sensorType, _sd.location, _sd.location_epsg or 4326 ) # if self.sql: # self.sql.create_table(_sd) L.i("added wrapper with ID", _sd.sensorID) except Exception as ex: L.e("Error deploying wrapper:", str(ex)) else: try: sd.test() if ResourceManagement.args.aggregate: self.aggregator.wrapper_added(sd) if self.gdiInterface: self.gdiInterface.registerSensorStreamFromWKT( sd.uuid, sd.sensorID, sd.sensorType, sd.location, sd.location_epsg or 4326 ) # if self.sql: # self.sql.create_table(sd) L.i("added wrapper with ID", sd.sensorID) except Exception as ex: L.e("Error deploying wrapper:", str(ex)) if ResourceManagement.args.triplestore or ResourceManagement.args.messagebus: # StaticAnnotator.staticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.rabbitmqchannel) StaticAnnotator.threadedStaticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.ui.api) if ResourceManagement.args.messagebus: wrapper.setMessageBusQueue(self.messageBusQueue) self.wrappers.append(wrapper) except Exception as ex: L.e(self.__class__.__name__, "Error in addWrapper:", str(ex))