def __init__(self, args): L(args.log) self.config = JOb(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb")) self.status = JSONObject() self.status.running = False self.wrappers = [] ResourceManagement.args = args ResourceManagement.config = self.config self.clock = None self.annotator = GenericAnnotation() self.stoppedClock = False # init triplestore if args.triplestore: ThreadedTriplestoreAdapter.triplestore = TripleStoreFactory.getTripleStore( self.config.triplestore.driver, self.config.triplestore ) # dump the graph with the static annotations StaticAnnotator.dumpGraph() if args.aggregate: # self.aggregator = AggregatorFactory.make(self.config.aggregationmethod, self.config.aggregation_configuration) # self.aggregator = AggregatorFactory.make() # TODO self.aggregator = GenericAggregator() self.aggregationQueue = QueueThread(handler=self.aggregateHandler) self.receiverQueue = QueueThread(handler=self.receiveHandler) # establish connection to the message bus if args.messagebus: threading.Thread(name="messageBusConnector", target=self.start_messagebus, args=(args,)).start() else: self.messageBusQueue = None if args.eventannotation: self.eventWrapper = GenericEventWrapper(self.messageBusQueue) else: self.eventWrapper = None if args.gdi: from CityPulseGdi.eu.citypulse.uaso.gdi.CityPulseGDInterface import CityPulseGDInterface self.gdiInterface = CityPulseGDInterface(self.config.gdi_db) # self.gdiInterface.removeAllSensorStreams() else: self.gdiInterface = None self.averageStreamQuality = None if args.sql: self.sql = SQL(self.config.gdi_db, self) else: self.sql = None self.startInterface() self.autodeploy()
def __init__(self): self.config = JSONObject(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb")) self.host = self.config.rabbitmq.host self.port = self.config.rabbitmq.port # self.rabbitmqconnection, self.rabbitmqchannel = RabbitMQ.establishConnection(self.host, self.port) self.messageBusReceiveQueue = QueueThread(handler=self.receiveEventHandler) self.messageBusSendQueue = QueueThread(handler=self.sendAnnotatedEventHandler) self.wrappers = [] self.splitters = None self.annotator = GenericAnnotation() self.exchange = RabbitMQ.exchange_annotated_event
class ResourceManagement(virtualisation.wrapper.wrapperoutputreceiver.AbstractReceiver): deployfoldername = os.path.join(os.path.dirname(__file__), "..", "autodeploy") eventdescriptionfoldername = os.path.join(os.path.dirname(__file__), "..", "eventdescriptions") args = None config = None def __init__(self, args): L(args.log) self.config = JOb(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb")) self.status = JSONObject() self.status.running = False self.wrappers = [] ResourceManagement.args = args ResourceManagement.config = self.config self.clock = None self.annotator = GenericAnnotation() self.stoppedClock = False # init triplestore if args.triplestore: ThreadedTriplestoreAdapter.triplestore = TripleStoreFactory.getTripleStore( self.config.triplestore.driver, self.config.triplestore ) # dump the graph with the static annotations StaticAnnotator.dumpGraph() if args.aggregate: # self.aggregator = AggregatorFactory.make(self.config.aggregationmethod, self.config.aggregation_configuration) # self.aggregator = AggregatorFactory.make() # TODO self.aggregator = GenericAggregator() self.aggregationQueue = QueueThread(handler=self.aggregateHandler) self.receiverQueue = QueueThread(handler=self.receiveHandler) # establish connection to the message bus if args.messagebus: threading.Thread(name="messageBusConnector", target=self.start_messagebus, args=(args,)).start() else: self.messageBusQueue = None if args.eventannotation: self.eventWrapper = GenericEventWrapper(self.messageBusQueue) else: self.eventWrapper = None if args.gdi: from CityPulseGdi.eu.citypulse.uaso.gdi.CityPulseGDInterface import CityPulseGDInterface self.gdiInterface = CityPulseGDInterface(self.config.gdi_db) # self.gdiInterface.removeAllSensorStreams() else: self.gdiInterface = None self.averageStreamQuality = None if args.sql: self.sql = SQL(self.config.gdi_db, self) else: self.sql = None self.startInterface() self.autodeploy() def _startQueues(self): if not self.receiverQueue.is_alive(): self.receiverQueue.start() # since we need to send messages when a new wrapper is added, the messageBusQueue needs to be started in __init__ # if self.args.messagebus: # self.messageBusQueue.start() if ResourceManagement.args.aggregate: if not self.aggregationQueue.is_alive(): self.aggregationQueue.start() if self.eventWrapper: if not self.eventWrapper.isStarted(): self.eventWrapper.start() def start_messagebus(self, args): L.i("Connecting to the message bus") self.messageBusQueue = QueueThread(handler=self.sendMessageHandler) try: # prepare RabbitMQ configuration rmq_host = str(self.config.rabbitmq.host) rmq_port = self.config.rabbitmq.port rmq_username = self.config.rabbitmq.username if "username" in self.config.rabbitmq else None rmq_password = self.config.rabbitmq.username if "password" in self.config.rabbitmq else None if rmq_username: if rmq_password: RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username, rmq_password) else: RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username) else: RabbitMQ.establishConnection(rmq_host, rmq_port) L.i("Connected to the message bus") self.messageBusQueue.start() except MessageBusConnectionError: self.args.messagebus = False args.messagebus = False L.w("Could not connect to MessageBus server. Disabling MessageBus feature.") def __all_sensordescriptions(self): sds = [] for w in self.wrappers: if not isinstance(w.getSensorDescription(), list): sds += [w.getSensorDescription()] else: sds += w.getSensorDescription() return sds def autodeploy(self): deployFolder = os.path.join(ResourceManagement.deployfoldername, "*.zip") files = glob.glob(deployFolder) for f in files: self.deploy(f) self.status.running = True def deploy(self, f, autostart=False): """ :param f: :param autostart: :return: a tuple with 3 elements. 1. status as string, 2. error message as string, 3. list of uuids of added wrapper """ L.i("Deploying", f) sensordescriptions = [] try: zFile = zipfile.ZipFile(f) if "deploy.json" in zFile.namelist(): deployDescription = JOb(zFile.open("deploy.json", "r")) sys.path.insert(0, f) if deployDescription.isList(): for dd in deployDescription: module = __import__(dd.module) wrapper = getattr(module, dd["class"])() self.addWrapper(wrapper) sensordescriptions.append(wrapper.getSensorDescription()) if autostart: self.startWrapper(wrapper) else: module = __import__(deployDescription.module) wrapper = getattr(module, deployDescription["class"])() self.addWrapper(wrapper) sensordescriptions.append(wrapper.getSensorDescription()) if autostart: self.startWrapper(wrapper) return "OK", "", sensordescriptions except Exception as e: L.w("Deployment of wrapper", f, "failed.", e.message) return "Fail", e.message, [] def registerEvent(self, eventDescriptionFile): if self.eventWrapper: eventDescription = EventDescription(open(eventDescriptionFile, "rb")) if not eventDescription.test(): return "Fail", "Invalid EventDescription" self.eventWrapper.registerEvent(eventDescription) return "Ok", "" else: return "Fail", "Event wrapper disabled" def startInterface(self): self.ui = Ui(self) self.ui.api = Api(self) self.ui.stat_api = Stat_Api(self) threading.Thread(target=cherrypy.quickstart, args=(self.ui, "/", self.config.interface.raw())).start() def stopInterface(self): cherrypy.engine.exit() def test(self): # from wrapper_dev.osna_dummy.osna_dummy_wrapper import OsnaDummyWrapper # wrapper = OsnaDummyWrapper() # self.addWrapper(wrapper) # from wrapper_dev.london_traffic.londontrafficwrapper import LondonTrafficWrapper # wrapper = LondonTrafficWrapper() # self.addWrapper(wrapper) # from wrapper_dev.aarhus_traffic.aarhustrafficwrapper import AarhusTrafficWrapper # wrapper = AarhusTrafficWrapper() # self.addWrapper(wrapper) # from wrapper_dev.aarhus_parking.aarhusparkingwrapper import AarhusParkingWrapper # wrapper = AarhusParkingWrapper() # self.addWrapper(wrapper) # from wrapper_dev.brasov_pollution.brasovpollutionwrapper import BrasovPollutionWrapper # wrapper = BrasovPollutionWrapper() # self.addWrapper(wrapper) # from wrapper_dev.romania_weather.romaniaweather_aw import RomanianWeatherAWWrapper # wrapper = RomanianWeatherAWWrapper() # self.addWrapper(wrapper) # from wrapper_dev.romania_weather.romaniaweather_mr import RomanianWeatherMRWrapper # wrapper = RomanianWeatherMRWrapper() # self.addWrapper(wrapper) # from wrapper_dev.brasov_incidents.brasov_incidents import BrasovIncidentWrapper0, BrasovIncidentWrapper1 # wrapper = BrasovIncidentWrapper0() # self.addWrapper(wrapper) # wrapper = BrasovIncidentWrapper1() # self.addWrapper(wrapper) # from wrapper_dev.aarhus_pollution.aarhuspollutionwrapper import ComposedAarhusPollutionWrapper # wrapper = ComposedAarhusPollutionWrapper() # self.addWrapper(wrapper) # from wrapper_dev.twitter_stream.twitter import AarhusTwitterWrapper # wrapper = AarhusTwitterWrapper() # self.addWrapper(wrapper) from wrapper_dev.WrapperBusStations.brasovbusstationwrapper import BrasovBusStationWrapper wrapper = BrasovBusStationWrapper() self.addWrapper(wrapper) if self.args.eventannotation: from wrapper_dev.aarhus_traffic.aarhustrafficeventwrapper import AarhusTrafficEventWrapper atew = AarhusTrafficEventWrapper() self.eventWrapper.registerEvent(atew.getEventDescription()) def addWrapper(self, wrapper): # TODO: this should not be here if ResourceManagement.args.cleartriplestore: self.deleteGraphs(wrapper) sd = wrapper.getSensorDescription() try: if isinstance(sd, list): for _sd in sd: try: _sd.test() if ResourceManagement.args.aggregate: self.aggregator.wrapper_added(_sd) if self.gdiInterface: self.gdiInterface.registerSensorStreamFromWKT( _sd.uuid, _sd.sensorID, _sd.sensorType, _sd.location, _sd.location_epsg or 4326 ) # if self.sql: # self.sql.create_table(_sd) L.i("added wrapper with ID", _sd.sensorID) except Exception as ex: L.e("Error deploying wrapper:", str(ex)) else: try: sd.test() if ResourceManagement.args.aggregate: self.aggregator.wrapper_added(sd) if self.gdiInterface: self.gdiInterface.registerSensorStreamFromWKT( sd.uuid, sd.sensorID, sd.sensorType, sd.location, sd.location_epsg or 4326 ) # if self.sql: # self.sql.create_table(sd) L.i("added wrapper with ID", sd.sensorID) except Exception as ex: L.e("Error deploying wrapper:", str(ex)) if ResourceManagement.args.triplestore or ResourceManagement.args.messagebus: # StaticAnnotator.staticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.rabbitmqchannel) StaticAnnotator.threadedStaticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.ui.api) if ResourceManagement.args.messagebus: wrapper.setMessageBusQueue(self.messageBusQueue) self.wrappers.append(wrapper) except Exception as ex: L.e(self.__class__.__name__, "Error in addWrapper:", str(ex)) def removeWrapper(self, uuid): """ Removes a wrapper identified by its UUID :param uuid: :return: True if successful, otherwise False """ for i in range(0, len(self.wrappers)): sd = self.wrappers[i].getSensorDescription() if isinstance(sd, list): for j in range(0, len(sd)): if str(sd[j].uuid) == uuid: self.wrappers[i].stop(j) if self.args.gdi: self.gdiInterface.removeSensorStream(uuid) return True else: if str(sd.uuid) == uuid: self.wrappers[i].stop() self.wrappers[i : i + 1] = [] self.gdiInterface.removeSensorStream(uuid) return True return False def getSensorDescriptionByUUID(self, uuid): for sd in self.__all_sensordescriptions(): if str(sd.uuid) == uuid: return sd return None def getEventDescriptionByUUID(self, uuid): if self.eventWrapper: return self.eventWrapper.getEventDescriptionByUUID(uuid) def getWrapperByUUID(self, uuid): from virtualisation.wrapper.abstractwrapper import AbstractComposedWrapper, AbstractWrapper for w in self.wrappers: if isinstance(w, AbstractWrapper): if str(w.getSensorDescription().uuid) == uuid: return w elif isinstance(w, AbstractComposedWrapper): for ww in w.wrappers: if str(ww.getSensorDescription().uuid) == uuid: return ww return None def getUUIDsForCategory(self, category): from virtualisation.wrapper.abstractwrapper import AbstractComposedWrapper, AbstractWrapper uuidList = [] for w in self.wrappers: if isinstance(w, AbstractWrapper): if str(w.getSensorDescription().sensorType) == category: uuidList.append(w.getSensorDescription().uuid) elif isinstance(w, AbstractComposedWrapper): for ww in w.wrappers: if str(ww.getSensorDescription().sensorType) == category: uuidList.append(ww.getSensorDescription().uuid) return uuidList def getEventWrapperDescriptions(self): if not self.eventWrapper: return None return map(lambda (k, v): v, self.eventWrapper.eventDescriptions.iteritems()) def deleteGraphs(self, wrapper): graphNames = wrapper.getGraphName() if type(graphNames) is str: # abstract wrapper returns one graph name ThreadedTriplestoreAdapter.getOrMake(graphNames).deleteGraph(graphNames) else: for graphName in graphNames: # composed abstract wrapper returns set with graph names ThreadedTriplestoreAdapter.getOrMake(graphName).deleteGraph(graphName) def startReplay(self): # cherrypy.tree.mount(dowser.Root(), '/dowser') self._startQueues() method = self.replayEnd args = None start_time = datetime.datetime.now() # if continuelive enabled set "start" as method to set the system to live mode if historic replay is finished if ResourceManagement.args.continuelive: method = self.start args = True if ResourceManagement.args.speed: self.clock = ReplayClock(ResourceManagement.args.speed, endCallback=method, endCallbackArgs=args) else: self.clock = ReplayClock(endCallback=method) if ResourceManagement.args.end and ResourceManagement.args.start: try: startDate = datetime.datetime.strptime(ResourceManagement.args.start, ReplayClock.parserformat) endDate = datetime.datetime.strptime(ResourceManagement.args.end, ReplayClock.parserformat) # for w in self.wrappers: # w.setTimeframe(startDate, endDate) if startDate > endDate: L.w("start date after end date. Changing both") tmp = endDate endDate = startDate startDate = tmp self.clock.setTimeframe(startDate, endDate) except Exception as e: L.e("Problem parsing start- or end date:", str(e)) raise e else: raise Exception("start- and enddate required for replay mode") if ResourceManagement.args.pt: from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes performancetest = PerformanceMeterMinutes() # PerformanceMeterSeconds() for w in self.wrappers: w.setReplayMode(True) w.setClock(self.clock) w.setTimeframe(startDate, endDate) w.addReceiver(self) if ResourceManagement.args.pt: w.addReceiver(performancetest) w.start() w.runReplay() if not self.args.noQuality: if not self.averageStreamQuality: self.averageStreamQuality = AverageStreamQuality(self, self.clock) else: self.averageStreamQuality.setClock(self.clock) self.clock.runAsync() self.startMonitor() if not ResourceManagement.args.continuelive: raw_input("press Enter to end.\n") self.clock.stop() L.i("Runtime", datetime.datetime.now() - start_time) def start(self, restart=False): self._startQueues() if self.clock: self.clock.stop() self.clock = RealClock(self.end) if ResourceManagement.args.pt: from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes performancetest = PerformanceMeterMinutes() for w in self.wrappers: self.startWrapper(w, restart) if ResourceManagement.args.pt: w.addReceiver(performancetest) L.i(datetime.datetime.now()) if not self.args.noQuality: if not self.averageStreamQuality: self.averageStreamQuality = AverageStreamQuality(self, self.clock) else: self.averageStreamQuality.setClock(self.clock) self.clock.runAsync() raw_input("press Enter to end.\n") self.clock.stop() def startWrapper(self, w, restart=False): w.setReplayMode(False) w.setClock(self.clock) if not restart: w.addReceiver(self) w.start() w.run() w.update() def replayEnd(self): if self.monitorTimer: self.monitorTimer.cancel() self.monitorTimer = None self.end() def end(self): for w in self.wrappers: w.stop() if ResourceManagement.args.messagebus: self.messageBusQueue.stop() if ResourceManagement.args.aggregate: self.aggregationQueue.stop() if self.eventWrapper: self.eventWrapper.stop() self.receiverQueue.stop() if ResourceManagement.args.triplestore: ThreadedTriplestoreAdapter.stop() self.stopInterface() L.i("REPLAY ENDED") # def registerExchanges(self): # try: # for ex in RabbitMQ.exchanges: # RabbitMQ.declareExchange(ex, _type="topic") # except Exception as e: # L.e('Exchange could not be declare: %s' % e.message) def sendMessageHandler(self, item): message, exchange, key = item L.d2(message) RabbitMQ.sendMessage(message, exchange, key) del message def aggregateHandler(self, item): # TODO data, sensordescription = item aggs = self.aggregator.aggregate(data, sensordescription) if ResourceManagement.args.messagebus and aggs: for agg in aggs: annotated_agg = self.annotator.annotateAggregation(agg, sensordescription) msg = annotated_agg.serialize(format="n3") self.messageBusQueue.add( (msg, RabbitMQ.exchange_aggregated_data, sensordescription.messagebus.routingKey) ) del aggs pass def receive(self, parsedData, sensordescription, clock, quality): # L.i("Received data from", sensordescription.fullSensorID) if len(parsedData.fields) != 0: self.receiverQueue.add((parsedData, sensordescription, clock, quality)) def monitor(self): # change speed level in replay mode to avoid high memory consumption # L.e("Monitor, Queue Size:", self.receiverQueue.getQueueSize()) if self.receiverQueue.getQueueSize() > 1000: if not self.stoppedClock: self.clock.pause() self.stoppedClock = True # L.e("Monitor stop clock, Queue Size:", self.receiverQueue.getQueueSize()) elif self.receiverQueue.getQueueSize() < 500: if self.stoppedClock: self.clock.continue_running() self.stoppedClock = False # L.e("Monitor continue clock, Queue Size:", self.receiverQueue.getQueueSize()) if self.monitorTimer: self.startMonitor() def startMonitor(self): self.monitorTimer = threading.Timer(10, self.monitor) self.monitorTimer.start() def receiveHandler(self, item): parsedData, sensordescription, clock, quality = item L.d2(parsedData) # print parsedData if self.sql: self.sql.insert_observation(sensordescription, parsedData, dictdeepcopy(quality)) if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore: # if len(parsedData.fields)>0: g = self.annotator.annotateObservation(parsedData, sensordescription, clock, quality) del quality if ( ResourceManagement.args.messagebus and not sensordescription.no_publish_messagebus and "fields" in parsedData and len(parsedData.fields) > 0 ): message = g.serialize(format="n3") # print message key = sensordescription.messagebus.routingKey # self.messageBusQueue.add((parsedData.dumps(), self.rabbitmqchannel, RabbitMQ.exchange_data, key)) self.messageBusQueue.add((message, RabbitMQ.exchange_annotated_data, key)) if self.ui.api: self.ui.api.update_observation_cache(str(sensordescription.uuid), message) if ResourceManagement.args.triplestore: # TODO: The following line is commented out, since the Virtuoso makes so much trouble # ThreadedTriplestoreAdapter.getOrMake(sensordescription.graphName).addGraph(g) pass if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore: del g if ResourceManagement.args.aggregate: self.aggregationQueue.add((parsedData, sensordescription)) else: del parsedData
"""@Daniel P: Now I had time to look a bit closer into your code. I would suggest to make just an example EventDescription to show Dan what we need and to test the annotation part. You will probably need a pseudo EventGenerator for that. Later, the ResourceManagement API will receive new EventDescriptions and instanciate EventWrapper (or best case we can reuse one instance). There is no need to make the metadata file nor to instanciate an EventWrapper for each REPORT_ID.""" self.eventDescription = eventdescription def getEventDescription(self): return self.eventDescription if __name__ == "__main__": atew = AarhusTrafficEventWrapper() from virtualisation.annotation.genericannotation import GenericAnnotation annotator = GenericAnnotation() eventData = JSONObject() eventData.ceID = 123456 eventData.ceType = "traffic" eventData.ceName = "traffic jam" eventData.ceTime = 1438591234000L eventData.ceCoordinate = "(56.12 10.13)" eventData.ceLevel = 1 print "Incoming event data", eventData.dumps() print g = annotator.annotateEvent(eventData, atew.getEventDescription()) print "Resulting graph", g.serialize(format='n3') print
class AbstractEventWrapper(object): __metaclass__ = abc.ABCMeta def __init__(self): self.config = JSONObject(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb")) self.host = self.config.rabbitmq.host self.port = self.config.rabbitmq.port # self.rabbitmqconnection, self.rabbitmqchannel = RabbitMQ.establishConnection(self.host, self.port) self.messageBusReceiveQueue = QueueThread(handler=self.receiveEventHandler) self.messageBusSendQueue = QueueThread(handler=self.sendAnnotatedEventHandler) self.wrappers = [] self.splitters = None self.annotator = GenericAnnotation() self.exchange = RabbitMQ.exchange_annotated_event @abc.abstractmethod def getEventDescription(self): """ :return: a event description """ pass @classmethod def getFileObject(cls, currentfile, filename, mode="r"): parent = os.path.dirname(currentfile) if parent.endswith(".zip"): zFile = zipfile.ZipFile(parent) return zFile.open(filename, mode) else: return file(os.path.join(parent, filename), mode) # def addWrapper(self, wrapper): # """ # adds a wrapper to the internal wrapper list # :param wrapper: # :return: # """ # if not isinstance(wrapper, AbstractEventWrapper): # raise Exception(error="trying to add a wrapper of the wrong instance. Requires AbstractEventWRapper") # self.wrappers.append(wrapper) def start(self): "@Daniel P: The ResourceManagement declares all available exchanges. I guess this is unnecessary therefore." # RabbitMQ.declareExchange(self.rabbitmqchannel, self.exchange, _type="topic") queue = RabbitMQ.channel.queue_declare() queue_name = queue.method.queue # in the following line the exchange should be RabbitMQ.exchange_event RabbitMQ.channel.queue_bind(exchange=self.exchange, queue=queue_name, routing_key=self.getEventDescription().messagebus.routingKey) def run(self): """ start listening on the event detection component :return: """ # self.__forEachWrapper("run") self.runthread = threading.Thread(target=self._run) self.runthread.start() def _run(self): self.channel.basic_consume(self.receiveEventHandler, no_ack=True) self.channel.start_consuming() def receiveEventHandler(self, channel, method, properties, body): """ Receives messages throught the message bus, annotates the event and sends the annotated event :param channel: :param method: :param properties: :param body: :return: """ event = json.loads(body) annotatedevent = self.annotateEvent(event) self.messageBusSendQueue.add(annotatedevent) def annotateEvent(self, event): """ Annotates the event and saves the graph in the triple store :param event: :return: returns the annotated graph of the event """ graph = self.annotator.annotateEvent(event, self.getEventDescription()) ThreadedTriplestoreAdapter.getOrMake(self.getEventDescription().graphname) return graph def sendAnnotatedEventHandler(self, annotatedevent): key = self.getEventDescription().messagebus.routingKey message = annotatedevent.serialize(format='n3') RabbitMQ.sendMessage(message, self.exchange, key)