def __init__(self, args):
L(args.log)
self.config = JOb(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb"))
self.status = JSONObject()
self.status.running = False
self.wrappers = []
ResourceManagement.args = args
ResourceManagement.config = self.config
self.clock = None
self.annotator = GenericAnnotation()
self.stoppedClock = False
# init triplestore
if args.triplestore:
ThreadedTriplestoreAdapter.triplestore = TripleStoreFactory.getTripleStore(
self.config.triplestore.driver, self.config.triplestore
)
# dump the graph with the static annotations
StaticAnnotator.dumpGraph()
if args.aggregate:
# self.aggregator = AggregatorFactory.make(self.config.aggregationmethod, self.config.aggregation_configuration)
# self.aggregator = AggregatorFactory.make()
# TODO
self.aggregator = GenericAggregator()
self.aggregationQueue = QueueThread(handler=self.aggregateHandler)
self.receiverQueue = QueueThread(handler=self.receiveHandler)
# establish connection to the message bus
if args.messagebus:
threading.Thread(name="messageBusConnector", target=self.start_messagebus, args=(args,)).start()
else:
self.messageBusQueue = None
if args.eventannotation:
self.eventWrapper = GenericEventWrapper(self.messageBusQueue)
else:
self.eventWrapper = None
if args.gdi:
from CityPulseGdi.eu.citypulse.uaso.gdi.CityPulseGDInterface import CityPulseGDInterface
self.gdiInterface = CityPulseGDInterface(self.config.gdi_db)
#
self.gdiInterface.removeAllSensorStreams()
else:
self.gdiInterface = None
self.averageStreamQuality = None
if args.sql:
self.sql = SQL(self.config.gdi_db, self)
else:
self.sql = None
self.startInterface()
self.autodeploy()
class ResourceManagement(virtualisation.wrapper.wrapperoutputreceiver.AbstractReceiver):
deployfoldername = os.path.join(os.path.dirname(__file__), "..", "autodeploy")
eventdescriptionfoldername = os.path.join(os.path.dirname(__file__), "..", "eventdescriptions")
args = None
config = None
def __init__(self, args):
L(args.log)
self.config = JOb(file(os.path.join(os.path.dirname(__file__), "..", "config.json"), "rb"))
self.status = JSONObject()
self.status.running = False
self.wrappers = []
ResourceManagement.args = args
ResourceManagement.config = self.config
self.clock = None
self.annotator = GenericAnnotation()
self.stoppedClock = False
# init triplestore
if args.triplestore:
ThreadedTriplestoreAdapter.triplestore = TripleStoreFactory.getTripleStore(
self.config.triplestore.driver, self.config.triplestore
)
# dump the graph with the static annotations
StaticAnnotator.dumpGraph()
if args.aggregate:
# self.aggregator = AggregatorFactory.make(self.config.aggregationmethod, self.config.aggregation_configuration)
# self.aggregator = AggregatorFactory.make()
# TODO
self.aggregator = GenericAggregator()
self.aggregationQueue = QueueThread(handler=self.aggregateHandler)
self.receiverQueue = QueueThread(handler=self.receiveHandler)
# establish connection to the message bus
if args.messagebus:
threading.Thread(name="messageBusConnector", target=self.start_messagebus, args=(args,)).start()
else:
self.messageBusQueue = None
if args.eventannotation:
self.eventWrapper = GenericEventWrapper(self.messageBusQueue)
else:
self.eventWrapper = None
if args.gdi:
from CityPulseGdi.eu.citypulse.uaso.gdi.CityPulseGDInterface import CityPulseGDInterface
self.gdiInterface = CityPulseGDInterface(self.config.gdi_db)
#
self.gdiInterface.removeAllSensorStreams()
else:
self.gdiInterface = None
self.averageStreamQuality = None
if args.sql:
self.sql = SQL(self.config.gdi_db, self)
else:
self.sql = None
self.startInterface()
self.autodeploy()
def _startQueues(self):
if not self.receiverQueue.is_alive():
self.receiverQueue.start()
# since we need to send messages when a new wrapper is added, the messageBusQueue needs to be started in __init__
# if self.args.messagebus:
# self.messageBusQueue.start()
if ResourceManagement.args.aggregate:
if not self.aggregationQueue.is_alive():
self.aggregationQueue.start()
if self.eventWrapper:
if not self.eventWrapper.isStarted():
self.eventWrapper.start()
def start_messagebus(self, args):
L.i("Connecting to the message bus")
self.messageBusQueue = QueueThread(handler=self.sendMessageHandler)
try:
# prepare RabbitMQ configuration
rmq_host = str(self.config.rabbitmq.host)
rmq_port = self.config.rabbitmq.port
rmq_username = self.config.rabbitmq.username if "username" in self.config.rabbitmq else None
rmq_password = self.config.rabbitmq.username if "password" in self.config.rabbitmq else None
if rmq_username:
if rmq_password:
RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username, rmq_password)
else:
RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username)
else:
RabbitMQ.establishConnection(rmq_host, rmq_port)
L.i("Connected to the message bus")
self.messageBusQueue.start()
except MessageBusConnectionError:
self.args.messagebus = False
args.messagebus = False
L.w("Could not connect to MessageBus server. Disabling MessageBus feature.")
def __all_sensordescriptions(self):
sds = []
for w in self.wrappers:
if not isinstance(w.getSensorDescription(), list):
sds += [w.getSensorDescription()]
else:
sds += w.getSensorDescription()
return sds
def autodeploy(self):
deployFolder = os.path.join(ResourceManagement.deployfoldername, "*.zip")
files = glob.glob(deployFolder)
for f in files:
self.deploy(f)
self.status.running = True
def deploy(self, f, autostart=False):
"""
:param f:
:param autostart:
:return: a tuple with 3 elements. 1. status as string, 2. error message as string, 3. list of uuids of added wrapper
"""
L.i("Deploying", f)
sensordescriptions = []
try:
zFile = zipfile.ZipFile(f)
if "deploy.json" in zFile.namelist():
deployDescription = JOb(zFile.open("deploy.json", "r"))
sys.path.insert(0, f)
if deployDescription.isList():
for dd in deployDescription:
module = __import__(dd.module)
wrapper = getattr(module, dd["class"])()
self.addWrapper(wrapper)
sensordescriptions.append(wrapper.getSensorDescription())
if autostart:
self.startWrapper(wrapper)
else:
module = __import__(deployDescription.module)
wrapper = getattr(module, deployDescription["class"])()
self.addWrapper(wrapper)
sensordescriptions.append(wrapper.getSensorDescription())
if autostart:
self.startWrapper(wrapper)
return "OK", "", sensordescriptions
except Exception as e:
L.w("Deployment of wrapper", f, "failed.", e.message)
return "Fail", e.message, []
def registerEvent(self, eventDescriptionFile):
if self.eventWrapper:
eventDescription = EventDescription(open(eventDescriptionFile, "rb"))
if not eventDescription.test():
return "Fail", "Invalid EventDescription"
self.eventWrapper.registerEvent(eventDescription)
return "Ok", ""
else:
return "Fail", "Event wrapper disabled"
def startInterface(self):
self.ui = Ui(self)
self.ui.api = Api(self)
self.ui.stat_api = Stat_Api(self)
threading.Thread(target=cherrypy.quickstart, args=(self.ui, "/", self.config.interface.raw())).start()
def stopInterface(self):
cherrypy.engine.exit()
def test(self):
# from wrapper_dev.osna_dummy.osna_dummy_wrapper import OsnaDummyWrapper
# wrapper = OsnaDummyWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.london_traffic.londontrafficwrapper import LondonTrafficWrapper
# wrapper = LondonTrafficWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.aarhus_traffic.aarhustrafficwrapper import AarhusTrafficWrapper
# wrapper = AarhusTrafficWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.aarhus_parking.aarhusparkingwrapper import AarhusParkingWrapper
# wrapper = AarhusParkingWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.brasov_pollution.brasovpollutionwrapper import BrasovPollutionWrapper
# wrapper = BrasovPollutionWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.romania_weather.romaniaweather_aw import RomanianWeatherAWWrapper
# wrapper = RomanianWeatherAWWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.romania_weather.romaniaweather_mr import RomanianWeatherMRWrapper
# wrapper = RomanianWeatherMRWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.brasov_incidents.brasov_incidents import BrasovIncidentWrapper0, BrasovIncidentWrapper1
# wrapper = BrasovIncidentWrapper0()
# self.addWrapper(wrapper)
# wrapper = BrasovIncidentWrapper1()
# self.addWrapper(wrapper)
# from wrapper_dev.aarhus_pollution.aarhuspollutionwrapper import ComposedAarhusPollutionWrapper
# wrapper = ComposedAarhusPollutionWrapper()
# self.addWrapper(wrapper)
# from wrapper_dev.twitter_stream.twitter import AarhusTwitterWrapper
# wrapper = AarhusTwitterWrapper()
# self.addWrapper(wrapper)
from wrapper_dev.WrapperBusStations.brasovbusstationwrapper import BrasovBusStationWrapper
wrapper = BrasovBusStationWrapper()
self.addWrapper(wrapper)
if self.args.eventannotation:
from wrapper_dev.aarhus_traffic.aarhustrafficeventwrapper import AarhusTrafficEventWrapper
atew = AarhusTrafficEventWrapper()
self.eventWrapper.registerEvent(atew.getEventDescription())
def addWrapper(self, wrapper):
# TODO: this should not be here
if ResourceManagement.args.cleartriplestore:
self.deleteGraphs(wrapper)
sd = wrapper.getSensorDescription()
try:
if isinstance(sd, list):
for _sd in sd:
try:
_sd.test()
if ResourceManagement.args.aggregate:
self.aggregator.wrapper_added(_sd)
if self.gdiInterface:
self.gdiInterface.registerSensorStreamFromWKT(
_sd.uuid, _sd.sensorID, _sd.sensorType, _sd.location, _sd.location_epsg or 4326
)
# if self.sql:
# self.sql.create_table(_sd)
L.i("added wrapper with ID", _sd.sensorID)
except Exception as ex:
L.e("Error deploying wrapper:", str(ex))
else:
try:
sd.test()
if ResourceManagement.args.aggregate:
self.aggregator.wrapper_added(sd)
if self.gdiInterface:
self.gdiInterface.registerSensorStreamFromWKT(
sd.uuid, sd.sensorID, sd.sensorType, sd.location, sd.location_epsg or 4326
)
# if self.sql:
# self.sql.create_table(sd)
L.i("added wrapper with ID", sd.sensorID)
except Exception as ex:
L.e("Error deploying wrapper:", str(ex))
if ResourceManagement.args.triplestore or ResourceManagement.args.messagebus:
# StaticAnnotator.staticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.rabbitmqchannel)
StaticAnnotator.threadedStaticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.ui.api)
if ResourceManagement.args.messagebus:
wrapper.setMessageBusQueue(self.messageBusQueue)
self.wrappers.append(wrapper)
except Exception as ex:
L.e(self.__class__.__name__, "Error in addWrapper:", str(ex))
def removeWrapper(self, uuid):
"""
Removes a wrapper identified by its UUID
:param uuid:
:return: True if successful, otherwise False
"""
for i in range(0, len(self.wrappers)):
sd = self.wrappers[i].getSensorDescription()
if isinstance(sd, list):
for j in range(0, len(sd)):
if str(sd[j].uuid) == uuid:
self.wrappers[i].stop(j)
if self.args.gdi:
self.gdiInterface.removeSensorStream(uuid)
return True
else:
if str(sd.uuid) == uuid:
self.wrappers[i].stop()
self.wrappers[i : i + 1] = []
self.gdiInterface.removeSensorStream(uuid)
return True
return False
def getSensorDescriptionByUUID(self, uuid):
for sd in self.__all_sensordescriptions():
if str(sd.uuid) == uuid:
return sd
return None
def getEventDescriptionByUUID(self, uuid):
if self.eventWrapper:
return self.eventWrapper.getEventDescriptionByUUID(uuid)
def getWrapperByUUID(self, uuid):
from virtualisation.wrapper.abstractwrapper import AbstractComposedWrapper, AbstractWrapper
for w in self.wrappers:
if isinstance(w, AbstractWrapper):
if str(w.getSensorDescription().uuid) == uuid:
return w
elif isinstance(w, AbstractComposedWrapper):
for ww in w.wrappers:
if str(ww.getSensorDescription().uuid) == uuid:
return ww
return None
def getUUIDsForCategory(self, category):
from virtualisation.wrapper.abstractwrapper import AbstractComposedWrapper, AbstractWrapper
uuidList = []
for w in self.wrappers:
if isinstance(w, AbstractWrapper):
if str(w.getSensorDescription().sensorType) == category:
uuidList.append(w.getSensorDescription().uuid)
elif isinstance(w, AbstractComposedWrapper):
for ww in w.wrappers:
if str(ww.getSensorDescription().sensorType) == category:
uuidList.append(ww.getSensorDescription().uuid)
return uuidList
def getEventWrapperDescriptions(self):
if not self.eventWrapper:
return None
return map(lambda (k, v): v, self.eventWrapper.eventDescriptions.iteritems())
def deleteGraphs(self, wrapper):
graphNames = wrapper.getGraphName()
if type(graphNames) is str: # abstract wrapper returns one graph name
ThreadedTriplestoreAdapter.getOrMake(graphNames).deleteGraph(graphNames)
else:
for graphName in graphNames: # composed abstract wrapper returns set with graph names
ThreadedTriplestoreAdapter.getOrMake(graphName).deleteGraph(graphName)
def startReplay(self):
# cherrypy.tree.mount(dowser.Root(), '/dowser')
self._startQueues()
method = self.replayEnd
args = None
start_time = datetime.datetime.now()
# if continuelive enabled set "start" as method to set the system to live mode if historic replay is finished
if ResourceManagement.args.continuelive:
method = self.start
args = True
if ResourceManagement.args.speed:
self.clock = ReplayClock(ResourceManagement.args.speed, endCallback=method, endCallbackArgs=args)
else:
self.clock = ReplayClock(endCallback=method)
if ResourceManagement.args.end and ResourceManagement.args.start:
try:
startDate = datetime.datetime.strptime(ResourceManagement.args.start, ReplayClock.parserformat)
endDate = datetime.datetime.strptime(ResourceManagement.args.end, ReplayClock.parserformat)
# for w in self.wrappers:
# w.setTimeframe(startDate, endDate)
if startDate > endDate:
L.w("start date after end date. Changing both")
tmp = endDate
endDate = startDate
startDate = tmp
self.clock.setTimeframe(startDate, endDate)
except Exception as e:
L.e("Problem parsing start- or end date:", str(e))
raise e
else:
raise Exception("start- and enddate required for replay mode")
if ResourceManagement.args.pt:
from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes
performancetest = PerformanceMeterMinutes() # PerformanceMeterSeconds()
for w in self.wrappers:
w.setReplayMode(True)
w.setClock(self.clock)
w.setTimeframe(startDate, endDate)
w.addReceiver(self)
if ResourceManagement.args.pt:
w.addReceiver(performancetest)
w.start()
w.runReplay()
if not self.args.noQuality:
if not self.averageStreamQuality:
self.averageStreamQuality = AverageStreamQuality(self, self.clock)
else:
self.averageStreamQuality.setClock(self.clock)
self.clock.runAsync()
self.startMonitor()
if not ResourceManagement.args.continuelive:
raw_input("press Enter to end.\n")
self.clock.stop()
L.i("Runtime", datetime.datetime.now() - start_time)
def start(self, restart=False):
self._startQueues()
if self.clock:
self.clock.stop()
self.clock = RealClock(self.end)
if ResourceManagement.args.pt:
from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes
performancetest = PerformanceMeterMinutes()
for w in self.wrappers:
self.startWrapper(w, restart)
if ResourceManagement.args.pt:
w.addReceiver(performancetest)
L.i(datetime.datetime.now())
if not self.args.noQuality:
if not self.averageStreamQuality:
self.averageStreamQuality = AverageStreamQuality(self, self.clock)
else:
self.averageStreamQuality.setClock(self.clock)
self.clock.runAsync()
raw_input("press Enter to end.\n")
self.clock.stop()
def startWrapper(self, w, restart=False):
w.setReplayMode(False)
w.setClock(self.clock)
if not restart:
w.addReceiver(self)
w.start()
w.run()
w.update()
def replayEnd(self):
if self.monitorTimer:
self.monitorTimer.cancel()
self.monitorTimer = None
self.end()
def end(self):
for w in self.wrappers:
w.stop()
if ResourceManagement.args.messagebus:
self.messageBusQueue.stop()
if ResourceManagement.args.aggregate:
self.aggregationQueue.stop()
if self.eventWrapper:
self.eventWrapper.stop()
self.receiverQueue.stop()
if ResourceManagement.args.triplestore:
ThreadedTriplestoreAdapter.stop()
self.stopInterface()
L.i("REPLAY ENDED")
# def registerExchanges(self):
# try:
# for ex in RabbitMQ.exchanges:
# RabbitMQ.declareExchange(ex, _type="topic")
# except Exception as e:
# L.e('Exchange could not be declare: %s' % e.message)
def sendMessageHandler(self, item):
message, exchange, key = item
L.d2(message)
RabbitMQ.sendMessage(message, exchange, key)
del message
def aggregateHandler(self, item):
# TODO
data, sensordescription = item
aggs = self.aggregator.aggregate(data, sensordescription)
if ResourceManagement.args.messagebus and aggs:
for agg in aggs:
annotated_agg = self.annotator.annotateAggregation(agg, sensordescription)
msg = annotated_agg.serialize(format="n3")
self.messageBusQueue.add(
(msg, RabbitMQ.exchange_aggregated_data, sensordescription.messagebus.routingKey)
)
del aggs
pass
def receive(self, parsedData, sensordescription, clock, quality):
# L.i("Received data from", sensordescription.fullSensorID)
if len(parsedData.fields) != 0:
self.receiverQueue.add((parsedData, sensordescription, clock, quality))
def monitor(self):
# change speed level in replay mode to avoid high memory consumption
# L.e("Monitor, Queue Size:", self.receiverQueue.getQueueSize())
if self.receiverQueue.getQueueSize() > 1000:
if not self.stoppedClock:
self.clock.pause()
self.stoppedClock = True
# L.e("Monitor stop clock, Queue Size:", self.receiverQueue.getQueueSize())
elif self.receiverQueue.getQueueSize() < 500:
if self.stoppedClock:
self.clock.continue_running()
self.stoppedClock = False
# L.e("Monitor continue clock, Queue Size:", self.receiverQueue.getQueueSize())
if self.monitorTimer:
self.startMonitor()
def startMonitor(self):
self.monitorTimer = threading.Timer(10, self.monitor)
self.monitorTimer.start()
def receiveHandler(self, item):
parsedData, sensordescription, clock, quality = item
L.d2(parsedData)
# print parsedData
if self.sql:
self.sql.insert_observation(sensordescription, parsedData, dictdeepcopy(quality))
if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore:
# if len(parsedData.fields)>0:
g = self.annotator.annotateObservation(parsedData, sensordescription, clock, quality)
del quality
if (
ResourceManagement.args.messagebus
and not sensordescription.no_publish_messagebus
and "fields" in parsedData
and len(parsedData.fields) > 0
):
message = g.serialize(format="n3")
# print message
key = sensordescription.messagebus.routingKey
# self.messageBusQueue.add((parsedData.dumps(), self.rabbitmqchannel, RabbitMQ.exchange_data, key))
self.messageBusQueue.add((message, RabbitMQ.exchange_annotated_data, key))
if self.ui.api:
self.ui.api.update_observation_cache(str(sensordescription.uuid), message)
if ResourceManagement.args.triplestore:
# TODO: The following line is commented out, since the Virtuoso makes so much trouble
# ThreadedTriplestoreAdapter.getOrMake(sensordescription.graphName).addGraph(g)
pass
if ResourceManagement.args.messagebus or ResourceManagement.args.triplestore:
del g
if ResourceManagement.args.aggregate:
self.aggregationQueue.add((parsedData, sensordescription))
else:
del parsedData