def __init__(self, gdi_config, rm):
self.rm = rm
self.PAGINATION_LIMIT = 100
connect_str = "host='%s' dbname='%s' user='******' password='******' port=%d" % (
gdi_config.host, gdi_config.database, gdi_config.username, gdi_config.password, gdi_config.port)
self.conn = psycopg2.connect(connect_str)
self.curs = self.conn.cursor()
try:
self.curs.execute("CREATE SCHEMA IF NOT EXISTS %s;" % SQL.SCHEMA)
# self.curs.execute("CREATE TABLE IF NOT EXISTS %s.cp_sensors (sensor_uuid UUID CONSTRAINT uuid_key PRIMARY KEY, sensor_annotation_id VARCHAR, sercvice_category VARCHAR, traffic INTEGER, geom GEOMETRY(GEOMETRY, 4326) );" % (SQL.SCHEMA,))
self.curs.execute("CREATE TABLE IF NOT EXISTS %s.cp_sensors (sensor_uuid UUID CONSTRAINT uuid_key PRIMARY KEY, sensor_annotation_id VARCHAR, sercvice_category VARCHAR, traffic INTEGER, geom GEOMETRY(GEOMETRY, 4326) );" % ("public",))
cols = ["sampling_time TIMESTAMP", "sensor_uuid UUID", "observation_uuid UUID", "data JSON", "quality JSON"]
query = 'CREATE TABLE IF NOT EXISTS %s.cp_observations ( %s, PRIMARY KEY (%s), FOREIGN KEY (sensor_uuid) REFERENCES %s.cp_sensors(sensor_uuid));\n' % (SQL.SCHEMA, ', '.join(cols), ", ".join(["observation_uuid"]), "public")
self.curs.execute(query)
# index over sampling_time and sensor_uuid
# since a 'IF NOT EXISTS' is not available for us (version < 9.5)
# the error is catched in a separate try-catch
try:
query = 'CREATE INDEX "timeindex" ON %s.cp_observations USING btree (sampling_time);' % (SQL.SCHEMA,)
self.curs.execute(query)
query = 'CREATE INDEX uuidindex ON %s.cp_observations USING btree (sensor_uuid);' % (SQL.SCHEMA,)
self.curs.execute(query)
except:
pass
# primary secondary observation_uuid map
query = 'CREATE TABLE IF NOT EXISTS %s.p_s_observation_uuid (main UUID, secondary UUID);' % (SQL.SCHEMA,)
self.curs.execute(query)
self.conn.commit()
L.i("SQL: schema/tables created")
except Exception as e:
L.e("SQL: Could not create schema/tables", e)
self.conn.rollback()
def start(self, restart=False):
self._startQueues()
if self.clock:
self.clock.stop()
self.clock = RealClock(self.end)
if ResourceManagement.args.pt:
from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes
performancetest = PerformanceMeterMinutes()
for w in self.wrappers:
self.startWrapper(w, restart)
if ResourceManagement.args.pt:
w.addReceiver(performancetest)
L.i(datetime.datetime.now())
if not self.args.noQuality:
if not self.averageStreamQuality:
self.averageStreamQuality = AverageStreamQuality(self, self.clock)
else:
self.averageStreamQuality.setClock(self.clock)
self.clock.runAsync()
raw_input("press Enter to end.\n")
self.clock.stop()
def deploy(self, f, autostart=False):
"""
:param f:
:param autostart:
:return: a tuple with 3 elements. 1. status as string, 2. error message as string, 3. list of uuids of added wrapper
"""
L.i("Deploying", f)
sensordescriptions = []
try:
zFile = zipfile.ZipFile(f)
if "deploy.json" in zFile.namelist():
deployDescription = JOb(zFile.open("deploy.json", "r"))
sys.path.insert(0, f)
if deployDescription.isList():
for dd in deployDescription:
module = __import__(dd.module)
wrapper = getattr(module, dd["class"])()
self.addWrapper(wrapper)
sensordescriptions.append(wrapper.getSensorDescription())
if autostart:
self.startWrapper(wrapper)
else:
module = __import__(deployDescription.module)
wrapper = getattr(module, deployDescription["class"])()
self.addWrapper(wrapper)
sensordescriptions.append(wrapper.getSensorDescription())
if autostart:
self.startWrapper(wrapper)
return "OK", "", sensordescriptions
except Exception as e:
L.w("Deployment of wrapper", f, "failed.", e.message)
return "Fail", e.message, []
def __run(self):
while not self.stop:
sleep(self.delay)
Log.i(self.counter, self.txt)
# print self.counter, self.txt
# print "ThreadedTriplestoreAdapter Buffer Size:", ThreadedTriplestoreAdapter.getTotalBufferSize()
Log.i("ThreadedTriplestoreAdapter Buffer Size:", ThreadedTriplestoreAdapter.getTotalBufferSize())
self.counter = 0
def setTimeframe(self, startdate, enddate):
if not isinstance(self.data, list):
Log.i("Searching start date in historic data for", self.wrapper.getSensorDescription().sensorID, "...")
if self.data.scrollTo(startdate):
Log.i("done")
else:
Log.w("no historic data beginning at", startdate, "found")
super(CSVHistoryReader, self).setTimeframe(startdate, enddate)
def setReplayMode(self, mode):
if mode:
try:
Log.i("loading history for", self.sensorDescription.sensorID, "...")
self.historyreader = CSVHistoryReader(self, AbstractWrapper.getFileObject(__file__, os.path.join("historicdata", "trafficData%d.csv" % self.sensorDescription.sensorID), "rU"))
self.historyreader.multiple_observations = False
Log.i("done")
self.historyparser = CSVParser(self, self.historyreader.headers)
# connection will be set automatically by the AbstractComposedWrapper to SplitterConnection
except Exception as e:
Log.e(e)
self.historyreader = None
super(InternalWrapper, self).setReplayMode(mode)
def end(self):
for w in self.wrappers:
w.stop()
if ResourceManagement.args.messagebus:
self.messageBusQueue.stop()
if ResourceManagement.args.aggregate:
self.aggregationQueue.stop()
if self.eventWrapper:
self.eventWrapper.stop()
self.receiverQueue.stop()
if ResourceManagement.args.triplestore:
ThreadedTriplestoreAdapter.stop()
self.stopInterface()
L.i("REPLAY ENDED")
def setReplayMode(self, mode):
if mode:
try:
Log.i("loading history data for", self.sensorDescription.sensorID, "...")
self.historyreader = CSVHistoryReader(self, AbstractWrapper.getFileObject(__file__, os.path.join("historicdata", "aarhus_parking-%s.csv" % self.sensorDescription.sensorID), "rU"), timestampfield="updatetime")
# Must preserve the order as in the CSV but use the names as in senordescription.fields
# vehiclecount,updatetime,_id,totalspaces,garagecode,streamtime
self.historyparser = CSVParser(self, ["vehicleCount", "updatetime", "_id", "totalSpaces", "garageCode", "st"], timestampfield="updatetime")
Log.i("done")
except Exception as e:
Log.e(e)
self.historyreader = None
super(InternalWrapper, self).setReplayMode(mode)
def addWrapper(self, wrapper):
# TODO: this should not be here
if ResourceManagement.args.cleartriplestore:
self.deleteGraphs(wrapper)
sd = wrapper.getSensorDescription()
try:
if isinstance(sd, list):
for _sd in sd:
try:
_sd.test()
if ResourceManagement.args.aggregate:
self.aggregator.wrapper_added(_sd)
if self.gdiInterface:
self.gdiInterface.registerSensorStreamFromWKT(
_sd.uuid, _sd.sensorID, _sd.sensorType, _sd.location, _sd.location_epsg or 4326
)
# if self.sql:
# self.sql.create_table(_sd)
L.i("added wrapper with ID", _sd.sensorID)
except Exception as ex:
L.e("Error deploying wrapper:", str(ex))
else:
try:
sd.test()
if ResourceManagement.args.aggregate:
self.aggregator.wrapper_added(sd)
if self.gdiInterface:
self.gdiInterface.registerSensorStreamFromWKT(
sd.uuid, sd.sensorID, sd.sensorType, sd.location, sd.location_epsg or 4326
)
# if self.sql:
# self.sql.create_table(sd)
L.i("added wrapper with ID", sd.sensorID)
except Exception as ex:
L.e("Error deploying wrapper:", str(ex))
if ResourceManagement.args.triplestore or ResourceManagement.args.messagebus:
# StaticAnnotator.staticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.rabbitmqchannel)
StaticAnnotator.threadedStaticAnnotationSensor(wrapper, self.config, self.messageBusQueue, self.ui.api)
if ResourceManagement.args.messagebus:
wrapper.setMessageBusQueue(self.messageBusQueue)
self.wrappers.append(wrapper)
except Exception as ex:
L.e(self.__class__.__name__, "Error in addWrapper:", str(ex))
def tick(self):
# Log.d2("AbstractClock tick")
self.dayPrintCounter -= 1
if self.dayPrintCounter == 0:
self.dayPrintCounter = AbstractClock.secondsInADay
Log.i("It is", self.now())
print "It is", self.now()
# Log.d2("AbstractClock notification handling")
notifications = filter(self.filtermethod, self.notifications.copy().iteritems())
self.pause(len(notifications))
for k, v in notifications:
v.do()
del notifications
# Log.d2("AbstractClock notification handling finished")
# Log.d2("AbstractClock job")
jobs = filter(self.filtermethod, self.jobs.copy().iteritems())
for k, v in jobs:
if self.execute_jobs_async:
threading.Thread(target=v.do).start()
else:
v.do()
del jobs
# Log.d2("AbstractClock job handling finished")
# Log.d2("AbstractClock delete self.notifications/jobs")
self.lock.acquire()
notifications = filter(self.removefiltermethod, self.notifications.copy().iteritems())
for k, v in notifications:
del self.notifications[k]
jobs = filter(self.removefiltermethod, self.jobs.copy().iteritems())
for k, v in jobs:
if k in self.jobs:
del self.jobs[k]
# Log.d2("AbstractClock delete finished")
# tmp = filter(self.removefiltermethod, self.notifications.copy().iteritems())
# del self.notifications
# self.notifications = tmp
# tmp = filter(self.removefiltermethod, self.jobs.copy().iteritems())
# del self.jobs
# self.jobs = tmp
self.lock.release()
def start_messagebus(self, args):
L.i("Connecting to the message bus")
self.messageBusQueue = QueueThread(handler=self.sendMessageHandler)
try:
# prepare RabbitMQ configuration
rmq_host = str(self.config.rabbitmq.host)
rmq_port = self.config.rabbitmq.port
rmq_username = self.config.rabbitmq.username if "username" in self.config.rabbitmq else None
rmq_password = self.config.rabbitmq.username if "password" in self.config.rabbitmq else None
if rmq_username:
if rmq_password:
RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username, rmq_password)
else:
RabbitMQ.establishConnection(rmq_host, rmq_port, rmq_username)
else:
RabbitMQ.establishConnection(rmq_host, rmq_port)
L.i("Connected to the message bus")
self.messageBusQueue.start()
except MessageBusConnectionError:
self.args.messagebus = False
args.messagebus = False
L.w("Could not connect to MessageBus server. Disabling MessageBus feature.")
def startReplay(self):
# cherrypy.tree.mount(dowser.Root(), '/dowser')
self._startQueues()
method = self.replayEnd
args = None
start_time = datetime.datetime.now()
# if continuelive enabled set "start" as method to set the system to live mode if historic replay is finished
if ResourceManagement.args.continuelive:
method = self.start
args = True
if ResourceManagement.args.speed:
self.clock = ReplayClock(ResourceManagement.args.speed, endCallback=method, endCallbackArgs=args)
else:
self.clock = ReplayClock(endCallback=method)
if ResourceManagement.args.end and ResourceManagement.args.start:
try:
startDate = datetime.datetime.strptime(ResourceManagement.args.start, ReplayClock.parserformat)
endDate = datetime.datetime.strptime(ResourceManagement.args.end, ReplayClock.parserformat)
# for w in self.wrappers:
# w.setTimeframe(startDate, endDate)
if startDate > endDate:
L.w("start date after end date. Changing both")
tmp = endDate
endDate = startDate
startDate = tmp
self.clock.setTimeframe(startDate, endDate)
except Exception as e:
L.e("Problem parsing start- or end date:", str(e))
raise e
else:
raise Exception("start- and enddate required for replay mode")
if ResourceManagement.args.pt:
from virtualisation.resourcemanagement.performancetestreceiver import PerformanceMeterMinutes
performancetest = PerformanceMeterMinutes() # PerformanceMeterSeconds()
for w in self.wrappers:
w.setReplayMode(True)
w.setClock(self.clock)
w.setTimeframe(startDate, endDate)
w.addReceiver(self)
if ResourceManagement.args.pt:
w.addReceiver(performancetest)
w.start()
w.runReplay()
if not self.args.noQuality:
if not self.averageStreamQuality:
self.averageStreamQuality = AverageStreamQuality(self, self.clock)
else:
self.averageStreamQuality.setClock(self.clock)
self.clock.runAsync()
self.startMonitor()
if not ResourceManagement.args.continuelive:
raw_input("press Enter to end.\n")
self.clock.stop()
L.i("Runtime", datetime.datetime.now() - start_time)
def update(self):
from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
# print "time", self.clock.now()
latStart = datetime.now()
L.d("processing:", self.getSensorDescription().sensorID)
# L.d(self.clock.now())
if self.replaymode:
self.stats.startMeasurement("Update_replay")
# self.clock.pause()
if self.historyreader:
L.d2("abstractwrapper get data")
self.stats.startMeasurement("Update_replay.Historyreader")
data_raw = self.historyreader.tick(self.clock)
self.stats.stopMeasurement("Update_replay.Historyreader")
L.d2("abstractwrapper received data:", str(data_raw))
if data_raw:
data_list = [data_raw] if not self.historyreader.multiple_observations else data_raw
for data in data_list:
try:
L.d2("abstractwrapper parse data")
# print "data to parse", data
self.stats.startMeasurement("Update_replay.Historyparser")
parsed = self.historyparser.parse(data, self.clock)
self.stats.stopMeasurement("Update_replay.Historyparser")
L.d2("abstractwrapper parsed data:", str(parsed))
del data
if parsed:
self.stats.startMeasurement("Update_replay.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = True
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_replay.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
L.d2("abstractwrapper get quality")
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
L.d2("abstractwrapper quality:", quality)
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_replay.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_replay.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
L.d2("abstractwrapper start receiver", r)
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
L.d2("abstractwrapper receiver", r, "finished")
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
if ResourceManagement.args.gentle:
self.clock.sleep()
else:
L.d("there is no data, ask fault recovery1")
# L.i(self.getSensorDescription().sensorID)
# L.i(self.clock.now())
try:
self.stats.startMeasurement("Update_replay.Recovery")
data = JSONObject()
data.latency = 0
data.producedInReplayMode = True
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
# at this point the dataType is in FAULT_RECOVERY_SUPPORTED_DATATYPES and we can safely use cast
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_replay.Recovery")
self.stats.startMeasurement("Update_replay.ObservationIDGenerator")
ObservationIDGenerator.addObservationIDToFields(data)
self.stats.stopMeasurement("Update_replay.ObservationIDGenerator")
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_replay.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_replay.Quality")
self.stats.startMeasurement("Update_replay.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_replay.Receiver")
except Exception as e:
L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
else:
pass # no history reader - nothing to do
self.stats.stopMeasurement("Update_replay")
else: # no replay mode
self.stats.startMeasurement("Update_live")
if self.connection:
try:
self.stats.startMeasurement("Update_live.Connection")
data_raw = self.connection.next()
self.stats.stopMeasurement("Update_live.Connection")
if data_raw:
data_list = [data_raw] if not self.connection.multiple_observations else data_raw
for data in data_list:
self.stats.startMeasurement("Update_live.Parser")
parsed = self.parser.parse(data, self.clock)
self.stats.stopMeasurement("Update_live.Parser")
if parsed:
self.stats.startMeasurement("Update_live.Preparation")
ObservationIDGenerator.addObservationIDToFields(parsed)
parsed.producedInReplayMode = False
parsed.recovered = False
parsed.latency = (datetime.now() - latStart).total_seconds()
self.stats.stopMeasurement("Update_live.Preparation")
# QoI Start
quality = None
if self.qoiSystem:
# TODO update the timestamp
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
if self.faultRecoveryActive:
L.d2("abstractwrapper update fault recovery")
self.stats.startMeasurement("Update_live.FaultRecoveryUpdate")
self.updateFaultRecoveries(parsed, quality)
self.stats.stopMeasurement("Update_live.FaultRecoveryUpdate")
L.d2("abstractwrapper fault recovery updated")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(parsed, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
else:
# fault recovery
L.i("there is no data, ask fault recovery2")
try:
self.stats.startMeasurement("Update_live.Recovery")
data = JSONObject()
data.latency = 0
data.recovered = True
data.fields = []
for n in self.getSensorDescription().fields:
if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
data.fields.append(n)
data[n] = JSONObject()
data[n].value = self.faultRecoveryCast(
self.faultRecoveries[n].getEstimation(),
self.getSensorDescription().field[n].dataType,
)
data[n].propertyName = self.getSensorDescription().field[n].propertyName
data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
if "unit" in self.getSensorDescription().field[n]:
data[n].unit = self.getSensorDescription().field[n].unit
data[n].sensorID = self.getSensorDescription().fullSensorID
data[n].observationSamplingTime = self.clock.timeAsString()
data[n].observationResultTime = data[n].observationSamplingTime
self.stats.stopMeasurement("Update_live.Recovery")
ObservationIDGenerator.addObservationIDToFields(data)
quality = None
if self.qoiSystem:
self.stats.startMeasurement("Update_live.Quality")
quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
self.stats.stopMeasurement("Update_live.Quality")
self.stats.startMeasurement("Update_live.Receiver")
for r in self.receiver:
r.receive(data, self.getSensorDescription(), self.clock, quality)
self.stats.stopMeasurement("Update_live.Receiver")
except Exception as e:
L.e(
"Error while updating sensor (fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
finally:
pass
# if ResourceManagement.args.gentle:
# self.clock.sleep()
except Exception as e:
L.e(
"Error while updating sensor (not fault recovery)",
self.getSensorDescription().fullSensorID,
str(e),
)
else:
pass # no live mode supported
self.stats.stopMeasurement("Update_live")
