def getQualityValues(self, jOb, name, avgQualityType):
     q = JSONObject()
     q.absoluteValues = self.qList.get(avgQualityType).absoluteBuffer.items
     q.ratedValues = self.qList.get(avgQualityType).ratedBuffer.items
     q.name = name
     q.unit = self.unit
     jOb.values.append(q)
Esempio n. 2
0
 def listwrapper(self):
     r = JOb()
     c = 0
     r.wrappers = []
     for w in self.rm.wrappers:
         sd = w.getSensorDescription()
         if isinstance(sd, list):
             for _sd in sd:
                 r.wrappers.append(JOb())
                 r.wrappers[c].information = _sd.information
                 r.wrappers[c].uuid = _sd.uuid
                 r.wrappers[c].sensorName = _sd.sensorName
                 r.wrappers[c].author = _sd.author
                 r.wrappers[c].sensorID = _sd.sensorID
                 r.wrappers[c].fullSensorID = _sd.fullSensorID
                 r.wrappers[c].source = _sd.source
                 r.wrappers[c].sensorType = _sd.sensorType
                 r.wrappers[c].sourceFormat = _sd.sourceFormat
                 r.wrappers[c].messagebus.routingKey = _sd.messagebus.routingKey
                 c += 1
         else:
             r.wrappers.append(JOb())
             r.wrappers[c].information = sd.information
             r.wrappers[c].uuid = sd.uuid
             r.wrappers[c].sensorName = sd.sensorName
             r.wrappers[c].author = sd.author
             r.wrappers[c].sensorID = sd.sensorID
             r.wrappers[c].fullSensorID = sd.fullSensorID
             r.wrappers[c].source = sd.source
             r.wrappers[c].sensorType = sd.sensorType
             r.wrappers[c].sourceFormat = sd.sourceFormat
             r.wrappers[c].messagebus.routingKey = sd.messagebus.routingKey
             c += 1
     return r.dumps()
Esempio n. 3
0
    def data_timeframe(self, uuid):
        """
        Returns the time frame (start and end date) for which data from the stream, identified by the UUID,
        is available.
        :param uuid:
        :return:
        """
        resp = JOb()
        resp.uuid = uuid
        try:
            if ThreadedTriplestoreAdapter.triplestore:
                sd = self.rm.getSensorDescriptionByUUID(uuid)
                if sd:
                    data = ThreadedTriplestoreAdapter.triplestore.getStreamMinMaxDate(sd.graphName, formatSensorID(sd))
                    resp.status = "Ok"
                    resp.message = ""
                    resp.data = data
                else:
                    raise Exception("no stream with given uuid known")
            else:
                resp.status = "Fail"
                resp.message = "Triplestore not activated."
        except Exception as e:
            resp.status = "Fail"
            resp.message = e.message

        return resp.dumps()
    def deploy(self, f, autostart=False):
        """

        :param f:
        :param autostart:
        :return: a tuple with 3 elements. 1. status as string, 2. error message as string, 3. list of uuids of added wrapper
        """
        L.i("Deploying", f)
        sensordescriptions = []
        try:
            zFile = zipfile.ZipFile(f)
            if "deploy.json" in zFile.namelist():
                deployDescription = JOb(zFile.open("deploy.json", "r"))
                sys.path.insert(0, f)
                if deployDescription.isList():
                    for dd in deployDescription:
                        module = __import__(dd.module)
                        wrapper = getattr(module, dd["class"])()
                        self.addWrapper(wrapper)
                        sensordescriptions.append(wrapper.getSensorDescription())
                        if autostart:
                            self.startWrapper(wrapper)
                else:
                    module = __import__(deployDescription.module)
                    wrapper = getattr(module, deployDescription["class"])()
                    self.addWrapper(wrapper)
                    sensordescriptions.append(wrapper.getSensorDescription())
                    if autostart:
                        self.startWrapper(wrapper)
            return "OK", "", sensordescriptions
        except Exception as e:
            L.w("Deployment of wrapper", f, "failed.", e.message)
            return "Fail", e.message, []
Esempio n. 5
0
 def listeventwrapper(self):
     r = JOb()
     r.eventwrapper = []
     eds = self.rm.getEventWrapperDescriptions()
     for ed in eds:
         r.eventwrapper.append(ed)
     return r.dumps()
 def valueMapToJson(self, test, joblist, job=None, parent=None):
     if not parent:
         keys = [key for key in test if "." not in key]
     else:
         parentString = parent + "."
         keys = [key for key in test if parentString in key]
         keys = [key for key in keys if "." not in key.split(parentString)[1]]
     
     for key in keys:
         job2 = JSONObject()
         job2.name = key.split(".")[len(key.split("."))-1]
         job2.value = test[key]
 
         if job:
             if job.values:
                 job.values.append(job2)
             else:
                 job.values = []
                 job.values.append(job2)
         else:
             joblist.append(job2)
     
         test.pop(key)
         if len(test) > 0:
             self.valueMapToJson(test, joblist, job2, key)    
 def getAvgQualities(self, currentDate):
     avgQualities = JSONObject()
     for t in self.avgTypes:
         avgQualities[t] = JSONObject()
         avgQualities[t].values = []
         avgQualities[t].startDate = str(self.getStartDate(currentDate, t))
     for key in self.qoiMetrics:
         self.qoiMetrics[key].getAvgQualities(avgQualities, key)
     return avgQualities
Esempio n. 8
0
 def finish(cls):
     jobs = JSONObject()
     jobs.stats = []
     for e in Stats.instances:
         job = JSONObject()
         job.name = e
         job.value = e.getvalue()
         jobs.stats.append(job)
     return jobs
Esempio n. 9
0
 def listwrapperfull(self):
     r = JOb()
     r.wrappers = []
     for w in self.rm.wrappers:
         sd = w.getSensorDescription()
         if isinstance(sd, list):
             for _sd in sd:
                 r.wrappers.append(_sd)
         else:
             r.wrappers.append(sd)
     return r.dumps()
Esempio n. 10
0
    def snapshot(self, uuid, start=None, end=None):
        """
        Get previous observations
        :param uuid: The uuid of the wrapper
        :param start: The start date in the format %Y-%m-%dT%H:%M:%S
        :param end: The end date in the format %Y-%m-%dT%H:%M:%S
        :return: a JSON answer
        """
        resp = JOb()
        resp.uuid = uuid
        try:
            from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
            if ResourceManagement.args.triplestore:
                sd = self.rm.getSensorDescriptionByUUID(uuid)
                if sd:
                    data = ThreadedTriplestoreAdapter.triplestore.getObservationGraph(sd.graphName, formatSensorID(sd), start, end, False)
                    resp.status = "Ok"
                    resp.message = ""
                    resp.data = data
                else:
                    raise Exception("no stream with given uuid known")
            else:
                raise Exception("Triplestore not enabled in Resource Management")

        except Exception as e:
            resp.status = "Fail"
            resp.message = e.message

        return resp.dumps()
Esempio n. 11
0
 def getAllLastQualities(self):
     from virtualisation.wrapper.abstractwrapper import AbstractWrapper, AbstractComposedWrapper
     qualities = []
     wrappers = self.rm.wrappers
     for wrapper in wrappers:
         if isinstance(wrapper, AbstractWrapper):
             if wrapper.qoiSystem.initialised:
                 qualities.append(wrapper.qoiSystem.getLastQoI())
                 qualities[-1].uuid = wrapper.getSensorDescription().uuid
             else:
                 resp = JOb()
                 resp.uuid = wrapper.getSensorDescription().uuid
                 resp.status = "Fail"
                 resp.message = "Quality System not initialised for given uuid"
                 qualities.append(resp)
         elif isinstance(wrapper, AbstractComposedWrapper):
             for aWrapper in wrapper.wrappers:
                 if aWrapper.qoiSystem.initialised:
                     qualities.append(aWrapper.qoiSystem.getLastQoI())
                     qualities[-1].uuid = aWrapper.getSensorDescription().uuid
                 else:
                     resp = JOb()
                     resp.uuid = aWrapper.getSensorDescription().uuid
                     resp.status = "Fail"
                     resp.message = "Quality System not initialised for given uuid"
                     qualities.append(resp)
     return JOb(qualities).dumps()
    def update(self, data):
        self.updateDescription()

        # special case when no fields are in data
        # (fault recovery is not ready yet)
        if len(data.fields) == 0:
            self.rewardAndPunishment.update(False)
            self.absoluteValue = float("inf")
            self.ratedValue = self.rewardAndPunishment.value()
            return

        ts = self.repsys.timestamp


        self.updatecounter += 1

        samplingTime = datetime.datetime.strptime(data[data.fields[0]].observationSamplingTime,
                                                  AbstractClock.parserformat)

        age = (samplingTime - ts).total_seconds()
#         print "age:", age, "ts", ts, "sampling", samplingTime

        if self.lastUpdate == None:
            self.lastUpdate = ts
            self.rewardAndPunishment.update(True)
            self.absoluteValue = age
            self.ratedValue = 1.0
            self.min = age
            self.mean = age
        else:
            # 			delta = ts - self.lastUpdate
            self.lastUpdate = ts
            if data.recovered:
                self.rewardAndPunishment.update(False)
            else:
                self.rewardAndPunishment.update(age <= self.annotatedAge)

            # 			delay = delta.days * 86400 + delta.seconds
            self.absoluteValue = age
            self.ratedValue = self.rewardAndPunishment.value()
            age = float(age)
            self.min = min(self.min, age)
            self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + age / self.updatecounter

        ageReturn = JSONObject()

        ageReturn.absoluteValue = self.absoluteValue
        ageReturn.ratedValue = self.ratedValue
        ageReturn.unit = self.unit
        return (self.name, ageReturn)
 def getAvgQualities(self, jOb, name):
     for key in self.qList:
         q = JSONObject()
         q.absoluteValue = self.qList.get(key).getAbsoluteAverage()
         q.absoluteValueMin = self.qList.get(key).getAbsoluteMin()
         q.absoluteValueMax = self.qList.get(key).getAbsoluteMax()
         q.ratedValue = self.qList.get(key).getRatedAverage()
         q.ratedValueMin = self.qList.get(key).getRatedMin()
         q.ratedValueMax = self.qList.get(key).getRatedMax()
         q.name = name
         q.unit = self.unit
         jOb[key].values.append(q)
Esempio n. 14
0
 def snapshot_sql(self, uuid, start=None, end=None, format='json', last=False, fields=None, offset=0):
     if self.rm.sql:
         offset = int(offset)
         observations = self.rm.sql.get_observations(uuid, start, end, format, last, fields, offset)
         data = JOb()
         data.data = observations
         #TODO                  remove the 2 below
         data.next_url = "/api/snapshot_sql2?uuid=%s%s%s%s%s%s%s" % (uuid,
               "&start=" + start if start else "",
               "&end=" + end if end else "" ,
               "&format=" + format,
               "&last=" + last if last else "",
               "&fields=" + fields if fields else "",
               "&offset=" + str(offset + self.rm.sql.PAGINATION_LIMIT)) if len(observations) == self.rm.sql.PAGINATION_LIMIT else ""
         return data.dumps()
     else:
         return "Error. SQL feature not enabled in Resource Management"
 def aggregate(self, data, sensordescription):
     result = []
     try:
         dftobjs = self.dftobjects[sensordescription.uuid]
         for f in dftobjs:
             g = dftobjs[f].control(data[f])
             if g:
                 r = JSONObject()
                 r.graph = g
                 r.sensorID = sensordescription.sensorID
                 r.propertyType = sensordescription.field[f].propertyName
                 r.category = sensordescription.sensorType
                 result.append(r)
         return result
     except KeyError:
         Log.e("Dft aggregation failed")
         return None
Esempio n. 16
0
 def get_description(self, uuid):
     resp = JOb()
     sd = self.rm.getSensorDescriptionByUUID(uuid)
     if sd:
         resp.status = "Ok"
         resp.message = ""
         resp.data = sd
     else:
         ed = self.rm.getEventDescriptionByUUID(uuid)
         if ed:
             resp.status = "Ok"
             resp.message = ""
             resp.data = ed
         else:
             resp.status = "Fail"
             resp.message = "no stream with given uuid known"
     return resp.dumps()
    def update(self, data):
        self.updateDescription()

        # special case when no fields are in data
        # (fault recovery is not ready yet)
        if len(data.fields) == 0:
            self.rewardAndPunishment.update(False)
            self.absoluteValue = float("inf")
            self.ratedValue = self.rewardAndPunishment.value()
            return

        ts = self.repsys.timestamp

        self.updatecounter += 1

        latency = data.latency

        annotatedLatency = self.repsys.description.maxLatency

        if self.lastUpdate is None:
            self.lastUpdate = ts
            self.rewardAndPunishment.update(True)
            self.absoluteValue = latency
            self.ratedValue = 1.0
            self.min = latency
            self.mean = latency
        else:
            self.lastUpdate = ts
            
            if data.recovered:
                self.rewardAndPunishment.update(False)
            else:
                self.rewardAndPunishment.update(annotatedLatency > latency)

            self.absoluteValue = latency
            self.ratedValue = self.rewardAndPunishment.value()
            self.min = min(self.min, latency)
            self.mean = ((self.updatecounter - 1) * self.mean) / self.updatecounter + latency / self.updatecounter

        lat = JSONObject()

        lat.absoluteValue = self.absoluteValue
        lat.ratedValue = self.ratedValue
        lat.unit = self.unit
        return (self.name, lat)
    def makeRandomEvent(self):
        eventData = JSONObject()
        eventData.ceID = 123456
        eventData.ceType = "Aarhus_Road_Traffic_Event"
        eventData.ceName = "traffic jam"
        eventData.ceTime = 1438591234000L
        eventData.ceCoordinate = "(56.12 10.13)"
        eventData.ceLevel = 1
        eventData.dummy = True

        return eventData
Esempio n. 19
0
 def get_static_stream_data(self, uuid):
     resp = JOb()
     resp.uuid = uuid
     if uuid in self.static_stream_data_cache:
         resp.status = "Ok"
         resp.message = ""
         resp.data = self.static_stream_data_cache[uuid]
     else:
         resp.status = "Fail"
         resp.message = "No sensor stream with uuid " + uuid + " known."
     return resp.dumps()
Esempio n. 20
0
 def avg_processing_time(self, uuid = None, category = None):
     job = JSONObject()
     if uuid:
         uuid = uuid.split(",")
         for u in uuid:
             stats = Stats.getOrMake(u)
             job[u] = []
             pTimes = stats.getAverageProcessingTimes()
             if pTimes:
                 job[u].extend(pTimes)
             else:
                 failJob = JSONObject()
                 failJob.status = "Fail"
                 failJob.message = "UUID not found"
                 job[u] = failJob
     elif category:
         category = category.split(",")
         avgList = {}
         for c in category:
             uuidList = self.rm.getUUIDsForCategory(c)
             statList = []
             valueMap={}
             for uuid in uuidList:
                 stats = Stats.getOrMake(uuid)
                 self.test(valueMap, stats.getAverageProcessingTimes())
                 statList += stats.getAverageProcessingTimes()
             for key, value in valueMap.items():
                 valueMap[key] = value / len(uuidList)
             
             avgList[c] = []
             self.valueMapToJson(valueMap, avgList[c], None, None)
         job = JSONObject()
         for element in avgList:
             job[element] = []
             pTimes = avgList[element]
             if pTimes:
                 job[element].extend(pTimes)
             else:
                 failJob = JSONObject()
                 failJob.status = "Fail"
                 failJob.message = "Category not found"
                 job[element] = failJob
     else:
         statList = Stats.getAllStats()
         for s in statList:
             job[s.name] = []
             job[s.name].extend(s.getAverageProcessingTimes())
     return job.dumps()
 def getValues(self, t, avg=None, minimum=None, maximum=None):
     q = JSONObject()
     if avg:
         q.absoluteAvg = self.qList.get(t).getAbsoluteAverage()
         q.ratedAvg = self.qList.get(t).getRatedAverage()
     if minimum:
         q.absoluteMin = self.qList.get(t).getAbsoluteMin()
         q.ratedMin = self.qList.get(t).getRatedMin()
     if maximum:
         q.absoluteMax = self.qList.get(t).getAbsoluteMax()
         q.ratedMax = self.qList.get(t).getRatedMax()
     return q
Esempio n. 22
0
    def data_timeframe_sql(self, uuid):
        """
        Returns the time frame (start and end date) for which data from the stream, identified by the UUID,
        is available.
        :param uuid:
        :return:
        """
        resp = JOb()
        resp.uuid = uuid
        try:
            if self.rm.sql:
                data = self.rm.sql.data_timeframe(uuid)
                resp.status = "Ok"
                resp.message = ""
                resp.data = data
            else:
                resp.status = "Fail"
                resp.message = "SQL feature not activated."
        except Exception as e:
            resp.status = "Fail"
            resp.message = e.message

        return resp.dumps()
 def aggregate(self, data, sensordescription):
     if not data.fields:
         # Log.e("There was no data available so it could not be aggregated")
         return None
     result = []
     try:
         saxobjs = self.aggregation_objs[sensordescription.uuid]
         for f in saxobjs:
             g = saxobjs[f].control(data[f])
             if g:
                 r = JSONObject()
                 r.graph = g
                 r.sensorID = sensordescription.sensorID
                 r.propertyType = sensordescription.field[f].propertyName
                 r.category = sensordescription.sensorType
                 result.append(r)
         del data
         if result is not None and len(result) is not 0:
             Log.d2('SaxAggregator: Aggregation successful %s' % str(result))
         return result
     except Exception as e:
         Log.e("aggregation failed due to Exception", e)
         return None
Esempio n. 24
0
    def getQualityValues(self, uuid=None, types=None, avg=None, minimum=None, maximum=None):
        if types:
            types = types.split(",")
        if uuid:
            uuid = uuid.split(",")
        else:   #get all uuids from wrapper list
            from virtualisation.wrapper.abstractwrapper import AbstractWrapper, AbstractComposedWrapper
            uuid = []
            wrappers = self.rm.wrappers
            for wrapper in wrappers:
                if isinstance(wrapper, AbstractWrapper):
                    uuid.append(wrapper.getSensorDescription().uuid)
                elif isinstance(wrapper, AbstractComposedWrapper):
                    for aWrapper in wrapper.wrappers:
                        uuid.append(aWrapper.getSensorDescription().uuid)

        qualities = []
        for _uuid in uuid:
            wrapper = self.rm.getWrapperByUUID(_uuid)
            message = ""
            if wrapper:
                if wrapper.qoiSystem.initialised:
                    avgQualities = []
                    avgQualities.append(wrapper.qoiSystem.getLastQoI(types=types, avg=valueToBoolean(avg), minimum=valueToBoolean(minimum), maximum=valueToBoolean(maximum)))
                    avgQualities[-1].uuid = wrapper.getSensorDescription().uuid
                    qualities.extend(JOb(avgQualities))
                else:
                    message = "AVG Quality System not initialised for given uuid"
            else:
                message = "no stream with given uuid known"
            if message:
                resp = JOb()
                resp.uuid = _uuid
                resp.status = "Fail"
                resp.message = message
                qualities.append(resp)
        return JOb(qualities).dumps()
Esempio n. 25
0
 def snapshot_last(self, uuid):
     """
     Get previous observations
     :param uuid: The uuid of the wrapper
     :param start: The start date in the format %Y-%m-%dT%H:%M:%S
     :param end: The end date in the format %Y-%m-%dT%H:%M:%S
     :return: a JSON answer
     """
     resp = JOb()
     resp.uuid = uuid
     if uuid in self.observation_cache:
         resp.status = "Ok"
         resp.message = ""
         resp.data = self.observation_cache[uuid]
     else:
         resp.status = "Fail"
         resp.message = "No observation with the UUID " + uuid + " cached."
     return resp.dumps()
Esempio n. 26
0
 def getAverageProcessingTimes(self):
     job = JSONObject()
     job.name = self.name
     job.value = self.mean()
     if len(self.timeElementMap) > 0:
         job.values = []
     for element in self.timeElementMap:
         if len(self.timeElementMap[element].timeElementMap) > 0:
             job.values.append(self.timeElementMap[element].getAverageProcessingTimes())
         else:
             job2 = JSONObject()
             job2.name = element
             job2.value = self.timeElementMap[element].mean()
             job.values.append(job2)
     return job
 def getLastQoI(self, types=None, avg=None, minimum=None, maximum=None):
     qoiData = JSONObject()
     for metric in self.reputationSystem.metrics:
         q = JSONObject()
         currentValues = JSONObject()
         currentValues.absoluteValue = metric.absoluteValue
         currentValues.ratedValue = metric.ratedValue
         q.unit = metric.unit
         qoiData[metric.name] = q
         q.CURRENT = currentValues
         if types:
             self.reputationSystem.avgQoIManager.getValues(q, metric.name, types, avg, minimum, maximum)
     return qoiData
Esempio n. 28
0
 def register_event(self, eventdescription):
     resp = JOb()
     if eventdescription.filename.endswith(".json"):
         from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
         trg = os.path.join(ResourceManagement.eventdescriptionfoldername, eventdescription.filename)
         dst = open(trg, "w")
         dst.write(eventdescription.file.read())
         dst.close()
         try:
             resp.status, resp.message = self.rm.registerEvent(trg)
         except Exception as e:
             resp.status = "Fail"
             resp.message = e.message
     else:
         resp.status = "Fail"
         resp.message = "Wrong file type."
     return resp.dumps()
Esempio n. 29
0
 def deactivate_fault_recovery(self, uuid):
     """
     Disable the fault recovery for a wrapper
     :param uuid: UUID identifying the wrapper
     :return:
     """
     resp = JOb()
     resp.uuid = uuid
     w = self.rm.getWrapperByUUID(uuid)
     if w:
         w.deactivateFaultRecovery()
         resp.status = "Ok"
         resp.message = ""
     else:
         resp.status = "Fail"
         resp.message = "no stream with given uuid known"
     return resp.dumps()
Esempio n. 30
0
 def deploy(self, deployunit):
     resp = JOb()
     if deployunit.filename.endswith(".zip"):
         from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement
         trg = os.path.join(ResourceManagement.deployfoldername, deployunit.filename)
         dst = open(trg, "w")
         dst.write(deployunit.file.read())
         dst.close()
         try:
             resp.status, resp.message, resp.sensordescriptions = self.rm.deploy(trg, True)
         except Exception as e:
             resp.status = "Fail"
             resp.message = e.message
     else:
         resp.status = "Fail"
         resp.message = "Wrong file type."
     return resp.dumps()