Пример #1
0
    def parse(self, data, clock):
        if not data:  # nothing received or nothing in the history -> nothing to parse
            return None

        if self.lenNames > len(data):
            raise Exception("CSVParser: not enough data fields")

        result = JOb()
        sd = self.wrapper.getSensorDescription()
        result.fields = sd.fields
        # print data
        for i in range(0, self.lenNames):
            n = self.names[i]
            if n in sd.fields:
                result[n] = JOb()
                result[n].propertyName = sd.field[n].propertyName
                result[n].propertyURI = sd.field[n].propertyURI
                if "unit" in sd.field[n]:
                    result[n].unit = sd.field[n].unit
                result[n].observationSamplingTime = clock.timeAsString()
                result[n].sensorID = sd.fullSensorID
                if self.timestampcell >= 0:
                    result[n].observationResultTime = sd.parseTimestamp(data[self.timestampcell]).strftime(AbstractClock.format)
                else:
                    result[n].observationResultTime = result[n].observationSamplingTime

                if sd.field[n].dataType == "datetime.datetime":
                    result[n].value = dt.datetime.strptime(data[i], sd.field[n].format)
                else:
                    result[n].value = str2Type(u(data[i]), sd.field[n].dataType)

        return result
Пример #2
0
    def parse(self, data, clock):
        if not data:  # nothing received or nothing in the history -> nothing to parse
            return None

        if isinstance(data, str) or isinstance(data, unicode):
            data = JOb(u(data))

        result = JOb()
        sd = self.wrapper.getSensorDescription()
        result.fields = sd.fields
        for fieldname in sd.fields:
            result[fieldname] = JOb()
            result[fieldname].propertyName = sd.field[fieldname].propertyName
            result[fieldname].propertyURI = sd.field[fieldname].propertyURI
            if "unit" in sd.field[fieldname]:
                result[fieldname].unit = sd.field[fieldname].unit
            result[fieldname].sensorID = sd.fullSensorID
            result[fieldname].observationSamplingTime = clock.timeAsString()
            if self.timestampfield and self.timestampfield in data:
                result[fieldname].observationResultTime = sd.parseTimestamp(data[self.timestampfield]).strftime(
                    AbstractClock.format
                )
            else:
                result[fieldname].observationResultTime = result[fieldname].observationSamplingTime

            if sd.field[fieldname].dataType == "datetime.datetime":
                if sd.field[fieldname].format.startswith("UNIX"):
                    result[fieldname].value = sd.parseTimestamp(data[fieldname]) if fieldname in data else None
                else:
                    result[fieldname].value = (
                        dt.datetime.strptime(data[fieldname], sd.field[fieldname].format) if fieldname in data else None
                    )
            else:
                result[fieldname].value = (
                    str2Type(u(data[fieldname]), sd.field[fieldname].dataType) if fieldname in data else None
                )

        return result
    def update(self):
        from virtualisation.resourcemanagement.resourcemanagement import ResourceManagement

        # print "time", self.clock.now()
        latStart = datetime.now()
        L.d("processing:", self.getSensorDescription().sensorID)
        # L.d(self.clock.now())
        if self.replaymode:
            self.stats.startMeasurement("Update_replay")
            #             self.clock.pause()
            if self.historyreader:
                L.d2("abstractwrapper get data")
                self.stats.startMeasurement("Update_replay.Historyreader")
                data_raw = self.historyreader.tick(self.clock)
                self.stats.stopMeasurement("Update_replay.Historyreader")
                L.d2("abstractwrapper received data:", str(data_raw))
                if data_raw:
                    data_list = [data_raw] if not self.historyreader.multiple_observations else data_raw
                    for data in data_list:
                        try:
                            L.d2("abstractwrapper parse data")
                            # print "data to parse", data
                            self.stats.startMeasurement("Update_replay.Historyparser")
                            parsed = self.historyparser.parse(data, self.clock)
                            self.stats.stopMeasurement("Update_replay.Historyparser")
                            L.d2("abstractwrapper parsed data:", str(parsed))
                            del data
                            if parsed:
                                self.stats.startMeasurement("Update_replay.Preparation")
                                ObservationIDGenerator.addObservationIDToFields(parsed)
                                parsed.producedInReplayMode = True
                                parsed.recovered = False
                                parsed.latency = (datetime.now() - latStart).total_seconds()
                                self.stats.stopMeasurement("Update_replay.Preparation")

                                # QoI Start
                                quality = None
                                if self.qoiSystem:
                                    L.d2("abstractwrapper get quality")
                                    self.stats.startMeasurement("Update_replay.Quality")
                                    quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
                                    self.stats.stopMeasurement("Update_replay.Quality")
                                    L.d2("abstractwrapper quality:", quality)
                                if self.faultRecoveryActive:
                                    L.d2("abstractwrapper update fault recovery")
                                    self.stats.startMeasurement("Update_replay.FaultRecoveryUpdate")
                                    self.updateFaultRecoveries(parsed, quality)
                                    self.stats.stopMeasurement("Update_replay.FaultRecoveryUpdate")
                                    L.d2("abstractwrapper fault recovery updated")

                                self.stats.startMeasurement("Update_replay.Receiver")
                                for r in self.receiver:
                                    L.d2("abstractwrapper start receiver", r)
                                    r.receive(parsed, self.getSensorDescription(), self.clock, quality)
                                    L.d2("abstractwrapper receiver", r, "finished")
                                self.stats.stopMeasurement("Update_replay.Receiver")
                        except Exception as e:
                            L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
                        finally:
                            if ResourceManagement.args.gentle:
                                self.clock.sleep()
                else:
                    L.d("there is no data, ask fault recovery1")
                    # L.i(self.getSensorDescription().sensorID)
                    # L.i(self.clock.now())
                    try:
                        self.stats.startMeasurement("Update_replay.Recovery")
                        data = JSONObject()
                        data.latency = 0
                        data.producedInReplayMode = True
                        data.recovered = True

                        data.fields = []
                        for n in self.getSensorDescription().fields:
                            if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
                                data.fields.append(n)
                                data[n] = JSONObject()
                                # at this point the dataType is in FAULT_RECOVERY_SUPPORTED_DATATYPES and we can safely use cast
                                data[n].value = self.faultRecoveryCast(
                                    self.faultRecoveries[n].getEstimation(),
                                    self.getSensorDescription().field[n].dataType,
                                )
                                data[n].propertyName = self.getSensorDescription().field[n].propertyName
                                data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
                                if "unit" in self.getSensorDescription().field[n]:
                                    data[n].unit = self.getSensorDescription().field[n].unit
                                data[n].sensorID = self.getSensorDescription().fullSensorID
                                data[n].observationSamplingTime = self.clock.timeAsString()
                                data[n].observationResultTime = data[n].observationSamplingTime
                        self.stats.stopMeasurement("Update_replay.Recovery")

                        self.stats.startMeasurement("Update_replay.ObservationIDGenerator")
                        ObservationIDGenerator.addObservationIDToFields(data)
                        self.stats.stopMeasurement("Update_replay.ObservationIDGenerator")

                        quality = None
                        if self.qoiSystem:
                            self.stats.startMeasurement("Update_replay.Quality")
                            quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
                            self.stats.stopMeasurement("Update_replay.Quality")

                        self.stats.startMeasurement("Update_replay.Receiver")
                        for r in self.receiver:
                            r.receive(data, self.getSensorDescription(), self.clock, quality)
                        self.stats.stopMeasurement("Update_replay.Receiver")
                    except Exception as e:
                        L.e("Error while updating sensor", self.getSensorDescription().fullSensorID, e)
                    finally:
                        pass
                        # if ResourceManagement.args.gentle:
                        #     self.clock.sleep()
            else:
                pass  # no history reader - nothing to do
            self.stats.stopMeasurement("Update_replay")
        else:  # no replay mode
            self.stats.startMeasurement("Update_live")
            if self.connection:
                try:
                    self.stats.startMeasurement("Update_live.Connection")
                    data_raw = self.connection.next()
                    self.stats.stopMeasurement("Update_live.Connection")
                    if data_raw:
                        data_list = [data_raw] if not self.connection.multiple_observations else data_raw
                        for data in data_list:
                            self.stats.startMeasurement("Update_live.Parser")
                            parsed = self.parser.parse(data, self.clock)
                            self.stats.stopMeasurement("Update_live.Parser")
                            if parsed:
                                self.stats.startMeasurement("Update_live.Preparation")
                                ObservationIDGenerator.addObservationIDToFields(parsed)
                                parsed.producedInReplayMode = False
                                parsed.recovered = False
                                parsed.latency = (datetime.now() - latStart).total_seconds()
                                self.stats.stopMeasurement("Update_live.Preparation")

                                # QoI Start
                                quality = None
                                if self.qoiSystem:
                                    # TODO update the timestamp
                                    self.stats.startMeasurement("Update_live.Quality")
                                    quality = self.qoiSystem.addData(self.getSensorDescription(), parsed, self.clock)
                                    self.stats.stopMeasurement("Update_live.Quality")
                                if self.faultRecoveryActive:
                                    L.d2("abstractwrapper update fault recovery")
                                    self.stats.startMeasurement("Update_live.FaultRecoveryUpdate")
                                    self.updateFaultRecoveries(parsed, quality)
                                    self.stats.stopMeasurement("Update_live.FaultRecoveryUpdate")
                                    L.d2("abstractwrapper fault recovery updated")

                                self.stats.startMeasurement("Update_live.Receiver")
                                for r in self.receiver:
                                    r.receive(parsed, self.getSensorDescription(), self.clock, quality)
                                self.stats.stopMeasurement("Update_live.Receiver")
                    else:
                        # fault recovery
                        L.i("there is no data, ask fault recovery2")
                        try:
                            self.stats.startMeasurement("Update_live.Recovery")
                            data = JSONObject()
                            data.latency = 0
                            data.recovered = True
                            data.fields = []
                            for n in self.getSensorDescription().fields:
                                if n in self.faultRecoveries and self.faultRecoveries[n].isReady():
                                    data.fields.append(n)
                                    data[n] = JSONObject()
                                    data[n].value = self.faultRecoveryCast(
                                        self.faultRecoveries[n].getEstimation(),
                                        self.getSensorDescription().field[n].dataType,
                                    )
                                    data[n].propertyName = self.getSensorDescription().field[n].propertyName
                                    data[n].propertyURI = self.getSensorDescription().field[n].propertyURI
                                    if "unit" in self.getSensorDescription().field[n]:
                                        data[n].unit = self.getSensorDescription().field[n].unit
                                    data[n].sensorID = self.getSensorDescription().fullSensorID
                                    data[n].observationSamplingTime = self.clock.timeAsString()
                                    data[n].observationResultTime = data[n].observationSamplingTime
                            self.stats.stopMeasurement("Update_live.Recovery")

                            ObservationIDGenerator.addObservationIDToFields(data)
                            quality = None
                            if self.qoiSystem:
                                self.stats.startMeasurement("Update_live.Quality")
                                quality = self.qoiSystem.addData(self.getSensorDescription(), data, self.clock)
                                self.stats.stopMeasurement("Update_live.Quality")

                            self.stats.startMeasurement("Update_live.Receiver")
                            for r in self.receiver:
                                r.receive(data, self.getSensorDescription(), self.clock, quality)
                            self.stats.stopMeasurement("Update_live.Receiver")
                        except Exception as e:
                            L.e(
                                "Error while updating sensor (fault recovery)",
                                self.getSensorDescription().fullSensorID,
                                str(e),
                            )
                        finally:
                            pass
                            # if ResourceManagement.args.gentle:
                            #     self.clock.sleep()
                except Exception as e:
                    L.e(
                        "Error while updating sensor (not fault recovery)",
                        self.getSensorDescription().fullSensorID,
                        str(e),
                    )
            else:
                pass  # no live mode supported
            self.stats.stopMeasurement("Update_live")