def testAsScatter(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) s1 = sensor('name') s1.add(1.0, d1) s1.add(2.1, d2) s1.add(3.1, d3) s2 = sensor('name') s2.add(0.5, d1) s2.add(1, d2) s2.add(3.1, d3) s3 = sensor('name') s3.add(6, d1) s3.add(8, d2) s3.add(5, d3) sc1 = s1.asScatter() assert list(sc1.x) == [d1, d2, d3] assert list(sc1.y) == [1.0, 2.1, 3.1] assert sc1.name == "name" assert sc1.yaxis == "y" sc2 = s1.asScatter('xname', s2, s3, 'y2') assert list(sc2.x) == [d1, d2, d3] assert list(sc2.y) == [1.0, 2.1, 3.1] assert sc2.name == "xname" assert sc2.yaxis == "y2" assert list(sc2.error_y.array) == [5.0, 5.9, 1.9] assert list(sc2.error_y.arrayminus) == [0.5, 1.1, 0]
def testHash(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert hash(s1) == hash('name1') assert hash(s2) == hash('name2') assert hash(s3) == hash('name2')
def testEq(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert (s1 == s2) == False assert s2 == s3 assert (s2 == s1) == False assert s3 == s2 assert (s1 == 'x') == False
def testLt(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert s1 < s2 assert (s2 < s3) == False assert (s2 < s1) == False assert (s3 < s2) == False assert (s1 < 'x') == False
def testLe(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert s1 <= s2 assert s2 <= s3 assert (s2 <= s1) == False assert s3 <= s2 assert (s1 <= 'x') == False
def testGt(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert (s1 > s2) == False assert (s2 > s3) == False assert s2 > s1 assert (s3 > s2) == False assert (s1 > 'x') == False
def testGe(): s1 = sensor('name1') s2 = sensor('name2') s3 = sensor('name2', unit='RH%') assert (s1 >= s2) == False assert s2 >= s3 assert s2 >= s1 assert s3 >= s2 assert (s1 >= 'x') == False
def testSensorInitOnlyMandataParameter(): s = sensor('name') assert s.name == 'name' assert s.unit == 'V' assert s.pos == 1 assert len(s.values) == 0 assert s.clazz == 'Sensor'
def testSensorInitExplicit(): s = sensor('name', 'RH%', 2, {'x': 'y'}, 'c') assert s.name == 'name' assert s.unit == 'RH%' assert s.pos == 2 assert s.values == {'x': 'y'} assert s.clazz == 'c'
def __computePointRosee(self): for msn, ms in list(self.__metasensors.items()): if len(ms) < 2: continue tl = [ s for s in ms.values() if s.unit == '°C' and s.clazz == 'Sensor' ] if len(tl) != 1: continue rhl = [ s for s in ms.values() if s.unit == 'RH%' and s.clazz == 'Sensor' ] if len(rhl) != 1: continue logger.info( "Post processing data : computing point de rosee for %s", msn) t = tl[0] rh = rhl[0] gs = [g for g, v in self.__groups.items() if t in v or rh in v] pt = { d: t.values[d] - (100 - v) / 5 for d, v in rh.values.items() if d in t.values } n = msn + ' - Point de rosée' s = sensor(n, "°C", -1, pt, 'Point de rosée') self.__sensors[n] = s self.__metasensors[msn][n] = s for g in gs: self.__groups[g].append(s)
def testAdd(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) s = sensor('name') assert len(s.values) == 0 s.add(1.0, d1) assert len(s.values) == 1 assert d1 in s assert s.values[d1] == 1.0
def testSub(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) s1 = sensor('name') s1.add(1.0, d1) s1.add(2.1, d2) s1.add(3.1, d3) s1.add(4.1, d4) s2 = sensor('name') s2.add(0.5, d1) s2.add(1, d2) s2.add(5, d3) s3 = s1 - s2 assert s3.values[d1] == approx(0.5) assert s3.values[d2] == approx(1.1) assert s3.values[d3] == approx(-1.9) with raises(ValueError): s1 - 1
def testItem(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) s = sensor('name') s.add(1.0, d1) s.add(2.1, d2) s.add(3.1, d3) s.add(4.1, d4) assert s[d1] == 1.0
def __computeDistribution(self): groups = self.__groups.keys() units = set(map(lambda v: v.unit, self.__sensors.values())) clazzs = set(map(lambda v: v.clazz, self.__sensors.values())) for g in groups: for u in units: for c in clazzs: tsensor = self.sensorsByFunctionInGroup( lambda this: this.unit == u and this.clazz == c, g) if len(tsensor) > 1: logger.info( "Post processing data : compute distribution for %s > %s > %s", g, u, c) if g not in self.__metasensors: self.__metasensors[g] = {} tvalues = {} for sn in tsensor: for dt, v in sn.values.items(): if (dt not in tvalues): tvalues[dt] = [] tvalues[dt].append(v) target = { 'p-Variance': statistics.pvariance, 'Mean': statistics.mean, 'Median': statistics.median, 'Min': min, 'Max': max } mn = '{} - {} [{}]'.format(g, c, u) self.__metasensors[mn] = {s.name: s for s in tsensor} for s in tsensor: self.__metasensors[g][s.name] = s for name, method in target.items(): logger.info( "Post processing data : compute distribution for %s > %s > %s > %s", g, u, c, name) n = '{} - {} [{}] / {}'.format(g, c, u, name) s = sensor( n, u, -1, {d: method(v) for d, v in tvalues.items()}, c + "->" + name) self.__groups[g].append(s) self.__sensors[n] = s self.__metasensors[mn][n] = s self.__metasensors[g][n] = s
def testIter(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) s = sensor('name') s.add(1.0, d1) s.add(2.1, d2) s.add(3.1, d3) s.add(4.1, d4) nt = [] for x in sorted(s): nt.append(x) assert nt == [d1, d2, d3, d4]
def testMergeValueBy(): m = sensor.dateTimeToMinute() d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) d5 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:00', '%d.%m.%Y %H:%M:%S')) s = sensor('name') s.add(1.2, d1) s.add(1.8, d2) s.add(2.4, d3) s.add(2.5, d4) s.mergeValueBy(m) assert s.values == {d1: 1.8, d5: 2.5}
def testToBaseLine(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) s = sensor('name') s.add(1.0, d1) s.add(2.1, d2) s.add(3.1, d3) s.add(4.1, d4) s2 = s.toBaseLine() assert s2.values[d1] == approx(1) assert s2.values[d2] == approx(2.1) assert s2.values[d3] == approx(3.1) assert s2.values[d4] == approx(4.1)
def testLen(): d1 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) d2 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S')) d3 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S')) d4 = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S')) s = sensor('name') assert len(s) == 0 s.add(1.0, d1) assert len(s) == 1 s.add(2.1, d2) assert len(s) == 2 s.add(3.1, d3) assert len(s) == 3 s.add(4.1, d4) assert len(s) == 4
def testStr(): s = sensor('name', 'RH%', 2, {'x': 'y'}, 'c') assert str(s) == "name:\tunit:RH%\tposition:2\tvalues count:1\tclazz:c"
def __init__(self, filename, mergeFunction=sensor.dateTimeToMinute(), groupFunction=lambda n: 'default', metaFunction=lambda n: {'def': {k: v for k, v in n.items()}}, filterOutFunction=None): self.__sensors = {} self.__groups = {} sensorByPos = {} dtlimit = timezone('Europe/Zurich').localize( datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S')) dtzone = timezone('Europe/Zurich') try: s = 0 f = open(filename, 'r') for n, l in enumerate(f, 1): if n % 100000 == 0: logger.info("Processing line:{}".format(n)) if s == 0 and l.startswith('Sensors:'): s = 1 continue if s == 1 and l.startswith('Column'): s = 2 continue if s == 2 and l.startswith('Start date'): s = 3 continue if s == 3 and l.startswith('Time'): s = 4 continue if s == 2 and l != '\n': ls = l.rstrip('\n').split('\t') n = ls[1] cs = sensor(n, ls[2], ls[0]) sensorByPos[int(ls[0])] = cs self.__sensors[cs.name] = cs if s == 4 and l != '\n': ls = l.rstrip('\n').split('\t') dt_obj = dtzone.localize(sensors.__fastparsedate(ls[0])) if dtlimit > dt_obj: continue for p, v in enumerate(ls[1:]): if v != '': self.__sensors[sensorByPos[p + 1].name].add( float(v), dt_obj) finally: f.close() if filterOutFunction != None: logger.info("Filter sensor by using %s", filterOutFunction) self.__sensors = { n: v for n, v in self.__sensors.items() if not filterOutFunction(v) } logger.info("Create group sensor by using %s", groupFunction) for s in self.__sensors.values(): gname = groupFunction(s.name) if gname not in self.__groups: self.__groups[gname] = [] self.__groups[gname].append(s) logger.info("Create meta sensor by using %s", metaFunction) self.__metasensors = metaFunction(self.__sensors) logger.info("Post processing data : merging by %s", mergeFunction) self.__mergeValueBy(mergeFunction) logger.info("Post processing data : computing point de rosee") self.__computePointRosee() logger.info("Post processing data : computing distribution") self.__computeDistribution() logger.info("Post processing data : computing baseline") self.__computeBaseLine()