def testAsScatter():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
s1 = sensor('name')
s1.add(1.0, d1)
s1.add(2.1, d2)
s1.add(3.1, d3)
s2 = sensor('name')
s2.add(0.5, d1)
s2.add(1, d2)
s2.add(3.1, d3)
s3 = sensor('name')
s3.add(6, d1)
s3.add(8, d2)
s3.add(5, d3)
sc1 = s1.asScatter()
assert list(sc1.x) == [d1, d2, d3]
assert list(sc1.y) == [1.0, 2.1, 3.1]
assert sc1.name == "name"
assert sc1.yaxis == "y"
sc2 = s1.asScatter('xname', s2, s3, 'y2')
assert list(sc2.x) == [d1, d2, d3]
assert list(sc2.y) == [1.0, 2.1, 3.1]
assert sc2.name == "xname"
assert sc2.yaxis == "y2"
assert list(sc2.error_y.array) == [5.0, 5.9, 1.9]
assert list(sc2.error_y.arrayminus) == [0.5, 1.1, 0]
def testHash():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert hash(s1) == hash('name1')
assert hash(s2) == hash('name2')
assert hash(s3) == hash('name2')
def testEq():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert (s1 == s2) == False
assert s2 == s3
assert (s2 == s1) == False
assert s3 == s2
assert (s1 == 'x') == False
def testLt():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert s1 < s2
assert (s2 < s3) == False
assert (s2 < s1) == False
assert (s3 < s2) == False
assert (s1 < 'x') == False
def testLe():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert s1 <= s2
assert s2 <= s3
assert (s2 <= s1) == False
assert s3 <= s2
assert (s1 <= 'x') == False
def testGt():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert (s1 > s2) == False
assert (s2 > s3) == False
assert s2 > s1
assert (s3 > s2) == False
assert (s1 > 'x') == False
def testGe():
s1 = sensor('name1')
s2 = sensor('name2')
s3 = sensor('name2', unit='RH%')
assert (s1 >= s2) == False
assert s2 >= s3
assert s2 >= s1
assert s3 >= s2
assert (s1 >= 'x') == False
def testSensorInitOnlyMandataParameter():
s = sensor('name')
assert s.name == 'name'
assert s.unit == 'V'
assert s.pos == 1
assert len(s.values) == 0
assert s.clazz == 'Sensor'
def testSensorInitExplicit():
s = sensor('name', 'RH%', 2, {'x': 'y'}, 'c')
assert s.name == 'name'
assert s.unit == 'RH%'
assert s.pos == 2
assert s.values == {'x': 'y'}
assert s.clazz == 'c'
def __computePointRosee(self):
for msn, ms in list(self.__metasensors.items()):
if len(ms) < 2:
continue
tl = [
s for s in ms.values()
if s.unit == '°C' and s.clazz == 'Sensor'
]
if len(tl) != 1:
continue
rhl = [
s for s in ms.values()
if s.unit == 'RH%' and s.clazz == 'Sensor'
]
if len(rhl) != 1:
continue
logger.info(
"Post processing data : computing point de rosee for %s", msn)
t = tl[0]
rh = rhl[0]
gs = [g for g, v in self.__groups.items() if t in v or rh in v]
pt = {
d: t.values[d] - (100 - v) / 5
for d, v in rh.values.items() if d in t.values
}
n = msn + ' - Point de rosée'
s = sensor(n, "°C", -1, pt, 'Point de rosée')
self.__sensors[n] = s
self.__metasensors[msn][n] = s
for g in gs:
self.__groups[g].append(s)
def testAdd():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
assert len(s.values) == 0
s.add(1.0, d1)
assert len(s.values) == 1
assert d1 in s
assert s.values[d1] == 1.0
def testSub():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
s1 = sensor('name')
s1.add(1.0, d1)
s1.add(2.1, d2)
s1.add(3.1, d3)
s1.add(4.1, d4)
s2 = sensor('name')
s2.add(0.5, d1)
s2.add(1, d2)
s2.add(5, d3)
s3 = s1 - s2
assert s3.values[d1] == approx(0.5)
assert s3.values[d2] == approx(1.1)
assert s3.values[d3] == approx(-1.9)
with raises(ValueError):
s1 - 1
def testItem():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
s.add(1.0, d1)
s.add(2.1, d2)
s.add(3.1, d3)
s.add(4.1, d4)
assert s[d1] == 1.0
def __computeDistribution(self):
groups = self.__groups.keys()
units = set(map(lambda v: v.unit, self.__sensors.values()))
clazzs = set(map(lambda v: v.clazz, self.__sensors.values()))
for g in groups:
for u in units:
for c in clazzs:
tsensor = self.sensorsByFunctionInGroup(
lambda this: this.unit == u and this.clazz == c, g)
if len(tsensor) > 1:
logger.info(
"Post processing data : compute distribution for %s > %s > %s",
g, u, c)
if g not in self.__metasensors:
self.__metasensors[g] = {}
tvalues = {}
for sn in tsensor:
for dt, v in sn.values.items():
if (dt not in tvalues):
tvalues[dt] = []
tvalues[dt].append(v)
target = {
'p-Variance': statistics.pvariance,
'Mean': statistics.mean,
'Median': statistics.median,
'Min': min,
'Max': max
}
mn = '{} - {} [{}]'.format(g, c, u)
self.__metasensors[mn] = {s.name: s for s in tsensor}
for s in tsensor:
self.__metasensors[g][s.name] = s
for name, method in target.items():
logger.info(
"Post processing data : compute distribution for %s > %s > %s > %s",
g, u, c, name)
n = '{} - {} [{}] / {}'.format(g, c, u, name)
s = sensor(
n, u, -1,
{d: method(v)
for d, v in tvalues.items()}, c + "->" + name)
self.__groups[g].append(s)
self.__sensors[n] = s
self.__metasensors[mn][n] = s
self.__metasensors[g][n] = s
def testIter():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
s.add(1.0, d1)
s.add(2.1, d2)
s.add(3.1, d3)
s.add(4.1, d4)
nt = []
for x in sorted(s):
nt.append(x)
assert nt == [d1, d2, d3, d4]
def testMergeValueBy():
m = sensor.dateTimeToMinute()
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
d5 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:00', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
s.add(1.2, d1)
s.add(1.8, d2)
s.add(2.4, d3)
s.add(2.5, d4)
s.mergeValueBy(m)
assert s.values == {d1: 1.8, d5: 2.5}
def testToBaseLine():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
s.add(1.0, d1)
s.add(2.1, d2)
s.add(3.1, d3)
s.add(4.1, d4)
s2 = s.toBaseLine()
assert s2.values[d1] == approx(1)
assert s2.values[d2] == approx(2.1)
assert s2.values[d3] == approx(3.1)
assert s2.values[d4] == approx(4.1)
def testLen():
d1 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
d2 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:01', '%d.%m.%Y %H:%M:%S'))
d3 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:02', '%d.%m.%Y %H:%M:%S'))
d4 = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:01:01', '%d.%m.%Y %H:%M:%S'))
s = sensor('name')
assert len(s) == 0
s.add(1.0, d1)
assert len(s) == 1
s.add(2.1, d2)
assert len(s) == 2
s.add(3.1, d3)
assert len(s) == 3
s.add(4.1, d4)
assert len(s) == 4
def testStr():
s = sensor('name', 'RH%', 2, {'x': 'y'}, 'c')
assert str(s) == "name:\tunit:RH%\tposition:2\tvalues count:1\tclazz:c"
def __init__(self,
filename,
mergeFunction=sensor.dateTimeToMinute(),
groupFunction=lambda n: 'default',
metaFunction=lambda n: {'def': {k: v
for k, v in n.items()}},
filterOutFunction=None):
self.__sensors = {}
self.__groups = {}
sensorByPos = {}
dtlimit = timezone('Europe/Zurich').localize(
datetime.strptime('01.01.2017 00:00:00', '%d.%m.%Y %H:%M:%S'))
dtzone = timezone('Europe/Zurich')
try:
s = 0
f = open(filename, 'r')
for n, l in enumerate(f, 1):
if n % 100000 == 0:
logger.info("Processing line:{}".format(n))
if s == 0 and l.startswith('Sensors:'):
s = 1
continue
if s == 1 and l.startswith('Column'):
s = 2
continue
if s == 2 and l.startswith('Start date'):
s = 3
continue
if s == 3 and l.startswith('Time'):
s = 4
continue
if s == 2 and l != '\n':
ls = l.rstrip('\n').split('\t')
n = ls[1]
cs = sensor(n, ls[2], ls[0])
sensorByPos[int(ls[0])] = cs
self.__sensors[cs.name] = cs
if s == 4 and l != '\n':
ls = l.rstrip('\n').split('\t')
dt_obj = dtzone.localize(sensors.__fastparsedate(ls[0]))
if dtlimit > dt_obj:
continue
for p, v in enumerate(ls[1:]):
if v != '':
self.__sensors[sensorByPos[p + 1].name].add(
float(v), dt_obj)
finally:
f.close()
if filterOutFunction != None:
logger.info("Filter sensor by using %s", filterOutFunction)
self.__sensors = {
n: v
for n, v in self.__sensors.items() if not filterOutFunction(v)
}
logger.info("Create group sensor by using %s", groupFunction)
for s in self.__sensors.values():
gname = groupFunction(s.name)
if gname not in self.__groups:
self.__groups[gname] = []
self.__groups[gname].append(s)
logger.info("Create meta sensor by using %s", metaFunction)
self.__metasensors = metaFunction(self.__sensors)
logger.info("Post processing data : merging by %s", mergeFunction)
self.__mergeValueBy(mergeFunction)
logger.info("Post processing data : computing point de rosee")
self.__computePointRosee()
logger.info("Post processing data : computing distribution")
self.__computeDistribution()
logger.info("Post processing data : computing baseline")
self.__computeBaseLine()