def testWriteConsistency(self):
# txt
line1 = (100.0, 1, 1, 25.6)
self.tw.write(line1)
line2 = (101.0, 2, 2, 23.0)
self.tw.write(line2)
self.tw.close()
tr = tracetools.TraceReader(self.filename)
file_contents = [d for d in tr.parse()]
tr.close()
self.assertEqual(file_contents, [line1, line2])
# arff
self.tw = tracetools.TraceWriter("output.arff",
["timestamp", "mote_1", "mote_2"])
data = OrderedDict()
data["timestamp"] = 100.0
data["mote_1"] = 25.6
data["mote_2"] = 23.0
self.tw.write(data)
self.tw.close()
tr = tracetools.TraceReader("output.arff")
file_contents = [d for d in tr.parse()]
self.assertEqual(file_contents, [data])
def _testSupressRepetitions(self):
self.parser = tp.TraceReader("trace_test_repetitions.txt",
supress_repetitions=True)
output = [values for values in self.parser.parse()]
oracle = [(1341607696.752916,134,18381,11.218947), (1341607697.512910,225,0,24.259934), \
(1341607697.832944,225,1,29.670462)]
self.assertEqual(oracle, output)
def main(args):
filename = "/home/giulio/Dropbox/Projeto Sensores/experiments/temperatura/sala_servidores/readings_02_05_12_18h06m45s.arff"
tr = tracetools.TraceReader(filename=filename, supress_repetitions=False)
detected_anomaly = False
window_size = 20
k = 15
anomaly_threshold = 1.5
timestamps = []
mid = window_size / 2
try:
i = 0
temp_data = []
for data in tr.parse():
temps = [data[c] for c in data.iterkeys() if c != "timestamp"]
temp_data.append(numpy.array(temps))
timestamps.append(data["timestamp"])
i += 1
if i >= window_size:
lof = LOF(temp_data[mid], temp_data, k, euclidian_dist)
if not numpy.isnan(lof):
if detected_anomaly and lof < anomaly_threshold:
print "%f" % (timestamps[mid])
detected_anomaly = False
elif not detected_anomaly and lof > anomaly_threshold:
print "%f" % timestamps[mid],
detected_anomaly = True
del temp_data[0]
del timestamps[0]
finally:
tr.close()
def _testAutomaticInterpolation(self):
'''
Test automatic interpolation of missing data
'''
self.parser = tp.TraceReader("trace_test_missing_data.arff",
auto_interpolation=True)
output = [d['timestamp'] for d in self.parser.parse()]
pred_ts = output[0]
for val in output:
self.assertEqual(val, pred_ts)
pred_ts += 1
def testAutomaticTimestamps(self):
'''
Test automatic creation of timestamps on arff files that doesn't have them.
'''
self.parser = tp.TraceReader("trace_test_nots.arff",
auto_timestamps=True)
output = [tuple(d.values()) for d in self.parser.parse()]
oracle = [(1, 24.163262,20.421106,32.007574,24.550229,24.744001,21.663573,18.133233,39.550011,20.612037,23.680575), \
(2, 24.163262,20.421106,32.007574,24.550229,24.840963,21.663573,18.228512,39.550011,20.612037,23.680575), \
(3, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.228512,39.550011,20.612037,23.680575), \
(4, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.133233,39.550011,20.612037,23.680575), \
(5, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.133233,39.550011,20.612037,23.680575)]
self.assertEqual(output, oracle)
self.assertTrue("timestamp" in self.parser.arff_attributes)
def parse(filename,
motes,
selected_attr,
custom_func=None,
motes_to_ignore=[]):
tr = tracetools.TraceReader(filename, auto_timestamps=True, suppress_rapid_changes=True, \
motes_to_ignore=motes_to_ignore)
if custom_func and tr.file_type != "arff":
raise Exception("Expected arff file when using a custom_func")
if tr.file_type == "arff":
for attr in tr.arff_attributes:
if attr != "timestamp":
mote_id = attr
mote = Mote(mote_id)
motes[mote_id] = mote
for line in tr.parse():
timestamp = line['timestamp']
for mote_id, v in line.items()[1:]:
motes[mote_id].add_reading(timestamp, v)
else:
# agg -> txt
if tr.file_type == "agg":
adapter = tracetools.TraceAdapter(tr, tracetools.RAW_FILE_FORMAT,
selected_attr)
parser_gen = adapter.parse()
else:
parser_gen = tr.parse()
for timestamp, mote_id, _counter, value in parser_gen:
mote = motes.get(mote_id, None)
if not mote:
mote = Mote(mote_id)
motes[mote_id] = mote
mote.add_reading(timestamp, value)
# make all keys integers
for key in motes.keys():
original_key = key
if isinstance(key, str):
tokens = key.split("_")
if len(tokens) > 1:
key = int(tokens[1])
else:
key = int(key)
motes[key] = motes[original_key]
del motes[original_key]
def testParseArffFile(self):
# test arff file
self.parser = tp.TraceReader("trace_test.arff")
gen = self.parser.parse()
output = [values for values in gen]
self.parser.close()
oracle = []
f = open("trace_test.arff")
try:
for data in parse_arff_file(iter(f)):
oracle.append(data)
finally:
f.close()
self.assertEqual(len(oracle), len(output))
self.assertEqual(oracle, output)
def detect_anomalies(trace_filename, detector, motes_to_ignore=tuple()):
'''
@return: summarized, anomaly_levels
'''
anomaly_levels = []
summarized_series = []
sp = spirit.Spirit(l=1., k=1, fixed_k=True)
tr = tracetools.TraceReader(trace_filename,
motes_to_ignore=motes_to_ignore)
trace_start = -1
for data in tr.read():
if trace_start < 0:
trace_start = data['timestamp']
data = np.matrix([v for k, v in data.iteritems()
if k != "timestamp"]).T
k, w, Y = sp.update(data)
summ_Y = Y[0, 0]
anomaly_level = detector.apply(summ_Y)
anomaly_levels.append(anomaly_level)
summarized_series.append(summ_Y)
return summarized_series, anomaly_levels, trace_start
def testRawAggregateFile(self):
self.parser = tp.TraceReader("trace_test.agg")
gen = self.parser.parse()
output = [values for values in gen]
self.parser.close()
oracle = [ (1349364575.434680,237, 0, 27.086772, 76, 3.124826),\
(1349364575.503515,237, 0, 27.086772, 76, 3.124826),\
(1349364575.630907,248, 0, 32.316614, 515, 3.242182),\
(1349364575.745322,239, 0, 26.399825, 44, 3.116148),\
(1349364575.970389,245, 0, 26.301951, 498, 3.186947),\
(1349364576.040562,216, 0, 31.597115, 523, 3.168949),\
(1349364576.173389,216, 0, 31.597115, 523, 3.168949),\
(1349364577.237405,219, 0, 40.350718, 534, 3.124826),\
(1349364577.314838,217, 0, 36.876537, 290, 3.023758),\
(1349364577.383776,252, 0, 42.937803, 275, 3.040149),\
(1349364577.452631,233, 0, 47.707470, 194, 3.124826),\
(1349364577.547215,233, 0, 47.707470, 194, 3.124826),\
(1349364578.059417,233, 0, 47.707470, 194, 3.124826),\
]
self.assertEqual(oracle, output)
def testSupressRapidChanges(self):
'''
Ommit a change to the value of a mote if it changes too fast.
'''
self.parser = tp.TraceReader("test_rapid_changes.txt",
suppress_rapid_changes=True)
gen = self.parser.parse()
self.assertTrue(isinstance(gen, type((i for i in range(2)))))
# test txt file
output = [values for values in gen]
oracle = [(1341607696.752916, 134, 18381, 11.218947),
(1341607697.832944, 248, 1537, 29.670462), \
(1341607697.853437, 249, 1210, 27.086772), \
(1341607699.753355, 134, 18382, 11.511165), \
(1341607701.283434, 241, 1159, 25.132156), \
(1341607701.753172, 134, 18384, 10.142534), \
(1341607702.712892, 245, 1379, 24.937975), \
(1341607702.753301, 134, 18385, 10.730671), \
(1341607703.753879, 134, 18386, 11.316414), \
]
self.assertEqual(oracle, output)
def main(args):
if len(args) < 2:
print "Usage: python %s {trace.arff} -p N -w N -u N" % (args[0])
exit()
# parameters
plevel = 90
history_window = 7200
update_interval = 3600
# temps_06_07_12_18h38m00s, temps_16_08_12_16h37m01s
motes_inlet = (253, 249, 245, 239, 241, 237)
motes_inlet = map(str, motes_inlet)
filename = args[1]
optlist, arguments = getopt.getopt(args[2:], "f:p:w:u:")
for opt, val in optlist:
if opt == "-p":
plevel = int(val)
elif opt == '-w':
history_window = int(val)
elif opt == '-u':
update_interval = int(update_interval)
else:
raise Exception("Unrecognized option '%s'" % opt)
print "plevel =", plevel
print "history_window =", history_window
print "update_interval =", update_interval
motes_to_ignore = []
errors = []
rf = ReadingFreq(max_readings=history_window)
outfile = open("percentile.txt", "w")
next_update = 0
tr = tracetools.TraceReader(filename, auto_interpolation=False, motes_to_ignore=motes_to_ignore)
p = 0
count = 0
for data in tr.parse():
# get all readings
readings = []
for key, val in data.items():
if key != "timestamp":
for m in motes_inlet:
if key.endswith(m):
readings.append(val)
break
max_val = max(readings)
rf.update(max_val)
timestamp = data["timestamp"]
if next_update < timestamp:
p = rf.percentile(level=plevel)
next_update = timestamp + update_interval
if rf.full:
avg_val = float(sum(readings)) / len(readings)
min_val = min(readings)
outfile.write("%s %s %s %s %s\n" % (data["timestamp"], min_val, avg_val, max_val, p))
errors.append(max_val - p)
count += 1
outfile.close()
# calc error
positive_error = float(sum(x ** 2 for x in errors if x > 0)) / count
negative_error = float(sum(x ** 2 for x in errors if x < 0)) / count
print "MSE positive:", positive_error
print "MSE negative:", negative_error
def setUp(self):
self.parser = tp.TraceReader("trace_test.agg")
attribute = tp.TEMP
self.adapter = tp.TraceAdapter(self.parser, tp.RAW_FILE_FORMAT,
attribute)
def setUp(self):
self.parser = tp.TraceReader("trace_test.txt")
self.adapter = tp.TraceAdapter(self.parser,
out_format=tp.UNIFORM_FILE_FORMAT)
def setUp(self):
self.parser = tp.TraceReader("trace_test.txt")