def testWriteConsistency(self): # txt line1 = (100.0, 1, 1, 25.6) self.tw.write(line1) line2 = (101.0, 2, 2, 23.0) self.tw.write(line2) self.tw.close() tr = tracetools.TraceReader(self.filename) file_contents = [d for d in tr.parse()] tr.close() self.assertEqual(file_contents, [line1, line2]) # arff self.tw = tracetools.TraceWriter("output.arff", ["timestamp", "mote_1", "mote_2"]) data = OrderedDict() data["timestamp"] = 100.0 data["mote_1"] = 25.6 data["mote_2"] = 23.0 self.tw.write(data) self.tw.close() tr = tracetools.TraceReader("output.arff") file_contents = [d for d in tr.parse()] self.assertEqual(file_contents, [data])
def _testSupressRepetitions(self): self.parser = tp.TraceReader("trace_test_repetitions.txt", supress_repetitions=True) output = [values for values in self.parser.parse()] oracle = [(1341607696.752916,134,18381,11.218947), (1341607697.512910,225,0,24.259934), \ (1341607697.832944,225,1,29.670462)] self.assertEqual(oracle, output)
def main(args): filename = "/home/giulio/Dropbox/Projeto Sensores/experiments/temperatura/sala_servidores/readings_02_05_12_18h06m45s.arff" tr = tracetools.TraceReader(filename=filename, supress_repetitions=False) detected_anomaly = False window_size = 20 k = 15 anomaly_threshold = 1.5 timestamps = [] mid = window_size / 2 try: i = 0 temp_data = [] for data in tr.parse(): temps = [data[c] for c in data.iterkeys() if c != "timestamp"] temp_data.append(numpy.array(temps)) timestamps.append(data["timestamp"]) i += 1 if i >= window_size: lof = LOF(temp_data[mid], temp_data, k, euclidian_dist) if not numpy.isnan(lof): if detected_anomaly and lof < anomaly_threshold: print "%f" % (timestamps[mid]) detected_anomaly = False elif not detected_anomaly and lof > anomaly_threshold: print "%f" % timestamps[mid], detected_anomaly = True del temp_data[0] del timestamps[0] finally: tr.close()
def _testAutomaticInterpolation(self): ''' Test automatic interpolation of missing data ''' self.parser = tp.TraceReader("trace_test_missing_data.arff", auto_interpolation=True) output = [d['timestamp'] for d in self.parser.parse()] pred_ts = output[0] for val in output: self.assertEqual(val, pred_ts) pred_ts += 1
def testAutomaticTimestamps(self): ''' Test automatic creation of timestamps on arff files that doesn't have them. ''' self.parser = tp.TraceReader("trace_test_nots.arff", auto_timestamps=True) output = [tuple(d.values()) for d in self.parser.parse()] oracle = [(1, 24.163262,20.421106,32.007574,24.550229,24.744001,21.663573,18.133233,39.550011,20.612037,23.680575), \ (2, 24.163262,20.421106,32.007574,24.550229,24.840963,21.663573,18.228512,39.550011,20.612037,23.680575), \ (3, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.228512,39.550011,20.612037,23.680575), \ (4, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.133233,39.550011,20.612037,23.680575), \ (5, 24.163262,20.421106,32.007574,24.550229,24.840963,21.759307,18.133233,39.550011,20.612037,23.680575)] self.assertEqual(output, oracle) self.assertTrue("timestamp" in self.parser.arff_attributes)
def parse(filename, motes, selected_attr, custom_func=None, motes_to_ignore=[]): tr = tracetools.TraceReader(filename, auto_timestamps=True, suppress_rapid_changes=True, \ motes_to_ignore=motes_to_ignore) if custom_func and tr.file_type != "arff": raise Exception("Expected arff file when using a custom_func") if tr.file_type == "arff": for attr in tr.arff_attributes: if attr != "timestamp": mote_id = attr mote = Mote(mote_id) motes[mote_id] = mote for line in tr.parse(): timestamp = line['timestamp'] for mote_id, v in line.items()[1:]: motes[mote_id].add_reading(timestamp, v) else: # agg -> txt if tr.file_type == "agg": adapter = tracetools.TraceAdapter(tr, tracetools.RAW_FILE_FORMAT, selected_attr) parser_gen = adapter.parse() else: parser_gen = tr.parse() for timestamp, mote_id, _counter, value in parser_gen: mote = motes.get(mote_id, None) if not mote: mote = Mote(mote_id) motes[mote_id] = mote mote.add_reading(timestamp, value) # make all keys integers for key in motes.keys(): original_key = key if isinstance(key, str): tokens = key.split("_") if len(tokens) > 1: key = int(tokens[1]) else: key = int(key) motes[key] = motes[original_key] del motes[original_key]
def testParseArffFile(self): # test arff file self.parser = tp.TraceReader("trace_test.arff") gen = self.parser.parse() output = [values for values in gen] self.parser.close() oracle = [] f = open("trace_test.arff") try: for data in parse_arff_file(iter(f)): oracle.append(data) finally: f.close() self.assertEqual(len(oracle), len(output)) self.assertEqual(oracle, output)
def detect_anomalies(trace_filename, detector, motes_to_ignore=tuple()): ''' @return: summarized, anomaly_levels ''' anomaly_levels = [] summarized_series = [] sp = spirit.Spirit(l=1., k=1, fixed_k=True) tr = tracetools.TraceReader(trace_filename, motes_to_ignore=motes_to_ignore) trace_start = -1 for data in tr.read(): if trace_start < 0: trace_start = data['timestamp'] data = np.matrix([v for k, v in data.iteritems() if k != "timestamp"]).T k, w, Y = sp.update(data) summ_Y = Y[0, 0] anomaly_level = detector.apply(summ_Y) anomaly_levels.append(anomaly_level) summarized_series.append(summ_Y) return summarized_series, anomaly_levels, trace_start
def testRawAggregateFile(self): self.parser = tp.TraceReader("trace_test.agg") gen = self.parser.parse() output = [values for values in gen] self.parser.close() oracle = [ (1349364575.434680,237, 0, 27.086772, 76, 3.124826),\ (1349364575.503515,237, 0, 27.086772, 76, 3.124826),\ (1349364575.630907,248, 0, 32.316614, 515, 3.242182),\ (1349364575.745322,239, 0, 26.399825, 44, 3.116148),\ (1349364575.970389,245, 0, 26.301951, 498, 3.186947),\ (1349364576.040562,216, 0, 31.597115, 523, 3.168949),\ (1349364576.173389,216, 0, 31.597115, 523, 3.168949),\ (1349364577.237405,219, 0, 40.350718, 534, 3.124826),\ (1349364577.314838,217, 0, 36.876537, 290, 3.023758),\ (1349364577.383776,252, 0, 42.937803, 275, 3.040149),\ (1349364577.452631,233, 0, 47.707470, 194, 3.124826),\ (1349364577.547215,233, 0, 47.707470, 194, 3.124826),\ (1349364578.059417,233, 0, 47.707470, 194, 3.124826),\ ] self.assertEqual(oracle, output)
def testSupressRapidChanges(self): ''' Ommit a change to the value of a mote if it changes too fast. ''' self.parser = tp.TraceReader("test_rapid_changes.txt", suppress_rapid_changes=True) gen = self.parser.parse() self.assertTrue(isinstance(gen, type((i for i in range(2))))) # test txt file output = [values for values in gen] oracle = [(1341607696.752916, 134, 18381, 11.218947), (1341607697.832944, 248, 1537, 29.670462), \ (1341607697.853437, 249, 1210, 27.086772), \ (1341607699.753355, 134, 18382, 11.511165), \ (1341607701.283434, 241, 1159, 25.132156), \ (1341607701.753172, 134, 18384, 10.142534), \ (1341607702.712892, 245, 1379, 24.937975), \ (1341607702.753301, 134, 18385, 10.730671), \ (1341607703.753879, 134, 18386, 11.316414), \ ] self.assertEqual(oracle, output)
def main(args): if len(args) < 2: print "Usage: python %s {trace.arff} -p N -w N -u N" % (args[0]) exit() # parameters plevel = 90 history_window = 7200 update_interval = 3600 # temps_06_07_12_18h38m00s, temps_16_08_12_16h37m01s motes_inlet = (253, 249, 245, 239, 241, 237) motes_inlet = map(str, motes_inlet) filename = args[1] optlist, arguments = getopt.getopt(args[2:], "f:p:w:u:") for opt, val in optlist: if opt == "-p": plevel = int(val) elif opt == '-w': history_window = int(val) elif opt == '-u': update_interval = int(update_interval) else: raise Exception("Unrecognized option '%s'" % opt) print "plevel =", plevel print "history_window =", history_window print "update_interval =", update_interval motes_to_ignore = [] errors = [] rf = ReadingFreq(max_readings=history_window) outfile = open("percentile.txt", "w") next_update = 0 tr = tracetools.TraceReader(filename, auto_interpolation=False, motes_to_ignore=motes_to_ignore) p = 0 count = 0 for data in tr.parse(): # get all readings readings = [] for key, val in data.items(): if key != "timestamp": for m in motes_inlet: if key.endswith(m): readings.append(val) break max_val = max(readings) rf.update(max_val) timestamp = data["timestamp"] if next_update < timestamp: p = rf.percentile(level=plevel) next_update = timestamp + update_interval if rf.full: avg_val = float(sum(readings)) / len(readings) min_val = min(readings) outfile.write("%s %s %s %s %s\n" % (data["timestamp"], min_val, avg_val, max_val, p)) errors.append(max_val - p) count += 1 outfile.close() # calc error positive_error = float(sum(x ** 2 for x in errors if x > 0)) / count negative_error = float(sum(x ** 2 for x in errors if x < 0)) / count print "MSE positive:", positive_error print "MSE negative:", negative_error
def setUp(self): self.parser = tp.TraceReader("trace_test.agg") attribute = tp.TEMP self.adapter = tp.TraceAdapter(self.parser, tp.RAW_FILE_FORMAT, attribute)
def setUp(self): self.parser = tp.TraceReader("trace_test.txt") self.adapter = tp.TraceAdapter(self.parser, out_format=tp.UNIFORM_FILE_FORMAT)
def setUp(self): self.parser = tp.TraceReader("trace_test.txt")