def test_indicators_threshold(self): """Test getting the on-off indicators with a threshold.""" ts = TimeSeries() ts.array.resize(3) ts.array[0] = (np.uint32(1), np.float32(0.0)) ts.array[1] = (np.uint32(2), np.float32(1.0)) ts.array[2] = (np.uint32(3), np.float32(2.0)) indicators_test = np.array([False, False, True]) indicators = ts.indicators(np.float32(1.1)) self.assertItemsEqual(indicators, indicators_test)
def test_creation(self): """Test TimeSeries creation from a file.""" ts_file = TimeSeries(path=DATA_PATH) ts_test = TimeSeries() ts_test.array.resize(3) ts_test.array[0] = (np.uint32(1), np.float32(0.0)) ts_test.array[1] = (np.uint32(2), np.float32(1.0)) ts_test.array[2] = (np.uint32(3), np.float32(0.0)) self.assertItemsEqual(ts_file.times, ts_test.times) self.assertItemsEqual(ts_file.powers, ts_test.powers)
def test_addition_no_intersect(self): """ Test adding two TimeSeries with different timestamps.""" ts1 = TimeSeries() ts1.array.resize(3) ts1.array[0] = (np.uint32(1), np.float32(0.5)) ts1.array[1] = (np.uint32(2), np.float32(1.0)) ts1.array[2] = (np.uint32(4), np.float32(2.0)) ts2 = TimeSeries() ts2.array.resize(3) ts2.array[0] = (np.uint32(1), np.float32(0.0)) ts2.array[1] = (np.uint32(3), np.float32(1.0)) ts2.array[2] = (np.uint32(5), np.float32(3.0)) ts_test = TimeSeries() ts_test.array.resize(1) ts_test.array[0] = (np.uint32(1), np.float32(0.5)) ts_add = ts1 + ts2 self.assertItemsEqual(ts_add.powers, ts_test.powers)
def test_subtraction_no_intersect(self): """ Test subtracting one TimeSeries from another with different timestamps. """ ts1 = TimeSeries() ts1.array.resize(3) ts1.array[0] = (np.uint32(1), np.float32(0.5)) ts1.array[1] = (np.uint32(2), np.float32(1.0)) ts1.array[2] = (np.uint32(4), np.float32(3.0)) ts2 = TimeSeries() ts2.array.resize(3) ts2.array[0] = (np.uint32(1), np.float32(0.0)) ts2.array[1] = (np.uint32(3), np.float32(1.0)) ts2.array[2] = (np.uint32(5), np.float32(2.0)) ts_test = TimeSeries() ts_test.array.resize(1) ts_test.array[0] = (np.uint32(1), np.float32(0.5)) ts_diff = ts1 - ts2 self.assertItemsEqual(ts_diff.powers, ts_test.powers)
def test_f_score_perfect(self): """Test F1 score evaluation with no false positives or negatives.""" ts_test = TimeSeries() ts_test.array.resize(5) ts_test.array[0] = (np.uint32(1), np.float32(0.0)) ts_test.array[1] = (np.uint32(2), np.float32(1.0)) ts_test.array[2] = (np.uint32(3), np.float32(1.0)) ts_test.array[3] = (np.uint32(4), np.float32(1.0)) ts_test.array[4] = (np.uint32(5), np.float32(0.0)) score = f_score(ts_test, ts_test, threshold=np.float32(0.5)) self.assertEqual(score, 1)
def setUp(self): """Set up the data needed for the confidence estimator.""" self.aggregate = TimeSeries() self.aggregate.array.resize(3) self.aggregate.array[0] = (np.uint32(1), np.float32(5.0)) self.aggregate.array[1] = (np.uint32(2), np.float32(2.0)) self.aggregate.array[2] = (np.uint32(3), np.float32(4.0)) device1 = TimeSeries('a') device1.array.resize(3) device1.array[0] = (np.uint32(1), np.float32(5.0)) device1.array[1] = (np.uint32(2), np.float32(5.0)) device1.array[2] = (np.uint32(3), np.float32(5.0)) device2 = TimeSeries('b') device2.array.resize(3) device2.array[0] = (np.uint32(1), np.float32(0.0)) device2.array[1] = (np.uint32(2), np.float32(5.0)) device2.array[2] = (np.uint32(3), np.float32(0.0)) self.devices = [device1, device2]
def test_addition(self): """Test adding two TimeSeries together.""" ts1 = TimeSeries() ts1.array.resize(3) ts1.array[0] = (np.uint32(1), np.float32(0.5)) ts1.array[1] = (np.uint32(2), np.float32(1.0)) ts1.array[2] = (np.uint32(3), np.float32(2.0)) ts2 = TimeSeries() ts2.array.resize(3) ts2.array[0] = (np.uint32(1), np.float32(0.0)) ts2.array[1] = (np.uint32(2), np.float32(1.0)) ts2.array[2] = (np.uint32(3), np.float32(3.0)) ts_test = TimeSeries() ts_test.array.resize(3) ts_test.array[0] = (np.uint32(1), np.float32(0.5)) ts_test.array[1] = (np.uint32(2), np.float32(2.0)) ts_test.array[2] = (np.uint32(3), np.float32(5.0)) ts_add = ts1 + ts2 self.assertItemsEqual(ts_add.powers, ts_test.powers)
def test_activations(self): """ Test retrieving the TimeSeries activation indices. """ ts = TimeSeries() ts.array.resize(11) ts.array[0] = (np.uint32(1), np.float32(1.0)) ts.array[1] = (np.uint32(2), np.float32(1.0)) ts.array[2] = (np.uint32(3), np.float32(3.0)) ts.array[3] = (np.uint32(4), np.float32(3.0)) ts.array[4] = (np.uint32(5), np.float32(0.0)) ts.array[5] = (np.uint32(6), np.float32(1.0)) ts.array[6] = (np.uint32(7), np.float32(1.0)) ts.array[7] = (np.uint32(8), np.float32(0.0)) ts.array[8] = (np.uint32(9), np.float32(3.0)) ts.array[9] = (np.uint32(10), np.float32(0.0)) ts.array[10] = (np.uint32(10), np.float32(1.0)) test = [(0, 4), (5, 7), (8, 9), (10, 11)] activations = ts.activations(np.float32(0.5)) self.assertItemsEqual(activations, test)
def test_padding_gap(self): """ Test padding a TimeSeries with a large gap. """ ts_missing = TimeSeries() ts_missing.array.resize(3) ts_missing.array[0] = (np.uint32(1), np.float32(1.0)) ts_missing.array[1] = (np.uint32(3), np.float32(3.0)) ts_missing.array[2] = (np.uint32(8), np.float32(0.0)) ts_test = TimeSeries() ts_test.array.resize(4) ts_test.array[0] = (np.uint32(1), np.float32(1.0)) ts_test.array[1] = (np.uint32(2), np.float32(1.0)) ts_test.array[2] = (np.uint32(3), np.float32(3.0)) ts_test.array[3] = (np.uint32(8), np.float32(0.0)) ts_missing.pad(3) self.assertItemsEqual(ts_missing.powers, ts_test.powers)
def test_padding(self): """ Test padding a TimeSeries with missing values. """ ts_missing = TimeSeries() ts_missing.array.resize(3) ts_missing.array[0] = (np.uint32(1), np.float32(1.0)) ts_missing.array[1] = (np.uint32(3), np.float32(3.0)) ts_missing.array[2] = (np.uint32(5), np.float32(0.0)) ts_test = TimeSeries() ts_test.array.resize(5) ts_test.array[0] = (np.uint32(1), np.float32(1.0)) ts_test.array[1] = (np.uint32(2), np.float32(1.0)) ts_test.array[2] = (np.uint32(3), np.float32(3.0)) ts_test.array[3] = (np.uint32(4), np.float32(3.0)) ts_test.array[4] = (np.uint32(5), np.float32(0.0)) ts_missing.pad(5) self.assertItemsEqual(ts_missing.powers, ts_test.powers)
def main(): parser = get_parser() args = parser.parse_args() logging.basicConfig(filename=args.log, level=logging.DEBUG) log = logging.getLogger(__name__) device_files = [os.path.abspath(os.path.join(args.dir, p)) for p in os.listdir(args.dir)] for dev_path in args.devices: try: device_files.remove(os.path.abspath(dev_path)) dev_data = TimeSeries(os.path.basename(dev_path).split('.')[0], path=os.path.abspath(dev_path)) dev_data.pad(600) dev_data.write(os.path.join(args.out, dev_data.name + '.dat')) except ValueError: log.error('Unable to find device file %s under %s' % (dev_path, args.dir)) for agg_path in args.aggregated: try: device_files.remove(os.path.abspath(agg_path)) except ValueError: log.warning('Unable to find aggregated power file %s under %s' % (agg_path, args.dir)) agg_data = TimeSeries(path=os.path.abspath(args.aggregated[0])) agg_data.pad(600) for agg_path in args.aggregated[1:]: agg_in = TimeSeries(path=os.path.abspath(agg_path)) agg_in.pad(600) agg_data += agg_in for dev_path in device_files: dev_data = TimeSeries(path=dev_path) dev_data.pad(600) agg_data -= dev_data agg_data.write(os.path.join(args.out, 'aggregate.dat')) return 0
def main(): parser = get_parser() args = parser.parse_args() format = '%(asctime)s %(message)s' logging.basicConfig(filename=args.log, level=logging.DEBUG, format=format) device_files = [os.path.abspath(os.path.join(args.dir, p)) for p in os.listdir(args.dir)] agg_path = os.path.abspath(args.aggregated) if agg_path in device_files: device_files.remove(agg_path) agg_data = TimeSeries(path=agg_path) agg_data.array = agg_data.array[0:len(agg_data.array)/5*4] device_in = [] for dev_path in device_files: dev = TimeSeries(os.path.basename(dev_path).split('.')[0], path=dev_path) dev.intersect(agg_data) device_in.append(dev) apply_preprocess(agg_data.powers, device_in, args.preprocess, np.float32(25.00)) for dev in device_in: log.info('Training: %s' % dev.name) activations = dev.activations(np.float32(25.0)) log.info('Activations:') for a in activations: log.info('From %s to %s lasting %s' % (dev.times[a[0]], dev.times[a[1]-1], a[1] - a[0])) if len(activations) == 0: log.info('No activations found.') continue window_size = sum([a[1] - a[0] for a in activations])/len(activations) length = min(len(dev.array), len(agg_data.array)) log.info('Window size: %s' % window_size) log.info('Series length: %s' % length) agg_windows = [] dev_windows = [] window_size = min(1500, window_size) window_size = max(8, window_size) log.info('Computing windows...') std_dev = np.std(np.random.choice(agg_data.powers, 10000)) max_power = dev.powers.max() log.info('Std Dev: %s' % std_dev) log.info('Max Power: %s' % max_power) for i in xrange(0, length - window_size + 1, window_size): if agg_data.times[i+window_size-1] - agg_data.times[i] > window_size: log.info('Throwing out from %s to %s' % (agg_data.times[i], agg_data.times[i+window_size-1])) continue dev_window = np.divide(dev.powers[i+3:i+window_size-3], max_power) agg_window = agg_data.powers[i:i+window_size] mean = agg_window.mean(axis=None) agg_window = np.divide(np.subtract(agg_window, mean), std_dev) dev_windows.append(dev_window) agg_windows.append(agg_window) agg_windows = np.array(agg_windows) dev_windows = np.array(dev_windows) agg_windows.shape = (len(agg_windows), window_size, 1) dev_windows.shape = (len(dev_windows), window_size - 6, 1) log.info('Training network...') network = DenoisingAutoencoder(window_size) network.train(agg_windows, dev_windows) log.info('Saving model to: %s' % os.path.join(args.out, dev.name + '.yml')) network.save_model(os.path.join(args.out, dev.name + '.yml')) log.info('Saving weight to: %s' % os.path.join(args.out, dev.name + '.h5')) network.save_weights(os.path.join(args.out, dev.name + '.h5')) return 0