def test_indicators_threshold(self):
"""Test getting the on-off indicators with a threshold."""
ts = TimeSeries()
ts.array.resize(3)
ts.array[0] = (np.uint32(1), np.float32(0.0))
ts.array[1] = (np.uint32(2), np.float32(1.0))
ts.array[2] = (np.uint32(3), np.float32(2.0))
indicators_test = np.array([False, False, True])
indicators = ts.indicators(np.float32(1.1))
self.assertItemsEqual(indicators, indicators_test)
def test_creation(self):
"""Test TimeSeries creation from a file."""
ts_file = TimeSeries(path=DATA_PATH)
ts_test = TimeSeries()
ts_test.array.resize(3)
ts_test.array[0] = (np.uint32(1), np.float32(0.0))
ts_test.array[1] = (np.uint32(2), np.float32(1.0))
ts_test.array[2] = (np.uint32(3), np.float32(0.0))
self.assertItemsEqual(ts_file.times, ts_test.times)
self.assertItemsEqual(ts_file.powers, ts_test.powers)
def test_addition_no_intersect(self):
""" Test adding two TimeSeries with different timestamps."""
ts1 = TimeSeries()
ts1.array.resize(3)
ts1.array[0] = (np.uint32(1), np.float32(0.5))
ts1.array[1] = (np.uint32(2), np.float32(1.0))
ts1.array[2] = (np.uint32(4), np.float32(2.0))
ts2 = TimeSeries()
ts2.array.resize(3)
ts2.array[0] = (np.uint32(1), np.float32(0.0))
ts2.array[1] = (np.uint32(3), np.float32(1.0))
ts2.array[2] = (np.uint32(5), np.float32(3.0))
ts_test = TimeSeries()
ts_test.array.resize(1)
ts_test.array[0] = (np.uint32(1), np.float32(0.5))
ts_add = ts1 + ts2
self.assertItemsEqual(ts_add.powers, ts_test.powers)
def test_subtraction_no_intersect(self):
"""
Test subtracting one TimeSeries from another with different timestamps.
"""
ts1 = TimeSeries()
ts1.array.resize(3)
ts1.array[0] = (np.uint32(1), np.float32(0.5))
ts1.array[1] = (np.uint32(2), np.float32(1.0))
ts1.array[2] = (np.uint32(4), np.float32(3.0))
ts2 = TimeSeries()
ts2.array.resize(3)
ts2.array[0] = (np.uint32(1), np.float32(0.0))
ts2.array[1] = (np.uint32(3), np.float32(1.0))
ts2.array[2] = (np.uint32(5), np.float32(2.0))
ts_test = TimeSeries()
ts_test.array.resize(1)
ts_test.array[0] = (np.uint32(1), np.float32(0.5))
ts_diff = ts1 - ts2
self.assertItemsEqual(ts_diff.powers, ts_test.powers)
def test_f_score_perfect(self):
"""Test F1 score evaluation with no false positives or negatives."""
ts_test = TimeSeries()
ts_test.array.resize(5)
ts_test.array[0] = (np.uint32(1), np.float32(0.0))
ts_test.array[1] = (np.uint32(2), np.float32(1.0))
ts_test.array[2] = (np.uint32(3), np.float32(1.0))
ts_test.array[3] = (np.uint32(4), np.float32(1.0))
ts_test.array[4] = (np.uint32(5), np.float32(0.0))
score = f_score(ts_test, ts_test, threshold=np.float32(0.5))
self.assertEqual(score, 1)
def setUp(self):
"""Set up the data needed for the confidence estimator."""
self.aggregate = TimeSeries()
self.aggregate.array.resize(3)
self.aggregate.array[0] = (np.uint32(1), np.float32(5.0))
self.aggregate.array[1] = (np.uint32(2), np.float32(2.0))
self.aggregate.array[2] = (np.uint32(3), np.float32(4.0))
device1 = TimeSeries('a')
device1.array.resize(3)
device1.array[0] = (np.uint32(1), np.float32(5.0))
device1.array[1] = (np.uint32(2), np.float32(5.0))
device1.array[2] = (np.uint32(3), np.float32(5.0))
device2 = TimeSeries('b')
device2.array.resize(3)
device2.array[0] = (np.uint32(1), np.float32(0.0))
device2.array[1] = (np.uint32(2), np.float32(5.0))
device2.array[2] = (np.uint32(3), np.float32(0.0))
self.devices = [device1, device2]
def test_addition(self):
"""Test adding two TimeSeries together."""
ts1 = TimeSeries()
ts1.array.resize(3)
ts1.array[0] = (np.uint32(1), np.float32(0.5))
ts1.array[1] = (np.uint32(2), np.float32(1.0))
ts1.array[2] = (np.uint32(3), np.float32(2.0))
ts2 = TimeSeries()
ts2.array.resize(3)
ts2.array[0] = (np.uint32(1), np.float32(0.0))
ts2.array[1] = (np.uint32(2), np.float32(1.0))
ts2.array[2] = (np.uint32(3), np.float32(3.0))
ts_test = TimeSeries()
ts_test.array.resize(3)
ts_test.array[0] = (np.uint32(1), np.float32(0.5))
ts_test.array[1] = (np.uint32(2), np.float32(2.0))
ts_test.array[2] = (np.uint32(3), np.float32(5.0))
ts_add = ts1 + ts2
self.assertItemsEqual(ts_add.powers, ts_test.powers)
def test_activations(self):
"""
Test retrieving the TimeSeries activation indices.
"""
ts = TimeSeries()
ts.array.resize(11)
ts.array[0] = (np.uint32(1), np.float32(1.0))
ts.array[1] = (np.uint32(2), np.float32(1.0))
ts.array[2] = (np.uint32(3), np.float32(3.0))
ts.array[3] = (np.uint32(4), np.float32(3.0))
ts.array[4] = (np.uint32(5), np.float32(0.0))
ts.array[5] = (np.uint32(6), np.float32(1.0))
ts.array[6] = (np.uint32(7), np.float32(1.0))
ts.array[7] = (np.uint32(8), np.float32(0.0))
ts.array[8] = (np.uint32(9), np.float32(3.0))
ts.array[9] = (np.uint32(10), np.float32(0.0))
ts.array[10] = (np.uint32(10), np.float32(1.0))
test = [(0, 4), (5, 7), (8, 9), (10, 11)]
activations = ts.activations(np.float32(0.5))
self.assertItemsEqual(activations, test)
def test_padding_gap(self):
"""
Test padding a TimeSeries with a large gap.
"""
ts_missing = TimeSeries()
ts_missing.array.resize(3)
ts_missing.array[0] = (np.uint32(1), np.float32(1.0))
ts_missing.array[1] = (np.uint32(3), np.float32(3.0))
ts_missing.array[2] = (np.uint32(8), np.float32(0.0))
ts_test = TimeSeries()
ts_test.array.resize(4)
ts_test.array[0] = (np.uint32(1), np.float32(1.0))
ts_test.array[1] = (np.uint32(2), np.float32(1.0))
ts_test.array[2] = (np.uint32(3), np.float32(3.0))
ts_test.array[3] = (np.uint32(8), np.float32(0.0))
ts_missing.pad(3)
self.assertItemsEqual(ts_missing.powers, ts_test.powers)
def test_padding(self):
"""
Test padding a TimeSeries with missing values.
"""
ts_missing = TimeSeries()
ts_missing.array.resize(3)
ts_missing.array[0] = (np.uint32(1), np.float32(1.0))
ts_missing.array[1] = (np.uint32(3), np.float32(3.0))
ts_missing.array[2] = (np.uint32(5), np.float32(0.0))
ts_test = TimeSeries()
ts_test.array.resize(5)
ts_test.array[0] = (np.uint32(1), np.float32(1.0))
ts_test.array[1] = (np.uint32(2), np.float32(1.0))
ts_test.array[2] = (np.uint32(3), np.float32(3.0))
ts_test.array[3] = (np.uint32(4), np.float32(3.0))
ts_test.array[4] = (np.uint32(5), np.float32(0.0))
ts_missing.pad(5)
self.assertItemsEqual(ts_missing.powers, ts_test.powers)
def main():
parser = get_parser()
args = parser.parse_args()
logging.basicConfig(filename=args.log, level=logging.DEBUG)
log = logging.getLogger(__name__)
device_files = [os.path.abspath(os.path.join(args.dir, p)) for p in
os.listdir(args.dir)]
for dev_path in args.devices:
try:
device_files.remove(os.path.abspath(dev_path))
dev_data = TimeSeries(os.path.basename(dev_path).split('.')[0],
path=os.path.abspath(dev_path))
dev_data.pad(600)
dev_data.write(os.path.join(args.out, dev_data.name + '.dat'))
except ValueError:
log.error('Unable to find device file %s under %s' % (dev_path,
args.dir))
for agg_path in args.aggregated:
try:
device_files.remove(os.path.abspath(agg_path))
except ValueError:
log.warning('Unable to find aggregated power file %s under %s' %
(agg_path, args.dir))
agg_data = TimeSeries(path=os.path.abspath(args.aggregated[0]))
agg_data.pad(600)
for agg_path in args.aggregated[1:]:
agg_in = TimeSeries(path=os.path.abspath(agg_path))
agg_in.pad(600)
agg_data += agg_in
for dev_path in device_files:
dev_data = TimeSeries(path=dev_path)
dev_data.pad(600)
agg_data -= dev_data
agg_data.write(os.path.join(args.out, 'aggregate.dat'))
return 0
def main():
parser = get_parser()
args = parser.parse_args()
format = '%(asctime)s %(message)s'
logging.basicConfig(filename=args.log, level=logging.DEBUG, format=format)
device_files = [os.path.abspath(os.path.join(args.dir, p)) for p in
os.listdir(args.dir)]
agg_path = os.path.abspath(args.aggregated)
if agg_path in device_files:
device_files.remove(agg_path)
agg_data = TimeSeries(path=agg_path)
agg_data.array = agg_data.array[0:len(agg_data.array)/5*4]
device_in = []
for dev_path in device_files:
dev = TimeSeries(os.path.basename(dev_path).split('.')[0],
path=dev_path)
dev.intersect(agg_data)
device_in.append(dev)
apply_preprocess(agg_data.powers, device_in, args.preprocess,
np.float32(25.00))
for dev in device_in:
log.info('Training: %s' % dev.name)
activations = dev.activations(np.float32(25.0))
log.info('Activations:')
for a in activations:
log.info('From %s to %s lasting %s' % (dev.times[a[0]],
dev.times[a[1]-1],
a[1] - a[0]))
if len(activations) == 0:
log.info('No activations found.')
continue
window_size = sum([a[1] - a[0] for a in activations])/len(activations)
length = min(len(dev.array), len(agg_data.array))
log.info('Window size: %s' % window_size)
log.info('Series length: %s' % length)
agg_windows = []
dev_windows = []
window_size = min(1500, window_size)
window_size = max(8, window_size)
log.info('Computing windows...')
std_dev = np.std(np.random.choice(agg_data.powers, 10000))
max_power = dev.powers.max()
log.info('Std Dev: %s' % std_dev)
log.info('Max Power: %s' % max_power)
for i in xrange(0, length - window_size + 1, window_size):
if agg_data.times[i+window_size-1] - agg_data.times[i] > window_size:
log.info('Throwing out from %s to %s' %
(agg_data.times[i], agg_data.times[i+window_size-1]))
continue
dev_window = np.divide(dev.powers[i+3:i+window_size-3], max_power)
agg_window = agg_data.powers[i:i+window_size]
mean = agg_window.mean(axis=None)
agg_window = np.divide(np.subtract(agg_window, mean), std_dev)
dev_windows.append(dev_window)
agg_windows.append(agg_window)
agg_windows = np.array(agg_windows)
dev_windows = np.array(dev_windows)
agg_windows.shape = (len(agg_windows), window_size, 1)
dev_windows.shape = (len(dev_windows), window_size - 6, 1)
log.info('Training network...')
network = DenoisingAutoencoder(window_size)
network.train(agg_windows, dev_windows)
log.info('Saving model to: %s' % os.path.join(args.out,
dev.name + '.yml'))
network.save_model(os.path.join(args.out, dev.name + '.yml'))
log.info('Saving weight to: %s' % os.path.join(args.out,
dev.name + '.h5'))
network.save_weights(os.path.join(args.out, dev.name + '.h5'))
return 0
