def setup_package(): """Nosetests package setup function (run when tests are done). See http://nose.readthedocs.org/en/latest/writing_tests.html#test-packages - Copies the original data files to be restored after testing. - Sets a non-interactive backend for matplotlib (even though we don't show any figures, it can import PyQt, for example). """ from nilmtk.tests.testingtools import data_dir # Create backups of the data files from glob import glob from os.path import join, isdir import os, shutil target_paths = glob(join(data_dir(), '*.h5')) target_paths.append(join(data_dir(), 'random_csv')) for original_fn in target_paths: target_fn = original_fn + '.original' if isdir(original_fn): shutil.copytree(original_fn, target_fn) else: shutil.copyfile(original_fn, target_fn) # Use the most basic Matplotlib backend import matplotlib matplotlib.use('AGG')
def test_total_energy(self): filename = join(data_dir(), 'random.h5') ds = DataSet(filename) ds.buildings[1].elec.total_energy() ds.buildings[1].elec.total_energy() # test cache ds.buildings[1].elec.clear_cache() ds.store.close()
def create_random_hdf5(): FILENAME = join(data_dir(), 'random.h5') N_METERS = 5 store = pd.HDFStore(FILENAME, 'w', complevel=9, complib='zlib') elec_meter_metadata = {} for meter in range(1, N_METERS+1): key = 'building1/elec/meter{:d}'.format(meter) print("Saving", key) store.put(key, create_random_df(), format='table') elec_meter_metadata[meter] = { 'device_model': TEST_METER['model'], 'submeter_of': 1, 'data_location': key } # Save dataset-wide metadata store.root._v_attrs.metadata = {'meter_devices': {TEST_METER['model']: TEST_METER}} print(store.root._v_attrs.metadata) # Building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def create_random_hdf5(): FILENAME = join(data_dir(), 'random.h5') N_METERS = 5 store = pd.HDFStore(FILENAME, 'w', complevel=9, complib='zlib') elec_meter_metadata = {} for meter in range(1, N_METERS + 1): key = 'building1/elec/meter{:d}'.format(meter) print("Saving", key) store.put(key, create_random_df(), format='table') elec_meter_metadata[meter] = { 'device_model': TEST_METER['model'], 'submeter_of': 1, 'data_location': key } # Save dataset-wide metadata store.root._v_attrs.metadata = { 'meter_devices': {TEST_METER['model']: TEST_METER}} print(store.root._v_attrs.metadata) # Building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def test_pipeline(self): meter1 = ElecMeter(store=self.datastore, metadata=self.meter_meta, meter_id=METER_ID) # load co_test.h5 dataset = DataSet(join(data_dir(), 'co_test.h5')) meter2 = dataset.buildings[1].elec.mains() for meter in [meter1, meter2]: for chunksize in [None, 2**10, 2**29]: if chunksize is None: load_kwargs = {} else: load_kwargs = {'chunksize': chunksize} source_node = meter.get_source_node(**load_kwargs) good_sections = GoodSections(source_node) good_sections.run() combined = good_sections.results #.simple() meter.clear_cache() meter.good_sections(**load_kwargs) meter.good_sections(**load_kwargs) meter.clear_cache() dataset.store.close()
def teardown_package(): """Nosetests package teardown function (run when tests are done). See http://nose.readthedocs.org/en/latest/writing_tests.html#test-packages Closes remaining open HDF5 files to avoid warnings and resets the files in data_dir to their original states. """ from nilmtk.tests.testingtools import data_dir # Workaround for open .h5 files on Windows from tables.file import _open_files _open_files.close_all() # Restore the original copies of the data files from glob import glob from os.path import join, isdir import os, shutil for original_fn in glob(join(data_dir(), '*.original')): target_fn = original_fn[:original_fn.rfind('.original')] try: if isdir(target_fn): shutil.rmtree(target_fn) else: os.remove(target_fn) os.rename(original_fn, target_fn) except: warnings.warn( 'Could not restore file or directory "{}"'.format(target_fn))
def teardown_package(): """Nosetests package teardown function (run when tests are done). See http://nose.readthedocs.org/en/latest/writing_tests.html#test-packages Uses git to reset data_dir after tests have run. """ from nilmtk.tests.testingtools import data_dir import subprocess cmd = "git checkout -- {}".format(data_dir()) try: output = subprocess.check_output(cmd, shell=True, cwd=data_dir()) except Exception: print "Failed to run '{}'".format(cmd) raise else: print "Succeeded in running '{}'".format(cmd)
def create_energy_hdf5(simple=True): fname = 'energy.h5' if simple else 'energy_complex.h5' FILENAME = join(data_dir(), fname) df = power_data(simple=simple) meter_device = { 'manufacturer': 'Test Manufacturer', 'model': 'Energy Meter', 'sample_period': 10, 'max_sample_period': MAX_SAMPLE_PERIOD, 'measurements': [] } for physical_quantity, ac_type in df.columns.tolist(): meter_device['measurements'].append({ 'physical_quantity': physical_quantity, 'type': ac_type, 'lower_limit': 0, 'upper_limit': 6000 }) store = pd.HDFStore(FILENAME, 'w', complevel=9, complib='zlib') elec_meter_metadata = {} # Save sensor data for meter_i in [1, 2, 3]: key = 'building1/elec/meter{:d}'.format(meter_i) print("Saving", key) store.put(key, df, format='table') meta = {'device_model': meter_device['model'], 'data_location': key} additional_meta = { 1: { 'site_meter': True }, 2: { 'submeter_of': 1 }, 3: { 'submeter_of': 2 } } meta.update(additional_meta[meter_i]) elec_meter_metadata[meter_i] = meta # Save dataset-wide metadata store.root._v_attrs.metadata = { 'meter_devices': { meter_device['model']: meter_device } } # Add building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def create_co_test_hdf5(): FILENAME = join(data_dir(), 'co_test.h5') N_METERS = 3 chunk = 1000 N_PERIODS = 4 * chunk rng = pd.date_range('2012-01-01', freq='S', periods=N_PERIODS) dfs = OrderedDict() data = OrderedDict() # mains meter data data[1] = np.array([0, 200, 1000, 1200] * chunk) # appliance 1 data data[2] = np.array([0, 200, 0, 200] * chunk) # appliance 2 data data[3] = np.array([0, 0, 1000, 1000] * chunk) for i in range(1, 4): dfs[i] = pd.DataFrame(data=data[i], index=rng, dtype=np.float32, columns=measurement_columns([('power', 'active') ])) store = pd.HDFStore(FILENAME, 'w', complevel=9, complib='zlib') elec_meter_metadata = {} for meter in range(1, N_METERS + 1): key = 'building1/elec/meter{:d}'.format(meter) print("Saving", key) store.put(key, dfs[meter], format='table') elec_meter_metadata[meter] = { 'device_model': TEST_METER['model'], 'submeter_of': 1, 'data_location': key } # For mains meter, we need to specify that it is a site meter del elec_meter_metadata[1]['submeter_of'] elec_meter_metadata[1]['site_meter'] = True # Save dataset-wide metadata store.root._v_attrs.metadata = { 'meter_devices': { TEST_METER['model']: TEST_METER } } print(store.root._v_attrs.metadata) # Building metadata add_building_metadata(store, elec_meter_metadata) for key in store.keys(): print(store[key]) store.flush() store.close()
def teardown_package(): """Nosetests package teardown function (run when tests are done). See http://nose.readthedocs.org/en/latest/writing_tests.html#test-packages Uses git to reset data_dir after tests have run. """ from nilmtk.tests.testingtools import data_dir import subprocess #Workaround for open .h5 files on Windows from tables.file import _open_files _open_files.close_all() cmd = "git checkout -- {}".format(data_dir()) try: subprocess.check_output(cmd, shell=True, cwd=data_dir()) except Exception: print("Failed to run '{}'".format(cmd)) raise else: print("Succeeded in running '{}'".format(cmd))
def test_load(self): filename = join(data_dir(), 'energy.h5') ds = DataSet(filename) elec = ds.buildings[1].elec df = next(elec.load()) self.assertEqual(len(df), 13) df = next(elec.load(chunksize=5)) self.assertEqual(len(df), 5) df = next(elec.load(physical_quantity='energy')) self.assertEqual(len(df), 13) self.assertEqual(df.columns.levels, [['energy'], ['reactive']]) df = next(elec.load(ac_type='active')) self.assertEqual(df.columns.levels, [['power'], ['active']])
def create_energy_hdf5(simple=True): fname = 'energy.h5' if simple else 'energy_complex.h5' FILENAME = join(data_dir(), fname) df = power_data(simple=simple) meter_device = { 'manufacturer': 'Test Manufacturer', 'model': 'Energy Meter', 'sample_period': 10, 'max_sample_period': MAX_SAMPLE_PERIOD, 'measurements': [] } for physical_quantity, ac_type in df.columns.tolist(): meter_device['measurements'].append({ 'physical_quantity': physical_quantity, 'type': ac_type, 'lower_limit': 0, 'upper_limit': 6000}) store = pd.HDFStore(FILENAME, 'w', complevel=9, complib='zlib') elec_meter_metadata = {} # Save sensor data for meter_i in [1,2,3]: key = 'building1/elec/meter{:d}'.format(meter_i) print("Saving", key) store.put(key, df, format='table') meta = { 'device_model': meter_device['model'], 'data_location': key } additional_meta = { 1: {'site_meter': True}, 2: {'submeter_of': 1}, 3: {'submeter_of': 2} } meta.update(additional_meta[meter_i]) elec_meter_metadata[meter_i] = meta # Save dataset-wide metadata store.root._v_attrs.metadata = {'meter_devices': {meter_device['model']: meter_device}} # Add building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def create_co_test_hdf5(): FILENAME = join(data_dir(), "co_test.h5") N_METERS = 3 chunk = 1000 N_PERIODS = 4 * chunk rng = pd.date_range("2012-01-01", freq="S", periods=N_PERIODS) dfs = OrderedDict() data = OrderedDict() # mains meter data data[1] = np.array([0, 200, 1000, 1200] * chunk) # appliance 1 data data[2] = np.array([0, 200, 0, 200] * chunk) # appliance 2 data data[3] = np.array([0, 0, 1000, 1000] * chunk) for i in range(1, 4): dfs[i] = pd.DataFrame( data=data[i], index=rng, dtype=np.float32, columns=measurement_columns([("power", "active")]) ) store = pd.HDFStore(FILENAME, "w", complevel=9, complib="zlib") elec_meter_metadata = {} for meter in range(1, N_METERS + 1): key = "building1/elec/meter{:d}".format(meter) print("Saving", key) store.put(key, dfs[meter], format="table") elec_meter_metadata[meter] = {"device_model": TEST_METER["model"], "submeter_of": 1, "data_location": key} # For mains meter, we need to specify that it is a site meter del elec_meter_metadata[1]["submeter_of"] elec_meter_metadata[1]["site_meter"] = True # Save dataset-wide metadata store.root._v_attrs.metadata = {"meter_devices": {TEST_METER["model"]: TEST_METER}} print(store.root._v_attrs.metadata) # Building metadata add_building_metadata(store, elec_meter_metadata) for key in store.keys(): print(store[key]) store.flush() store.close()
def create_energy_hdf5(simple=True): fname = "energy.h5" if simple else "energy_complex.h5" FILENAME = join(data_dir(), fname) df = power_data(simple=simple) meter_device = { "manufacturer": "Test Manufacturer", "model": "Energy Meter", "sample_period": 10, "max_sample_period": MAX_SAMPLE_PERIOD, "measurements": [], } for physical_quantity, ac_type in df.columns.tolist(): meter_device["measurements"].append( {"physical_quantity": physical_quantity, "type": ac_type, "lower_limit": 0, "upper_limit": 6000} ) store = pd.HDFStore(FILENAME, "w", complevel=9, complib="zlib") elec_meter_metadata = {} # Save sensor data for meter_i in [1, 2, 3]: key = "building1/elec/meter{:d}".format(meter_i) print("Saving", key) store.put(key, df, format="table") meta = {"device_model": meter_device["model"], "data_location": key} additional_meta = {1: {"site_meter": True}, 2: {"submeter_of": 1}, 3: {"submeter_of": 2}} meta.update(additional_meta[meter_i]) elec_meter_metadata[meter_i] = meta # Save dataset-wide metadata store.root._v_attrs.metadata = {"meter_devices": {meter_device["model"]: meter_device}} # Add building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def create_random_hdf5(): FILENAME = join(data_dir(), "random.h5") N_METERS = 5 store = pd.HDFStore(FILENAME, "w", complevel=9, complib="zlib") elec_meter_metadata = {} for meter in range(1, N_METERS + 1): key = "building1/elec/meter{:d}".format(meter) print("Saving", key) store.put(key, create_random_df(), format="table") elec_meter_metadata[meter] = {"device_model": TEST_METER["model"], "submeter_of": 1, "data_location": key} # Save dataset-wide metadata store.root._v_attrs.metadata = {"meter_devices": {TEST_METER["model"]: TEST_METER}} print(store.root._v_attrs.metadata) # Building metadata add_building_metadata(store, elec_meter_metadata) store.flush() store.close()
def setUpClass(cls): filename = join(data_dir(), 'energy.h5') cls.datastore = HDFDataStore(filename) ElecMeter.load_meter_devices(cls.datastore)
def setUpClass(cls): filename = join(data_dir(), 'energy_complex.h5') cls.datastore = HDFDataStore(filename) ElecMeter.load_meter_devices(cls.datastore) cls.meter_meta = cls.datastore.load_metadata('building1')['elec_meters'][METER_ID.instance]
def setUpClass(cls): filename = join(data_dir(), 'co_test.h5') cls.dataset = DataSet(filename)
def setUpClass(cls): filename = join(data_dir(), 'energy_complex.h5') cls.datastore = HDFDataStore(filename) ElecMeter.load_meter_devices(cls.datastore) cls.meter_meta = cls.datastore.load_metadata( 'building1')['elec_meters'][METER_ID.instance]
def setUpClass(cls): filename = join(data_dir(), 'random_csv') cls.datastore = CSVDataStore(filename) cls.keys = ['/building1/elec/meter{:d}'.format(i) for i in range(1, 6)]
def test_total_energy(self): filename = join(data_dir(), 'random.h5') ds = DataSet(filename) ds.buildings[1].elec.total_energy()
def teardown_package(): """Nosetests package teardown function (run when tests are done). See http://nose.readthedocs.org/en/latest/writing_tests.html#test-packages Closes remaining open HDF5 files to avoid warnings and resets the files in data_dir to their original states. """ from nilmtk.tests.testingtools import data_dir # Workaround for open .h5 files on Windows from tables.file import _open_files _open_files.close_all() # Restore the original copies of the data files from glob import glob from os.path import join, isdir import os, shutil for original_fn in glob(join(data_dir(), '*.original')): target_fn = original_fn[:original_fn.rfind('.original')] try: if isdir(target_fn): shutil.rmtree(target_fn) else: os.remove(target_fn) os.rename(original_fn, target_fn) except: warnings.warn('Could not restore file or directory "{}"'.format(target_fn))
def test_pipeline(self): meter1 = ElecMeter(store=self.datastore, metadata=self.meter_meta, meter_id=METER_ID) # load co_test.h5 dataset = DataSet(join(data_dir(), 'co_test.h5')) meter2 = dataset.buildings[1].elec.mains() for meter in [meter1, meter2]: for chunksize in [None, 2**10, 2**29]: if chunksize is None: kwargs = {} else: kwargs = {'chunksize': chunksize} source_node = meter.get_source_node(**kwargs) good_sections = GoodSections(source_node) good_sections.run() combined = good_sections.results.simple() meter.clear_cache() meter.good_sections(**kwargs) meter.good_sections(**kwargs) meter.clear_cache() dataset.store.close()