def main():
# set up logging
logger = logging.getLogger('store_data_locally')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('store_data_locally.log')
fh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)
# get arguments
parser = argparse.ArgumentParser(
description='Query sensor readings from the IDEAL database'
'and store locally.')
parser.add_argument('--dataset_path',
help='directory of the original IDEAL dataset')
parser.add_argument('--data_path',
default=LOCAL_DATA_DIR,
help='directory to store data')
args = parser.parse_args()
# store metadata locally
converter = IdealCSV2Hdf5(args.dataset_path, data_dir=args.data_path)
converter.store_metadata()
with MetaDataStore(data_dir=args.data_path) as s:
metadata = MetaData(s)
# get relevant sensorids
sensors = metadata.sensor_merged()
indices = pd.Series([False] * sensors.shape[0], index=sensors.index.copy())
indices = indices | sensors.sensorid.isin(metadata.electric_sensors())
indices = indices & sensors.homeid.astype(int).isin(metadata.gold_homes())
sensorids = sensors.sensorid[indices]
sensorids_to_store = sensorids
print('Query and store readings from {0} sensors'.format(
len(sensorids_to_store)))
for idx, sensorid in enumerate(sensorids_to_store):
converter = IdealCSV2Hdf5(args.dataset_path, data_dir=args.data_path)
logger.info('({0}/{1}) Sensorid: {2}'.format(idx + 1,
len(sensorids_to_store),
sensorid))
converter.store_readings(sensorid)
# try and read stored data
readings_store = ReadingDataStore(data_dir=args.data_path)
readings_count = 0
for idx, sensorid in enumerate(sensorids):
readings = readings_store.get_sensor_readings(sensorid)
readings_count += len(readings)
logger.info('Total readings : {0}'.format(readings_count))
description='Clean electrical sensor readings and merge .')
parser.add_argument('--home',
type=int,
default=-1,
help='home to process, default all')
parser.add_argument('--enddate',
type=valid_date,
default=None,
help='use only readings before this date')
args = parser.parse_args()
end_date = args.enddate
with MetaDataStore() as s:
metadata = MetaData(s)
home_reading_store = HomeReadingStore()
preprocessor = ElecPreprocessor()
if args.home == -1:
homeids = metadata.gold_homes()
else:
homeids = [args.home]
for homeid in homeids:
print('homeid: {0}'.format(homeid))
readings = preprocessor.get_home_readings(homeid)
if end_date is not None:
readings = readings[readings.index < end_date]
home_reading_store.set_readings(homeid, readings)