def generate_data(household, start, end): raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=household["appliance_status"], iIrradianceThreshold = household["lights_irradiance"], iRandomHouse = household["lights_house"], from_date=start, to_date=end) header = raw_data[0][3:] return [{'sensor_name': header[i], 'usage': v*1000, 'n_measurements': 1, 'timestamp': start} for i, v in enumerate(raw_data[1][3:])]
def generate_averages(household, appliance_status, from_date, to_date): raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold = household["lights_irradiance"], iRandomHouse = household["lights_house"], from_date=from_date, to_date=to_date) matrix = numpy.array(raw_data[1:]) matrix = matrix[:, 3:] header = raw_data[0][3:] return [{'sensor_name': header[i], 'average': v*1000, 'count': (len(raw_data)-1)} for i, v in enumerate(numpy.mean(matrix, axis=0))]
def generate_sensor_data(config_file: str, from_date: datetime, to_date: datetime): print("generate_sensor_date(config_file=%s, from=%s, to=%s" % (config_file, from_date, to_date)) random.seed(datetime.now()) # different result each time json_file = open(config_file, "r") config_data = json.load(json_file) json_file.close() config = None for config in config_data: appliance_status = [] # Edits by Evert print("Loading household {}".format(config["id_household"])) sensors = [] for name in config["appliances"]: sensor = Sensor(name) sensors.append(sensor) print( " - loaded sensor with ID {s.ID} with appliance name `{s.appname}`" .format(s=sensor)) lightSensor = LightSensor(config["lights_id"]) #for row in appliance.appliances: #status = "NO" #name = row[0] #if name in sensors: #status = "YES" #appliance_status.append(status) data = generate_data.generate_data_range( iResidents=config["no_residents"], Sensors=sensors, lightSensor=lightSensor, iIrradianceThreshold=config["lights_irradiance"], iRandomHouse=config["lights_house"], from_date=from_date, to_date=to_date) outputfile = "data_house_{}.csv".format(config["id_household"]) save_file = open(outputfile, 'w') for row in data: for col in row: save_file.write(str(col)) save_file.write(";") save_file.write("\n") save_file.close() print("saved %s" % outputfile)
def generate_sensor_data(config_file: str, from_date: datetime, to_date: datetime): print("generate_sensor_date(config_file=%s, from=%s, to=%s" % ( config_file, from_date, to_date)) random.seed(datetime.now()) # different result each time json_file = open(config_file, "r") config_data = json.load(json_file) json_file.close() config = None for config in config_data: appliance_status = [] # Edits by Evert print("Loading household {}".format(config["id_household"])) sensors = [] for name in config["appliances"]: sensor = Sensor(name) sensors.append(sensor) print(" - loaded sensor with ID {s.ID} with appliance name `{s.appname}`".format(s=sensor)) lightSensor = LightSensor(config["lights_id"]) #for row in appliance.appliances: #status = "NO" #name = row[0] #if name in sensors: #status = "YES" #appliance_status.append(status) data = generate_data.generate_data_range( iResidents=config["no_residents"], Sensors=sensors, lightSensor=lightSensor, iIrradianceThreshold=config["lights_irradiance"], iRandomHouse=config["lights_house"], from_date=from_date, to_date=to_date) outputfile = "data_house_{}.csv".format(config["id_household"]) save_file = open(outputfile, 'w') for row in data: for col in row: save_file.write(str(col)) save_file.write(";") save_file.write("\n") save_file.close() print("saved %s" % outputfile)
def generate_averages(household, appliance_status, from_date, to_date): raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold=household["lights_irradiance"], iRandomHouse=household["lights_house"], from_date=from_date, to_date=to_date) matrix = numpy.array(raw_data[1:]) matrix = matrix[:, 3:] header = raw_data[0][3:] return [{ 'sensor_name': header[i], 'average': v * 1000 } for i, v in enumerate(numpy.mean(matrix, axis=0))]
def populate_data(config, from_date, to_date): data = {} for household in config: household_id = household['id_household'] print("Generating household " + str(household_id)) appliance_status = [] for row in appliance.appliances: status = "NO" if row[0] in household["appliances"]: status = "YES" appliance_status.append(status) data[household_id] = { 'yearly':{}, 'monthly':{}, 'daily':{}, 'recent':{} } for year in range(from_date.year, to_date.year): start_date = datetime(year, 1, 1) data[household_id]['yearly'][start_date] = generate_averages( household, appliance_status, from_date, datetime(year+1, 1, 1) ) for month in range(1, to_date.month): start_date = datetime(to_date.year, month, 1) if start_date < from_date: continue data[household_id]['monthly'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, month+1, 1) ) for day in range(1, to_date.day): start_date = datetime(to_date.year, to_date.month, day) if start_date < from_date: continue data[household_id]['daily'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, to_date.month, day+1) ) start_date = max(to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date) raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold = household["lights_irradiance"], iRandomHouse = household["lights_house"], from_date=start_date, to_date=to_date) header = raw_data[0][3:] data[household_id]['recent'] = [ {v[0]:[{'sensor_name':header[i], 'average': v[i]*1000} for i in range(3, len(header))]} for v in raw_data[1:] ] return data
def populate_data(config, from_date, to_date): data = {} for household in config: household_id = household['id_household'] print("Generating household " + str(household_id)) appliance_status = [] for row in appliance.appliances: status = "NO" if row[0] in household["appliances"]: status = "YES" appliance_status.append(status) data[household_id] = { 'yearly': {}, 'monthly': {}, 'daily': {}, 'recent': {} } for year in range(from_date.year, to_date.year): start_date = datetime(year, 1, 1) data[household_id]['yearly'][start_date] = generate_averages( household, appliance_status, from_date, datetime(year + 1, 1, 1)) for month in range(1, to_date.month): start_date = datetime(to_date.year, month, 1) if start_date < from_date: continue data[household_id]['monthly'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, month + 1, 1)) for day in range(1, to_date.day): start_date = datetime(to_date.year, to_date.month, day) if start_date < from_date: continue data[household_id]['daily'][start_date] = generate_averages( household, appliance_status, start_date, datetime(to_date.year, to_date.month, day + 1)) start_date = max( to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date) raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold=household["lights_irradiance"], iRandomHouse=household["lights_house"], from_date=start_date, to_date=to_date) header = raw_data[0][3:] data[household_id]['recent'] = [{ v[0]: [{ 'sensor_name': header[i], 'average': v[i] * 1000 } for i in range(3, len(header))] } for v in raw_data[1:]] return data
def populate_data(config, from_date, to_date): data = {'yearly':[], 'monthly':[], 'daily':[], 'recent':[]} for household in config: household_id = household['id_household'] print("Generating household " + str(household_id)) appliance_status = [] for row in appliance.appliances: status = "NO" if row[0] in household["appliances"]: status = "YES" appliance_status.append(status) sensor_list = Sensor.objects.filter(home=household_id).values('name', 'id') sensor_map = {v['name']:v['id'] for v in sensor_list} for year in range(from_date.year, to_date.year): start_date = datetime(year, 1, 1) averages = generate_averages( household, appliance_status, start_date, datetime(year+1, 1, 1) ) data['yearly'] += [YearlyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages] monthly_start = to_date.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(years=1) for month in range(12): start_date = monthly_start + relativedelta(months=month) stop_date = monthly_start + relativedelta(months=month+1) if start_date < from_date: continue averages = generate_averages( household, appliance_status, start_date, stop_date ) data['monthly'] += [MonthlyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages] daily_start = to_date.replace(hour=0, minute=0, second=0, microsecond=0) - relativedelta(months=1) for day in range((to_date-daily_start).days): start_date = daily_start + timedelta(days=day) stop_date = daily_start + timedelta(days=day+1) if start_date < from_date: continue averages = generate_averages( household, appliance_status, start_date, stop_date ) data['daily'] += [DailyData(sensor_id=sensor_map[ v['sensor_name'] ], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages] start_date = max(to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date) raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold = household["lights_irradiance"], iRandomHouse = household["lights_house"], from_date=start_date, to_date=to_date) header = raw_data[0][3:] for v in raw_data[1:]: data['recent'] += [ RecentData(sensor_id=sensor_map[ header[i] ], usage=v[i]*1000, n_measurements=1, timestamp=v[0]) for i in range(3, len(header)) ] #data_count = len(data) #print("Saving " + str(data_count) + " objects...") #data_1000 = data_count/1000 #for i, datum in enumerate(data): # if ((i % data_1000) == 0): # print(str(datetime.now()) + " - progress: " + str(float(i*100)/float(data_count)) + "%") # super(SensorData, datum).save() print("Saving " + str(len(data['yearly'])) + " yearly data entries...") YearlyData.objects.bulk_create(data['yearly']) print("Saving " + str(len(data['monthly'])) + " monthly data entries...") MonthlyData.objects.bulk_create(data['monthly']) print("Saving " + str(len(data['daily'])) + " daily data entries...") DailyData.objects.bulk_create(data['daily']) print("Saving " + str(len(data['recent'])) + " recent data entries...") RecentData.objects.bulk_create(data['recent']) return data
def populate_data(config, from_date, to_date): data = {'yearly': [], 'monthly': [], 'daily': [], 'recent': []} for household in config: household_id = household['id_household'] print("Generating household " + str(household_id)) appliance_status = [] for row in appliance.appliances: status = "NO" if row[0] in household["appliances"]: status = "YES" appliance_status.append(status) sensor_list = Sensor.objects.filter(home=household_id).values( 'name', 'id') sensor_map = {v['name']: v['id'] for v in sensor_list} for year in range(from_date.year, to_date.year): start_date = datetime(year, 1, 1) averages = generate_averages(household, appliance_status, start_date, datetime(year + 1, 1, 1)) data['yearly'] += [ YearlyData(sensor_id=sensor_map[v['sensor_name']], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages ] monthly_start = to_date.replace( day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(years=1) for month in range(12): start_date = monthly_start + relativedelta(months=month) stop_date = monthly_start + relativedelta(months=month + 1) if start_date < from_date: continue averages = generate_averages(household, appliance_status, start_date, stop_date) data['monthly'] += [ MonthlyData(sensor_id=sensor_map[v['sensor_name']], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages ] daily_start = to_date.replace( hour=0, minute=0, second=0, microsecond=0) - relativedelta(months=1) for day in range((to_date - daily_start).days): start_date = daily_start + timedelta(days=day) stop_date = daily_start + timedelta(days=day + 1) if start_date < from_date: continue averages = generate_averages(household, appliance_status, start_date, stop_date) data['daily'] += [ DailyData(sensor_id=sensor_map[v['sensor_name']], usage=v['average'], n_measurements=v['count'], timestamp=start_date) for v in averages ] start_date = max( to_date.replace(hour=0, minute=0, second=0, microsecond=0), from_date) raw_data = generate_data.generate_data_range( iResidents=household["no_residents"], Dwell=appliance_status, iIrradianceThreshold=household["lights_irradiance"], iRandomHouse=household["lights_house"], from_date=start_date, to_date=to_date) header = raw_data[0][3:] for v in raw_data[1:]: data['recent'] += [ RecentData(sensor_id=sensor_map[header[i]], usage=v[i] * 1000, n_measurements=1, timestamp=v[0]) for i in range(3, len(header)) ] #data_count = len(data) #print("Saving " + str(data_count) + " objects...") #data_1000 = data_count/1000 #for i, datum in enumerate(data): # if ((i % data_1000) == 0): # print(str(datetime.now()) + " - progress: " + str(float(i*100)/float(data_count)) + "%") # super(SensorData, datum).save() print("Saving " + str(len(data['yearly'])) + " yearly data entries...") YearlyData.objects.bulk_create(data['yearly']) print("Saving " + str(len(data['monthly'])) + " monthly data entries...") MonthlyData.objects.bulk_create(data['monthly']) print("Saving " + str(len(data['daily'])) + " daily data entries...") DailyData.objects.bulk_create(data['daily']) print("Saving " + str(len(data['recent'])) + " recent data entries...") RecentData.objects.bulk_create(data['recent']) return data